@apifuse/provider-sdk 2.2.0-beta.44 → 2.2.0-beta.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +2 -1
- package/bin/apifuse-check.ts +14 -5
- package/dist/declaration-validation.d.ts +2 -1
- package/dist/declaration-validation.js +59 -0
- package/dist/define.d.ts +4 -0
- package/dist/lint.d.ts +15 -0
- package/dist/lint.js +171 -2
- package/dist/runtime/insights.js +2 -2
- package/dist/runtime/stealth.d.ts +3 -1
- package/dist/runtime/stealth.js +187 -22
- package/dist/server/serve-implementation.js +6 -4
- package/dist/stealth/profiles.d.ts +4 -0
- package/dist/stealth/profiles.js +118 -15
- package/dist/types.d.ts +4 -0
- package/package.json +1 -1
- package/src/declaration-validation.ts +65 -1
- package/src/define.ts +4 -0
- package/src/lint.ts +240 -2
- package/src/runtime/insights.ts +2 -2
- package/src/runtime/stealth.ts +252 -26
- package/src/server/serve-implementation.ts +6 -4
- package/src/stealth/profiles.ts +135 -16
- package/src/types.ts +4 -0
package/dist/runtime/stealth.js
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { DEFAULT_SMARTPROXY_POOL_SIZE, invalidateProxyResolutionCacheAsync, ProxyResolutionError, policyResolvesRegistryVendorChain, resolvePolicyProxyPoolSpan, resolvePolicyTransportAttemptCap, resolveProxyConfigAsync, vendorFromResolvedSource, } from "../config/loader.js";
|
|
3
3
|
import { SDKError, TransportError } from "../errors.js";
|
|
4
|
-
import { getStealthProfile } from "../stealth/profiles.js";
|
|
4
|
+
import { getStealthProfile, getStealthProfileIntentAlias } from "../stealth/profiles.js";
|
|
5
5
|
import { StealthCookieJar } from "./stealth-cookies.js";
|
|
6
6
|
import { createProxyAuthIpDeniedError, createProxyEdgeAuthRejectedError, createProxyEdgeTlsRejectedError, createProxyPoolExhaustedError, createProxyPoolStaleError, isProxyAuthIpDeniedMessage, isProxyEdgeAuthRejectedMessage, isProxyEdgeTlsRejectedResponse, isProxyPoolRefreshableError, isProxyPoolStaleMessage, isProxyPoolStaleStatus, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_STALE_CODE, } from "./proxy-errors.js";
|
|
7
7
|
import { computeProxyAttemptIndex, computeProxyTransportRetryDelayMs, createDefaultProxyTransportRetryOptions, normalizeProxyTransportRetryOptions, shouldRetryProxyTransportAttempt, validateUnsafeProxyTransportRetryMethods, } from "./proxy-retry-policy.js";
|
|
8
8
|
import { evaluateRedirectHop, isRedirectStatus, nextRedirectMethod, resolveRedirectUrl, } from "./redirects.js";
|
|
9
9
|
import { isSensitiveKey, normalizeSensitiveParams, redactSensitiveError, redactSensitiveRequestError, redactSensitiveText, redactUrlQueryParams, serializeRequestUrl, } from "./request-options.js";
|
|
10
|
-
export const DEFAULT_PROFILE = "chrome-
|
|
10
|
+
export const DEFAULT_PROFILE = "chrome-desktop";
|
|
11
11
|
const MISSING_PROXY_WARNING = "[provider-sdk] Provider requested proxy routing, but no proxy URL was configured. Continuing without proxy.";
|
|
12
12
|
const MAX_POLICY_PROXY_POOL_REFRESHES = 1;
|
|
13
13
|
const PROXY_CONNECT_FAILURE_CODE = "proxy_connect_failed";
|
|
@@ -108,7 +108,11 @@ function resolveDefaultWreqProfileMapping() {
|
|
|
108
108
|
}
|
|
109
109
|
return { identifier, os: profile.platform };
|
|
110
110
|
}
|
|
111
|
-
|
|
111
|
+
let defaultWreqProfileMapping;
|
|
112
|
+
function getDefaultWreqProfileMapping() {
|
|
113
|
+
defaultWreqProfileMapping ??= resolveDefaultWreqProfileMapping();
|
|
114
|
+
return defaultWreqProfileMapping;
|
|
115
|
+
}
|
|
112
116
|
export function resolveWreqProfile(profileName, wreqProfiles) {
|
|
113
117
|
if (REMOVED_CHROME_PROFILE_NAMES.has(profileName)) {
|
|
114
118
|
throw new SDKError(`Unknown stealth profile: ${profileName}`);
|
|
@@ -125,8 +129,9 @@ export function resolveWreqProfile(profileName, wreqProfiles) {
|
|
|
125
129
|
// profile strings still run with the transport default instead of failing
|
|
126
130
|
// before the request starts. Removed built-in profile aliases above remain
|
|
127
131
|
// explicit errors so callers do not accidentally pin retired fingerprints.
|
|
128
|
-
|
|
129
|
-
|
|
132
|
+
const defaultMapping = getDefaultWreqProfileMapping();
|
|
133
|
+
identifier = defaultMapping.identifier;
|
|
134
|
+
os = defaultMapping.os;
|
|
130
135
|
}
|
|
131
136
|
const browser = closestWreqProfile(identifier, wreqProfiles);
|
|
132
137
|
if (!browser) {
|
|
@@ -201,11 +206,14 @@ function splitCombinedSetCookieHeader(headerValue) {
|
|
|
201
206
|
return cookieStrings;
|
|
202
207
|
}
|
|
203
208
|
export async function normalizeResponse(response, requestUrl, maxBodyBytes) {
|
|
209
|
+
return normalizeResponseWithSignal(response, requestUrl, maxBodyBytes);
|
|
210
|
+
}
|
|
211
|
+
async function normalizeResponseWithSignal(response, requestUrl, maxBodyBytes, signal) {
|
|
204
212
|
const headers = Object.fromEntries(response.headers.entries());
|
|
205
213
|
const cookies = new StealthCookieJar(setCookieHeadersFromResponse(response.headers), response.url ?? requestUrl);
|
|
206
214
|
const bodyBytes = maxBodyBytes === undefined
|
|
207
|
-
? await response
|
|
208
|
-
: await readResponseBodyWithLimit(response, maxBodyBytes);
|
|
215
|
+
? await readResponseArrayBuffer(response, signal)
|
|
216
|
+
: await readResponseBodyWithLimit(response, maxBodyBytes, signal);
|
|
209
217
|
const body = new TextDecoder().decode(bodyBytes);
|
|
210
218
|
return {
|
|
211
219
|
status: response.status,
|
|
@@ -231,6 +239,40 @@ export async function normalizeResponse(response, requestUrl, maxBodyBytes) {
|
|
|
231
239
|
},
|
|
232
240
|
};
|
|
233
241
|
}
|
|
242
|
+
async function readResponseArrayBuffer(response, signal) {
|
|
243
|
+
if (!signal)
|
|
244
|
+
return response.arrayBuffer();
|
|
245
|
+
throwIfAmbientAborted(signal);
|
|
246
|
+
return new Promise((resolve, reject) => {
|
|
247
|
+
let settled = false;
|
|
248
|
+
const settle = (operation) => {
|
|
249
|
+
if (settled)
|
|
250
|
+
return;
|
|
251
|
+
settled = true;
|
|
252
|
+
signal.removeEventListener("abort", onAbort);
|
|
253
|
+
operation();
|
|
254
|
+
};
|
|
255
|
+
const onAbort = () => {
|
|
256
|
+
const error = toAmbientCancellationError(signal);
|
|
257
|
+
try {
|
|
258
|
+
void response.body?.cancel().catch(() => undefined);
|
|
259
|
+
}
|
|
260
|
+
catch {
|
|
261
|
+
// Preserve the cancellation error if accessing or cancelling the body fails.
|
|
262
|
+
}
|
|
263
|
+
settle(() => reject(error));
|
|
264
|
+
};
|
|
265
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
266
|
+
try {
|
|
267
|
+
void response.arrayBuffer().then((body) => settle(() => resolve(body)), (error) => settle(() => reject(error)));
|
|
268
|
+
}
|
|
269
|
+
catch (error) {
|
|
270
|
+
settle(() => reject(error));
|
|
271
|
+
}
|
|
272
|
+
if (signal.aborted)
|
|
273
|
+
onAbort();
|
|
274
|
+
});
|
|
275
|
+
}
|
|
234
276
|
function responseTooLargeError(maxBodyBytes, observedBytes) {
|
|
235
277
|
return new TransportError(`Response body exceeded maxBodyBytes limit of ${maxBodyBytes} bytes (observed ${observedBytes} bytes)`, {
|
|
236
278
|
code: "response_too_large",
|
|
@@ -246,7 +288,8 @@ function declaredContentLength(headers) {
|
|
|
246
288
|
const parsed = Number(contentLength);
|
|
247
289
|
return Number.isFinite(parsed) ? parsed : undefined;
|
|
248
290
|
}
|
|
249
|
-
async function readResponseBodyWithLimit(response, maxBodyBytes) {
|
|
291
|
+
async function readResponseBodyWithLimit(response, maxBodyBytes, signal) {
|
|
292
|
+
throwIfAmbientAborted(signal);
|
|
250
293
|
const contentLength = declaredContentLength(response.headers);
|
|
251
294
|
if (contentLength !== undefined && contentLength > maxBodyBytes) {
|
|
252
295
|
await response.body?.cancel().catch(() => undefined);
|
|
@@ -264,7 +307,7 @@ async function readResponseBodyWithLimit(response, maxBodyBytes) {
|
|
|
264
307
|
let receivedBytes = 0;
|
|
265
308
|
try {
|
|
266
309
|
while (true) {
|
|
267
|
-
const { done, value } = await reader
|
|
310
|
+
const { done, value } = await readResponseBodyChunk(reader, signal);
|
|
268
311
|
if (done)
|
|
269
312
|
break;
|
|
270
313
|
if (!value)
|
|
@@ -280,6 +323,9 @@ async function readResponseBodyWithLimit(response, maxBodyBytes) {
|
|
|
280
323
|
finally {
|
|
281
324
|
reader.releaseLock();
|
|
282
325
|
}
|
|
326
|
+
return concatenateResponseBodyChunks(chunks, receivedBytes);
|
|
327
|
+
}
|
|
328
|
+
function concatenateResponseBodyChunks(chunks, receivedBytes) {
|
|
283
329
|
const bodyBytes = new Uint8Array(receivedBytes);
|
|
284
330
|
let offset = 0;
|
|
285
331
|
for (const chunk of chunks) {
|
|
@@ -288,6 +334,30 @@ async function readResponseBodyWithLimit(response, maxBodyBytes) {
|
|
|
288
334
|
}
|
|
289
335
|
return bodyBytes.buffer;
|
|
290
336
|
}
|
|
337
|
+
function readResponseBodyChunk(reader, signal) {
|
|
338
|
+
if (!signal)
|
|
339
|
+
return reader.read();
|
|
340
|
+
throwIfAmbientAborted(signal);
|
|
341
|
+
return new Promise((resolve, reject) => {
|
|
342
|
+
let settled = false;
|
|
343
|
+
const settle = (operation) => {
|
|
344
|
+
if (settled)
|
|
345
|
+
return;
|
|
346
|
+
settled = true;
|
|
347
|
+
signal.removeEventListener("abort", onAbort);
|
|
348
|
+
operation();
|
|
349
|
+
};
|
|
350
|
+
const onAbort = () => {
|
|
351
|
+
const error = toAmbientCancellationError(signal);
|
|
352
|
+
void reader.cancel().catch(() => undefined);
|
|
353
|
+
settle(() => reject(error));
|
|
354
|
+
};
|
|
355
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
356
|
+
void reader.read().then((chunk) => settle(() => resolve(chunk)), (error) => settle(() => reject(error)));
|
|
357
|
+
if (signal.aborted)
|
|
358
|
+
onAbort();
|
|
359
|
+
});
|
|
360
|
+
}
|
|
291
361
|
function normalizeBody(body) {
|
|
292
362
|
if (body === undefined) {
|
|
293
363
|
return "";
|
|
@@ -420,8 +490,38 @@ function normalizeStealthTransportError(error) {
|
|
|
420
490
|
cause: error instanceof Error ? error : undefined,
|
|
421
491
|
});
|
|
422
492
|
}
|
|
423
|
-
function
|
|
424
|
-
|
|
493
|
+
function toAmbientCancellationError(signal, error = signal.reason) {
|
|
494
|
+
if (error instanceof TransportError && error.code === "transport_cancelled") {
|
|
495
|
+
return error;
|
|
496
|
+
}
|
|
497
|
+
return new TransportError("Request cancelled", {
|
|
498
|
+
code: "transport_cancelled",
|
|
499
|
+
status: 0,
|
|
500
|
+
retryable: false,
|
|
501
|
+
...(error !== undefined
|
|
502
|
+
? { cause: error instanceof Error ? error : new Error(String(error)) }
|
|
503
|
+
: {}),
|
|
504
|
+
});
|
|
505
|
+
}
|
|
506
|
+
function throwIfAmbientAborted(signal) {
|
|
507
|
+
if (signal?.aborted)
|
|
508
|
+
throw toAmbientCancellationError(signal);
|
|
509
|
+
}
|
|
510
|
+
function sleep(ms, signal) {
|
|
511
|
+
if (!signal)
|
|
512
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
513
|
+
throwIfAmbientAborted(signal);
|
|
514
|
+
return new Promise((resolve, reject) => {
|
|
515
|
+
const onAbort = () => {
|
|
516
|
+
clearTimeout(timer);
|
|
517
|
+
reject(toAmbientCancellationError(signal));
|
|
518
|
+
};
|
|
519
|
+
const timer = setTimeout(() => {
|
|
520
|
+
signal.removeEventListener("abort", onAbort);
|
|
521
|
+
resolve();
|
|
522
|
+
}, ms);
|
|
523
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
524
|
+
});
|
|
425
525
|
}
|
|
426
526
|
function normalizeMethod(method) {
|
|
427
527
|
switch (method.toUpperCase()) {
|
|
@@ -475,7 +575,7 @@ function discardStealthRedirectBody(response) {
|
|
|
475
575
|
// failure must not replace or delay that decision.
|
|
476
576
|
}
|
|
477
577
|
}
|
|
478
|
-
async function fetchStealthRedirectChain(transport, cookieJar, requestUrl, method, options) {
|
|
578
|
+
async function fetchStealthRedirectChain(transport, cookieJar, requestUrl, method, options, signal) {
|
|
479
579
|
let currentUrl = requestUrl;
|
|
480
580
|
let currentMethod = method;
|
|
481
581
|
let currentBody = options.body === undefined ? undefined : normalizeBody(options.body);
|
|
@@ -484,6 +584,7 @@ async function fetchStealthRedirectChain(transport, cookieJar, requestUrl, metho
|
|
|
484
584
|
let response;
|
|
485
585
|
const deadline = options.timeout ? performance.now() + options.timeout : undefined;
|
|
486
586
|
while (true) {
|
|
587
|
+
throwIfAmbientAborted(signal);
|
|
487
588
|
const headers = { ...currentHeaders };
|
|
488
589
|
if (!hasHeader(headers, "Cookie")) {
|
|
489
590
|
const cookieHeader = cookieJar.toHeader(currentUrl);
|
|
@@ -494,10 +595,12 @@ async function fetchStealthRedirectChain(transport, cookieJar, requestUrl, metho
|
|
|
494
595
|
headers: normalizeHeaders(headers),
|
|
495
596
|
method: currentMethod,
|
|
496
597
|
redirect: "manual",
|
|
598
|
+
...(signal ? { signal } : {}),
|
|
497
599
|
};
|
|
498
600
|
if (currentBody !== undefined)
|
|
499
601
|
requestInit.body = currentBody;
|
|
500
602
|
await transport.clearCookies();
|
|
603
|
+
throwIfAmbientAborted(signal);
|
|
501
604
|
const remainingTimeout = deadline === undefined ? undefined : Math.ceil(deadline - performance.now());
|
|
502
605
|
if (remainingTimeout !== undefined && remainingTimeout <= 0) {
|
|
503
606
|
throw new TransportError("Request timed out", {
|
|
@@ -508,6 +611,10 @@ async function fetchStealthRedirectChain(transport, cookieJar, requestUrl, metho
|
|
|
508
611
|
if (remainingTimeout !== undefined)
|
|
509
612
|
requestInit.timeout = remainingTimeout;
|
|
510
613
|
response = await transport.fetch(currentUrl, requestInit);
|
|
614
|
+
if (signal?.aborted) {
|
|
615
|
+
discardStealthRedirectBody(response);
|
|
616
|
+
throw toAmbientCancellationError(signal);
|
|
617
|
+
}
|
|
511
618
|
cookieJar.setFromCookieStrings(setCookieHeadersFromResponse(response.headers), response.url ?? currentUrl);
|
|
512
619
|
if (!isRedirectStatus(response.status) || options.redirect === "manual")
|
|
513
620
|
break;
|
|
@@ -536,7 +643,7 @@ async function fetchStealthRedirectChain(transport, cookieJar, requestUrl, metho
|
|
|
536
643
|
currentUrl = nextUrl;
|
|
537
644
|
followedHops += 1;
|
|
538
645
|
}
|
|
539
|
-
const normalized = await
|
|
646
|
+
const normalized = await normalizeResponseWithSignal(response, currentUrl, options.maxBodyBytes, signal);
|
|
540
647
|
if (followedHops > 0)
|
|
541
648
|
normalized.redirected = true;
|
|
542
649
|
return { normalized, response };
|
|
@@ -546,6 +653,10 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
546
653
|
let closed = false;
|
|
547
654
|
let hasWarnedMissingProxy = false;
|
|
548
655
|
const warn = clientOptions.warn ?? console.warn;
|
|
656
|
+
const intentAlias = getStealthProfileIntentAlias(defaultProfile);
|
|
657
|
+
if (intentAlias) {
|
|
658
|
+
warn(`[provider-sdk] Stealth profile "${defaultProfile}" pins a browser version and is deprecated. Use the intent profile "${intentAlias}" so TLS, headers, and User-Agent stay aligned with SDK updates; derive an explicit User-Agent with getStealthProfile("${intentAlias}").userAgent instead of hardcoding one.`);
|
|
659
|
+
}
|
|
549
660
|
const cookieJar = new StealthCookieJar([], baseUrl);
|
|
550
661
|
async function getClientEntry(profileName, proxyUrl, ignoreTlsErrors) {
|
|
551
662
|
if (closed) {
|
|
@@ -570,21 +681,57 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
570
681
|
}
|
|
571
682
|
return entry;
|
|
572
683
|
}
|
|
573
|
-
async function withClient(profileName, proxyUrl, ignoreTlsErrors, operation) {
|
|
684
|
+
async function withClient(profileName, proxyUrl, ignoreTlsErrors, operation, signal) {
|
|
685
|
+
throwIfAmbientAborted(signal);
|
|
574
686
|
const entry = await getClientEntry(profileName, proxyUrl, ignoreTlsErrors);
|
|
687
|
+
throwIfAmbientAborted(signal);
|
|
575
688
|
const previous = entry.tail;
|
|
576
689
|
let release;
|
|
577
690
|
entry.tail = new Promise((resolve) => {
|
|
578
691
|
release = resolve;
|
|
579
692
|
});
|
|
580
|
-
|
|
693
|
+
let acquired = false;
|
|
581
694
|
try {
|
|
582
|
-
|
|
695
|
+
await waitForClientTurn(previous, signal);
|
|
696
|
+
acquired = true;
|
|
697
|
+
throwIfAmbientAborted(signal);
|
|
698
|
+
const client = await entry.session;
|
|
699
|
+
throwIfAmbientAborted(signal);
|
|
700
|
+
const result = await operation(client);
|
|
701
|
+
throwIfAmbientAborted(signal);
|
|
702
|
+
return result;
|
|
583
703
|
}
|
|
584
704
|
finally {
|
|
585
|
-
|
|
705
|
+
if (acquired) {
|
|
706
|
+
release();
|
|
707
|
+
}
|
|
708
|
+
else {
|
|
709
|
+
void previous.then(release, release);
|
|
710
|
+
}
|
|
586
711
|
}
|
|
587
712
|
}
|
|
713
|
+
async function waitForClientTurn(previous, signal) {
|
|
714
|
+
if (!signal) {
|
|
715
|
+
await previous;
|
|
716
|
+
return;
|
|
717
|
+
}
|
|
718
|
+
throwIfAmbientAborted(signal);
|
|
719
|
+
await new Promise((resolve, reject) => {
|
|
720
|
+
let settled = false;
|
|
721
|
+
const settle = (operation) => {
|
|
722
|
+
if (settled)
|
|
723
|
+
return;
|
|
724
|
+
settled = true;
|
|
725
|
+
signal.removeEventListener("abort", onAbort);
|
|
726
|
+
operation();
|
|
727
|
+
};
|
|
728
|
+
const onAbort = () => settle(() => reject(toAmbientCancellationError(signal)));
|
|
729
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
730
|
+
void previous.then(() => settle(resolve), (error) => settle(() => reject(error)));
|
|
731
|
+
if (signal.aborted)
|
|
732
|
+
onAbort();
|
|
733
|
+
});
|
|
734
|
+
}
|
|
588
735
|
async function resolveRequestProxy(options, proxyAttempt, refreshEpoch) {
|
|
589
736
|
const resolvedProxy = await resolveProxyConfigAsync({
|
|
590
737
|
proxy: options?.proxy ?? clientOptions.proxy,
|
|
@@ -637,6 +784,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
637
784
|
throw redactSensitiveRequestError(error, url, options.sensitiveParams);
|
|
638
785
|
}
|
|
639
786
|
})();
|
|
787
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
640
788
|
const hasPolicyProxy = isPolicyManagedProxy(clientOptions);
|
|
641
789
|
const usesPolicyAllocator = hasPolicyProxy && !options.proxy && !clientOptions.proxy;
|
|
642
790
|
const retryAttemptCap = Math.max(1, stealthRetryOptions?.attempts ?? 1);
|
|
@@ -666,6 +814,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
666
814
|
let stalePoolDiagnosticProxy;
|
|
667
815
|
const attemptedProxies = new Set();
|
|
668
816
|
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
817
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
669
818
|
let proxy;
|
|
670
819
|
let attemptProxy;
|
|
671
820
|
// Reuse the exact serialization used by this outbound attempt in its catch path.
|
|
@@ -693,6 +842,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
693
842
|
});
|
|
694
843
|
};
|
|
695
844
|
try {
|
|
845
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
696
846
|
const sensitiveParams = normalizeSensitiveParams(options.sensitiveParams);
|
|
697
847
|
const structural = redactUrlQueryParams(url, Object.keys(sensitiveParams ?? {}));
|
|
698
848
|
fallbackSensitiveValues = [
|
|
@@ -705,6 +855,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
705
855
|
fallbackRedactedUrl = structural.redactedUrl;
|
|
706
856
|
assertNoUnsupportedFingerprintOverrides(options);
|
|
707
857
|
attemptProxy = await resolveRequestProxy(options, attempt, refreshAttempt);
|
|
858
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
708
859
|
proxy = attemptProxy.url;
|
|
709
860
|
if (proxy && dedupeAllocatorEndpoints) {
|
|
710
861
|
// An under-filled allocation repeats endpoints (via the modulo
|
|
@@ -722,7 +873,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
722
873
|
const profileName = options.profile ?? defaultProfile;
|
|
723
874
|
serializedUrl = serializeRequestUrl(resolveUrl(baseUrl, url), options.params, sensitiveParams);
|
|
724
875
|
const { requestUrl } = serializedUrl;
|
|
725
|
-
const { normalized, response } = await withClient(profileName, proxy, ignoreTlsErrors, (transport) => fetchStealthRedirectChain(transport, cookieJar, requestUrl, method, options));
|
|
876
|
+
const { normalized, response } = await withClient(profileName, proxy, ignoreTlsErrors, (transport) => fetchStealthRedirectChain(transport, cookieJar, requestUrl, method, options, clientOptions.signal), clientOptions.signal);
|
|
726
877
|
if (proxy && isProxyConnectFailureResponse(response, normalized.body)) {
|
|
727
878
|
throw createProxyConnectFailureError(normalized.body);
|
|
728
879
|
}
|
|
@@ -757,10 +908,16 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
757
908
|
const sensitiveValues = serializedUrl?.sensitiveValues ?? fallbackSensitiveValues;
|
|
758
909
|
let normalizedError;
|
|
759
910
|
try {
|
|
911
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
760
912
|
normalizedError = normalizeStealthTransportError(error);
|
|
761
913
|
}
|
|
762
914
|
catch (normalizationError) {
|
|
763
|
-
|
|
915
|
+
const redactedNormalizationError = redactSensitiveError(normalizationError, sensitiveValues, serializedUrl?.requestUrl ?? fallbackRequestUrl, serializedUrl?.redactedUrl ?? fallbackRedactedUrl);
|
|
916
|
+
if (normalizationError instanceof TransportError &&
|
|
917
|
+
normalizationError.code === "transport_cancelled") {
|
|
918
|
+
recordProxyAttempt("error", proxyAttemptErrorCode(normalizationError), proxyAttemptStatus(normalizationError));
|
|
919
|
+
}
|
|
920
|
+
throw redactedNormalizationError;
|
|
764
921
|
}
|
|
765
922
|
const retryErrorCode = proxyAttemptErrorCode(normalizedError);
|
|
766
923
|
const refreshableProxyError = isProxyPoolRefreshableError(normalizedError);
|
|
@@ -807,8 +964,9 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
807
964
|
proxyUsed: Boolean(proxy),
|
|
808
965
|
})) {
|
|
809
966
|
if (stealthRetryOptions) {
|
|
810
|
-
await sleep(computeProxyTransportRetryDelayMs(stealthRetryOptions, attempt + 1));
|
|
967
|
+
await sleep(computeProxyTransportRetryDelayMs(stealthRetryOptions, attempt + 1), clientOptions.signal);
|
|
811
968
|
}
|
|
969
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
812
970
|
continue;
|
|
813
971
|
}
|
|
814
972
|
throw normalizedError;
|
|
@@ -817,11 +975,13 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
817
975
|
if (rotatesRegistryChain &&
|
|
818
976
|
stalePoolError &&
|
|
819
977
|
refreshAttempt < MAX_POLICY_PROXY_POOL_REFRESHES) {
|
|
978
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
820
979
|
await invalidateProxyResolutionCacheAsync({
|
|
821
980
|
proxyPolicy: clientOptions.proxyPolicy,
|
|
822
981
|
upstream: clientOptions.upstream,
|
|
823
982
|
affinityKey: clientOptions.affinityKey,
|
|
824
983
|
});
|
|
984
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
825
985
|
continue;
|
|
826
986
|
}
|
|
827
987
|
const proxyAuthDiagnostic = stalePoolError && stalePoolDiagnosticProxy
|
|
@@ -842,6 +1002,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
842
1002
|
}
|
|
843
1003
|
break;
|
|
844
1004
|
}
|
|
1005
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
845
1006
|
throw normalizeStealthTransportError(lastError);
|
|
846
1007
|
},
|
|
847
1008
|
cookies: cookieJar,
|
|
@@ -1034,16 +1195,20 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
1034
1195
|
async function classifyProxyAuthDiagnostic(profileName, proxy) {
|
|
1035
1196
|
try {
|
|
1036
1197
|
return await withClient(profileName, proxy, false, async (client) => {
|
|
1198
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
1037
1199
|
await client.clearCookies();
|
|
1200
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
1038
1201
|
const response = await client.fetch(PROXY_AUTH_DIAGNOSTIC_URL, {
|
|
1039
1202
|
method: "GET",
|
|
1040
1203
|
timeout: PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS,
|
|
1204
|
+
...(clientOptions.signal ? { signal: clientOptions.signal } : {}),
|
|
1041
1205
|
});
|
|
1042
|
-
const normalized = await
|
|
1206
|
+
const normalized = await normalizeResponseWithSignal(response, undefined, undefined, clientOptions.signal);
|
|
1043
1207
|
return classifyProxyAuthDiagnosticMessage(normalized.body);
|
|
1044
|
-
});
|
|
1208
|
+
}, clientOptions.signal);
|
|
1045
1209
|
}
|
|
1046
1210
|
catch (error) {
|
|
1211
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
1047
1212
|
const message = error instanceof Error
|
|
1048
1213
|
? [error.message, error.cause instanceof Error ? error.cause.message : ""]
|
|
1049
1214
|
.filter(Boolean)
|
|
@@ -409,6 +409,7 @@ function createProviderContext(provider, request, operationId, options, state =
|
|
|
409
409
|
upstream: proxyClientOptions.upstream,
|
|
410
410
|
affinityKey: proxyClientOptions.affinityKey,
|
|
411
411
|
telemetry: proxyTelemetry,
|
|
412
|
+
...(signal ? { signal } : {}),
|
|
412
413
|
};
|
|
413
414
|
const { capabilityModules } = options;
|
|
414
415
|
const logStealthCleanupError = (error) => logProviderCleanupError(options.logger, provider, "operation", operationId, request.requestId, "stealth", error);
|
|
@@ -446,10 +447,10 @@ function createProviderContext(provider, request, operationId, options, state =
|
|
|
446
447
|
stealth: stealthBaseUrl
|
|
447
448
|
? capabilityModules.stealth
|
|
448
449
|
? stealthProfile
|
|
449
|
-
? capabilityModules.stealth.createStealthClient(stealthBaseUrl,
|
|
450
|
+
? capabilityModules.stealth.createStealthClient(stealthBaseUrl, provider.stealth.profile, stealthClientOptions)
|
|
450
451
|
: capabilityModules.stealth.createStealthClient(stealthBaseUrl, stealthClientOptions)
|
|
451
452
|
: stealthProfile
|
|
452
|
-
? createLazyStealthClient(logStealthCleanupError, stealthBaseUrl,
|
|
453
|
+
? createLazyStealthClient(logStealthCleanupError, stealthBaseUrl, provider.stealth.profile, stealthClientOptions)
|
|
453
454
|
: createLazyStealthClient(logStealthCleanupError, stealthBaseUrl, stealthClientOptions)
|
|
454
455
|
: createStealthStub(),
|
|
455
456
|
browser: provider.runtime === "browser"
|
|
@@ -566,6 +567,7 @@ function createAuthFlowContext(provider, request, options, state, proxyTelemetry
|
|
|
566
567
|
upstream: proxyClientOptions.upstream,
|
|
567
568
|
affinityKey: proxyClientOptions.affinityKey,
|
|
568
569
|
telemetry: proxyTelemetry,
|
|
570
|
+
...(signal ? { signal } : {}),
|
|
569
571
|
};
|
|
570
572
|
const { capabilityModules } = options;
|
|
571
573
|
const logStealthCleanupError = (error) => logProviderCleanupError(options.logger, provider, "auth", "flow", request.requestId, "stealth", error);
|
|
@@ -593,10 +595,10 @@ function createAuthFlowContext(provider, request, options, state, proxyTelemetry
|
|
|
593
595
|
stealth: stealthBaseUrl
|
|
594
596
|
? capabilityModules.stealth
|
|
595
597
|
? stealthProfile
|
|
596
|
-
? capabilityModules.stealth.createStealthClient(stealthBaseUrl,
|
|
598
|
+
? capabilityModules.stealth.createStealthClient(stealthBaseUrl, provider.stealth.profile, stealthClientOptions)
|
|
597
599
|
: capabilityModules.stealth.createStealthClient(stealthBaseUrl, stealthClientOptions)
|
|
598
600
|
: stealthProfile
|
|
599
|
-
? createLazyStealthClient(logStealthCleanupError, stealthBaseUrl,
|
|
601
|
+
? createLazyStealthClient(logStealthCleanupError, stealthBaseUrl, provider.stealth.profile, stealthClientOptions)
|
|
600
602
|
: createLazyStealthClient(logStealthCleanupError, stealthBaseUrl, stealthClientOptions)
|
|
601
603
|
: createStealthStub(),
|
|
602
604
|
...(provider.native
|
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
import type { StealthProfile } from "../types.js";
|
|
2
2
|
export declare function generateLayer2Headers(profile: StealthProfile): Record<string, string>;
|
|
3
3
|
export declare function getStealthProfile(name: string): StealthProfile;
|
|
4
|
+
/** Returns the intent alias that replaces a registered version-pinned profile. */
|
|
5
|
+
export declare function getStealthProfileIntentAlias(name: string): string | undefined;
|
|
6
|
+
/** Internal compatibility catalog used by transport-parity tests. */
|
|
7
|
+
export declare function listRegisteredStealthProfiles(): string[];
|
|
4
8
|
export declare function listStealthProfiles(): string[];
|
package/dist/stealth/profiles.js
CHANGED
|
@@ -1,4 +1,11 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
1
2
|
import { SDKError } from "../errors.js";
|
|
3
|
+
const requireModule = createRequire(import.meta.url);
|
|
4
|
+
let wreqProfileApi;
|
|
5
|
+
function getWreqProfileApi() {
|
|
6
|
+
wreqProfileApi ??= requireModule("wreq-js");
|
|
7
|
+
return wreqProfileApi;
|
|
8
|
+
}
|
|
2
9
|
const CHROMIUM_HEADER_ORDER = [
|
|
3
10
|
":method",
|
|
4
11
|
":authority",
|
|
@@ -65,6 +72,50 @@ const SAFARI_H2_SETTINGS = {
|
|
|
65
72
|
const CHROMIUM_JA3 = "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513-65037,29-23-24,0";
|
|
66
73
|
const FIREFOX_JA3 = "771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-28-27-43-45-51,29-23-24-25,0";
|
|
67
74
|
const SAFARI_JA3 = "771,4865-4866-4867-49196-49195-52393-49200-49199-49188-49192-159-158-107-103-57-51-157-156-61-60-53-47-255,0-23-65281-10-11-16-5-13-18-51-45-43-27,29-23-24-25,0";
|
|
75
|
+
/**
|
|
76
|
+
* The newest Chromium build wreq-js can emulate, resolved on first profile use.
|
|
77
|
+
*
|
|
78
|
+
* A literal version here rots: it went stale twice (146 was six releases behind
|
|
79
|
+
* stable when an upstream integrity analyzer flagged it), and a fingerprint that
|
|
80
|
+
* advertises an old Chrome is exactly what bot managers score against. wreq-js
|
|
81
|
+
* already ships the profile table, so the newest entry is derived from it rather
|
|
82
|
+
* than restated. `resolveProfile("chrome")` is deliberately NOT used because its
|
|
83
|
+
* conservative default can lag newer entries in that table.
|
|
84
|
+
*
|
|
85
|
+
* package.json pins wreq-js exactly on purpose. Even a semver-minor wreq-js update
|
|
86
|
+
* can change the profile table, emitted headers, native bindings, and therefore
|
|
87
|
+
* the wire fingerprint inherited by every provider. Upgrade that exact version
|
|
88
|
+
* only in a dedicated change with profile-parity and packed-native verification.
|
|
89
|
+
*/
|
|
90
|
+
function resolveLatestChromiumProfile() {
|
|
91
|
+
const { getProfiles } = getWreqProfileApi();
|
|
92
|
+
let newest;
|
|
93
|
+
for (const name of getProfiles()) {
|
|
94
|
+
const match = /^chrome_(\d+)$/.exec(name);
|
|
95
|
+
if (!match)
|
|
96
|
+
continue;
|
|
97
|
+
const version = Number(match[1]);
|
|
98
|
+
if (!newest || version > newest.version)
|
|
99
|
+
newest = { name, version };
|
|
100
|
+
}
|
|
101
|
+
if (!newest) {
|
|
102
|
+
throw new SDKError("wreq-js exposes no chrome_<version> emulation profile.", {
|
|
103
|
+
code: "STEALTH_PROFILE_UNAVAILABLE",
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
return { wreqName: newest.name, version: `${newest.version}.0.0.0` };
|
|
107
|
+
}
|
|
108
|
+
/** The user agent wreq-js itself emits for that profile, so the two never disagree. */
|
|
109
|
+
function chromiumUserAgent(latest) {
|
|
110
|
+
const { getEmulationHeaders } = getWreqProfileApi();
|
|
111
|
+
for (const [name, value] of getEmulationHeaders(latest.wreqName, "macos")) {
|
|
112
|
+
if (String(name).toLowerCase() === "user-agent")
|
|
113
|
+
return String(value);
|
|
114
|
+
}
|
|
115
|
+
throw new SDKError(`wreq-js profile ${latest.wreqName} exposes no user-agent header.`, {
|
|
116
|
+
code: "STEALTH_PROFILE_UNAVAILABLE",
|
|
117
|
+
});
|
|
118
|
+
}
|
|
68
119
|
function createProfile(name, definition) {
|
|
69
120
|
return {
|
|
70
121
|
name,
|
|
@@ -120,10 +171,21 @@ export function generateLayer2Headers(profile) {
|
|
|
120
171
|
}
|
|
121
172
|
return headers;
|
|
122
173
|
}
|
|
123
|
-
const
|
|
124
|
-
"
|
|
174
|
+
const STATIC_STEALTH_PROFILE_ALIASES = {
|
|
175
|
+
"firefox-desktop": "firefox-147",
|
|
176
|
+
"safari-desktop": "safari-17",
|
|
177
|
+
"safari-mobile": "ios-safari-26",
|
|
125
178
|
};
|
|
126
|
-
const
|
|
179
|
+
const PUBLIC_STEALTH_PROFILE_NAMES = [
|
|
180
|
+
"chrome-desktop",
|
|
181
|
+
"firefox-desktop",
|
|
182
|
+
"safari-desktop",
|
|
183
|
+
"safari-mobile",
|
|
184
|
+
"generic-desktop",
|
|
185
|
+
"generic-mobile",
|
|
186
|
+
];
|
|
187
|
+
const PUBLIC_STEALTH_PROFILE_NAME_SET = new Set(PUBLIC_STEALTH_PROFILE_NAMES);
|
|
188
|
+
const STATIC_STEALTH_PROFILES = {
|
|
127
189
|
"chrome-146": createProfile("chrome-146", {
|
|
128
190
|
platform: "macos",
|
|
129
191
|
version: "146.0.0.0",
|
|
@@ -223,15 +285,6 @@ const STEALTH_PROFILES = {
|
|
|
223
285
|
h2Settings: SAFARI_H2_SETTINGS,
|
|
224
286
|
headerOrder: SAFARI_HEADER_ORDER,
|
|
225
287
|
}),
|
|
226
|
-
"generic-desktop": createProfile("generic-desktop", {
|
|
227
|
-
platform: "macos",
|
|
228
|
-
version: "146.0.0.0",
|
|
229
|
-
userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
|
|
230
|
-
tlsClientIdentifier: "chrome_146",
|
|
231
|
-
ja3: CHROMIUM_JA3,
|
|
232
|
-
h2Settings: CHROMIUM_H2_SETTINGS,
|
|
233
|
-
headerOrder: CHROMIUM_HEADER_ORDER,
|
|
234
|
-
}),
|
|
235
288
|
"generic-mobile": createProfile("generic-mobile", {
|
|
236
289
|
platform: "ios",
|
|
237
290
|
version: "26.0",
|
|
@@ -242,9 +295,38 @@ const STEALTH_PROFILES = {
|
|
|
242
295
|
headerOrder: SAFARI_HEADER_ORDER,
|
|
243
296
|
}),
|
|
244
297
|
};
|
|
298
|
+
let stealthProfileCatalog;
|
|
299
|
+
function getStealthProfileCatalog() {
|
|
300
|
+
if (stealthProfileCatalog)
|
|
301
|
+
return stealthProfileCatalog;
|
|
302
|
+
const latest = resolveLatestChromiumProfile();
|
|
303
|
+
const currentName = `chrome-${latest.version.split(".")[0]}`;
|
|
304
|
+
const currentProfile = createProfile(currentName, {
|
|
305
|
+
platform: "macos",
|
|
306
|
+
version: latest.version,
|
|
307
|
+
userAgent: chromiumUserAgent(latest),
|
|
308
|
+
tlsClientIdentifier: latest.wreqName,
|
|
309
|
+
ja3: CHROMIUM_JA3,
|
|
310
|
+
h2Settings: CHROMIUM_H2_SETTINGS,
|
|
311
|
+
headerOrder: CHROMIUM_HEADER_ORDER,
|
|
312
|
+
});
|
|
313
|
+
stealthProfileCatalog = {
|
|
314
|
+
aliases: {
|
|
315
|
+
"chrome-desktop": currentName,
|
|
316
|
+
...STATIC_STEALTH_PROFILE_ALIASES,
|
|
317
|
+
},
|
|
318
|
+
profiles: {
|
|
319
|
+
[currentName]: currentProfile,
|
|
320
|
+
...STATIC_STEALTH_PROFILES,
|
|
321
|
+
"generic-desktop": createProfile("generic-desktop", currentProfile),
|
|
322
|
+
},
|
|
323
|
+
};
|
|
324
|
+
return stealthProfileCatalog;
|
|
325
|
+
}
|
|
245
326
|
export function getStealthProfile(name) {
|
|
246
|
-
const
|
|
247
|
-
const
|
|
327
|
+
const catalog = getStealthProfileCatalog();
|
|
328
|
+
const canonicalName = catalog.aliases[name] ?? name;
|
|
329
|
+
const profile = catalog.profiles[canonicalName];
|
|
248
330
|
if (!profile) {
|
|
249
331
|
throw new SDKError(`Unknown stealth profile: ${name}`);
|
|
250
332
|
}
|
|
@@ -254,6 +336,27 @@ export function getStealthProfile(name) {
|
|
|
254
336
|
headerOrder: profile.headerOrder ? [...profile.headerOrder] : undefined,
|
|
255
337
|
};
|
|
256
338
|
}
|
|
339
|
+
/** Returns the intent alias that replaces a registered version-pinned profile. */
|
|
340
|
+
export function getStealthProfileIntentAlias(name) {
|
|
341
|
+
if (PUBLIC_STEALTH_PROFILE_NAME_SET.has(name))
|
|
342
|
+
return undefined;
|
|
343
|
+
if (/^(?:chrome|chromium|edge)[-_]\d/i.test(name)) {
|
|
344
|
+
return "chrome-desktop";
|
|
345
|
+
}
|
|
346
|
+
if (/^firefox[-_]\d/i.test(name))
|
|
347
|
+
return "firefox-desktop";
|
|
348
|
+
if (/^(?:ios[-_]safari|safari[-_](?:ios|ipad))[-_]\d/i.test(name)) {
|
|
349
|
+
return "safari-mobile";
|
|
350
|
+
}
|
|
351
|
+
if (/^safari[-_]\d/i.test(name))
|
|
352
|
+
return "safari-desktop";
|
|
353
|
+
return undefined;
|
|
354
|
+
}
|
|
355
|
+
/** Internal compatibility catalog used by transport-parity tests. */
|
|
356
|
+
export function listRegisteredStealthProfiles() {
|
|
357
|
+
const catalog = getStealthProfileCatalog();
|
|
358
|
+
return [...Object.keys(catalog.profiles), ...Object.keys(catalog.aliases)];
|
|
359
|
+
}
|
|
257
360
|
export function listStealthProfiles() {
|
|
258
|
-
return [...
|
|
361
|
+
return [...PUBLIC_STEALTH_PROFILE_NAMES];
|
|
259
362
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -956,6 +956,10 @@ export interface ProviderMeta {
|
|
|
956
956
|
publicProfile?: ProviderPublicProfile;
|
|
957
957
|
contract?: {
|
|
958
958
|
publicSchemaFieldNames?: "normalized";
|
|
959
|
+
readonly pinnedWireFieldPaths?: readonly {
|
|
960
|
+
readonly path: string;
|
|
961
|
+
readonly reason: string;
|
|
962
|
+
}[];
|
|
959
963
|
};
|
|
960
964
|
}
|
|
961
965
|
export type RequestParamPrimitive = string | number | boolean | null | undefined;
|