@apifuse/provider-sdk 2.2.0-beta.25 → 2.2.0-beta.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +7 -6
- package/CHANGELOG.md +9 -1
- package/README.md +3 -3
- package/bin/apifuse-check.ts +62 -3
- package/bin/apifuse-pack-check.ts +8 -2
- package/bin/apifuse-pack-smoke.ts +43 -2
- package/bin/apifuse-pack-types.ts +58 -0
- package/dist/auth.js +29 -0
- package/dist/cli/templates/provider/README.md.tpl +4 -4
- package/dist/contract-serialization.js +4 -8
- package/dist/declaration-validation.d.ts +23 -0
- package/dist/declaration-validation.js +159 -0
- package/dist/define.d.ts +1 -1
- package/dist/define.js +13 -2
- package/dist/index.d.ts +1 -0
- package/dist/lint.js +85 -3
- package/dist/runtime/resolver-vendors/bindings.d.ts +42 -2
- package/dist/runtime/resolver-vendors/bindings.js +31 -6
- package/dist/runtime/resolver-vendors/browser.d.ts +3 -7
- package/dist/runtime/resolver-vendors/browser.js +7 -22
- package/dist/runtime/resolver-vendors/hosts.d.ts +2 -0
- package/dist/runtime/resolver-vendors/hosts.js +33 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +23 -0
- package/dist/runtime/resolver-vendors/twocaptcha.js +264 -0
- package/dist/runtime/resolver-vendors/types.d.ts +44 -3
- package/dist/runtime/resolver-vendors/types.js +10 -0
- package/dist/runtime/resolver.d.ts +17 -2
- package/dist/runtime/resolver.js +237 -15
- package/dist/runtime/stealth.d.ts +26 -4
- package/dist/runtime/stealth.js +224 -114
- package/dist/server/serve.js +8 -0
- package/dist/stealth/profiles.js +16 -7
- package/dist/types.d.ts +34 -1
- package/package.json +2 -2
- package/src/auth.ts +40 -0
- package/src/cli/templates/provider/README.md.tpl +4 -4
- package/src/contract-serialization.ts +5 -7
- package/src/declaration-validation.ts +202 -0
- package/src/define.ts +23 -2
- package/src/index.ts +1 -0
- package/src/lint.ts +98 -3
- package/src/runtime/resolver-vendors/bindings.ts +40 -15
- package/src/runtime/resolver-vendors/browser.ts +9 -31
- package/src/runtime/resolver-vendors/hosts.ts +38 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +366 -0
- package/src/runtime/resolver-vendors/types.ts +54 -0
- package/src/runtime/resolver.ts +304 -24
- package/src/runtime/stealth.ts +317 -136
- package/src/server/serve.ts +8 -0
- package/src/stealth/profiles.ts +17 -7
- package/src/types.ts +36 -3
package/dist/runtime/stealth.js
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { Impit } from "impit";
|
|
3
2
|
import { Cookie, CookieJar as ToughCookieJar } from "tough-cookie";
|
|
4
3
|
import { DEFAULT_SMARTPROXY_POOL_SIZE, invalidateProxyResolutionCacheAsync, ProxyResolutionError, policyResolvesRegistryVendorChain, resolvePolicyProxyPoolSpan, resolvePolicyTransportAttemptCap, resolveProxyConfigAsync, vendorFromResolvedSource, } from "../config/loader.js";
|
|
5
4
|
import { SDKError, StealthCookieStoreVersionError, TransportError } from "../errors.js";
|
|
6
5
|
import { getStealthProfile } from "../stealth/profiles.js";
|
|
7
6
|
import { createProxyAuthIpDeniedError, createProxyEdgeAuthRejectedError, createProxyEdgeTlsRejectedError, createProxyPoolExhaustedError, createProxyPoolStaleError, isProxyAuthIpDeniedMessage, isProxyEdgeAuthRejectedMessage, isProxyEdgeTlsRejectedResponse, isProxyPoolRefreshableError, isProxyPoolStaleMessage, isProxyPoolStaleStatus, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_STALE_CODE, } from "./proxy-errors.js";
|
|
8
7
|
import { computeProxyAttemptIndex, computeProxyTransportRetryDelayMs, createDefaultProxyTransportRetryOptions, normalizeProxyTransportRetryOptions, shouldRetryProxyTransportAttempt, validateUnsafeProxyTransportRetryMethods, } from "./proxy-retry-policy.js";
|
|
9
|
-
import { evaluateRedirectHop, isRedirectStatus, resolveRedirectUrl, } from "./redirects.js";
|
|
10
|
-
import { isSensitiveKey, redactSensitiveError, redactSensitiveRequestError, redactSensitiveText, redactUrlQueryParams,
|
|
8
|
+
import { evaluateRedirectHop, isRedirectStatus, nextRedirectMethod, resolveRedirectUrl, } from "./redirects.js";
|
|
9
|
+
import { isSensitiveKey, normalizeSensitiveParams, redactSensitiveError, redactSensitiveRequestError, redactSensitiveText, redactUrlQueryParams, serializeRequestUrl, } from "./request-options.js";
|
|
11
10
|
const DEFAULT_PROFILE = "chrome-146";
|
|
12
11
|
const MISSING_PROXY_WARNING = "[provider-sdk] Provider requested proxy routing, but no proxy URL was configured. Continuing without proxy.";
|
|
13
12
|
const MAX_POLICY_PROXY_POOL_REFRESHES = 1;
|
|
@@ -16,6 +15,13 @@ const PROXY_CONNECT_FAILURE_BODY_PATTERN = /\bproxy\b.*\b(non[\s-]?200|connect|t
|
|
|
16
15
|
const PROXY_AUTH_DIAGNOSTIC_URL = "http://example.com/";
|
|
17
16
|
const PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS = 5_000;
|
|
18
17
|
const STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES = [PROXY_CONNECT_FAILURE_CODE];
|
|
18
|
+
const MAX_STEALTH_REDIRECT_HOPS = 10;
|
|
19
|
+
const REDIRECT_BODY_HEADERS = new Set([
|
|
20
|
+
"content-encoding",
|
|
21
|
+
"content-language",
|
|
22
|
+
"content-location",
|
|
23
|
+
"content-type",
|
|
24
|
+
]);
|
|
19
25
|
function sensitiveQueryParamNames(url) {
|
|
20
26
|
const queryStart = url.indexOf("?");
|
|
21
27
|
if (queryStart === -1)
|
|
@@ -37,25 +43,6 @@ const REMOVED_CHROME_PROFILE_NAMES = new Set([
|
|
|
37
43
|
"chrome-130-psk",
|
|
38
44
|
"edge-131",
|
|
39
45
|
]);
|
|
40
|
-
const CHROME_IMPIT_BY_MAJOR = {
|
|
41
|
-
100: "chrome100",
|
|
42
|
-
101: "chrome101",
|
|
43
|
-
104: "chrome104",
|
|
44
|
-
107: "chrome107",
|
|
45
|
-
110: "chrome110",
|
|
46
|
-
116: "chrome116",
|
|
47
|
-
124: "chrome124",
|
|
48
|
-
125: "chrome125",
|
|
49
|
-
131: "chrome131",
|
|
50
|
-
136: "chrome136",
|
|
51
|
-
142: "chrome142",
|
|
52
|
-
};
|
|
53
|
-
const FIREFOX_IMPIT_BY_MAJOR = {
|
|
54
|
-
128: "firefox128",
|
|
55
|
-
133: "firefox133",
|
|
56
|
-
135: "firefox135",
|
|
57
|
-
144: "firefox144",
|
|
58
|
-
};
|
|
59
46
|
function isRecord(value) {
|
|
60
47
|
return typeof value === "object" && value !== null;
|
|
61
48
|
}
|
|
@@ -182,47 +169,93 @@ class CookieJarImpl {
|
|
|
182
169
|
});
|
|
183
170
|
}
|
|
184
171
|
}
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
172
|
+
let wreqModulePromise;
|
|
173
|
+
function getWreqModule() {
|
|
174
|
+
if (!wreqModulePromise) {
|
|
175
|
+
wreqModulePromise = import("wreq-js").catch((error) => {
|
|
176
|
+
throw new SDKError(`Stealth transport is unavailable on ${process.platform}-${process.arch}: the wreq-js native binary could not be loaded.`, {
|
|
177
|
+
code: "stealth_transport_unavailable",
|
|
178
|
+
cause: error instanceof Error ? error : undefined,
|
|
179
|
+
});
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
return wreqModulePromise;
|
|
183
|
+
}
|
|
184
|
+
function parseProfileIdentifier(identifier) {
|
|
185
|
+
const match = /^(safari_ios|safari_ipad|firefox_android|firefox_private|chrome|edge|firefox|opera|safari|okhttp)_(\d+(?:[._]\d+)*)$/.exec(identifier.toLowerCase());
|
|
186
|
+
if (!match?.[1] || !match[2])
|
|
187
|
+
return null;
|
|
188
|
+
return {
|
|
189
|
+
family: match[1],
|
|
190
|
+
version: match[2].split(/[._]/).map(Number),
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
function compareVersionDistance(target, left, right) {
|
|
194
|
+
const width = Math.max(target.length, left.length, right.length);
|
|
195
|
+
for (let index = 0; index < width; index += 1) {
|
|
196
|
+
const targetPart = target[index] ?? 0;
|
|
197
|
+
const leftDistance = Math.abs((left[index] ?? 0) - targetPart);
|
|
198
|
+
const rightDistance = Math.abs((right[index] ?? 0) - targetPart);
|
|
199
|
+
if (leftDistance !== rightDistance)
|
|
200
|
+
return leftDistance - rightDistance;
|
|
201
|
+
}
|
|
202
|
+
return 0;
|
|
203
|
+
}
|
|
204
|
+
function closestWreqProfile(identifier, wreqProfiles) {
|
|
205
|
+
const requested = parseProfileIdentifier(identifier);
|
|
206
|
+
if (!requested)
|
|
207
|
+
return undefined;
|
|
208
|
+
let closest;
|
|
209
|
+
for (const candidateName of wreqProfiles) {
|
|
210
|
+
const candidate = parseProfileIdentifier(candidateName);
|
|
211
|
+
if (!candidate || candidate.family !== requested.family)
|
|
212
|
+
continue;
|
|
213
|
+
if (!closest ||
|
|
214
|
+
compareVersionDistance(requested.version, candidate.version, closest.version) < 0) {
|
|
215
|
+
closest = { name: candidateName, version: candidate.version };
|
|
194
216
|
}
|
|
195
217
|
}
|
|
196
|
-
return
|
|
218
|
+
return closest?.name;
|
|
219
|
+
}
|
|
220
|
+
function resolveDefaultWreqProfileMapping() {
|
|
221
|
+
let profile;
|
|
222
|
+
try {
|
|
223
|
+
profile = getStealthProfile(DEFAULT_PROFILE);
|
|
224
|
+
}
|
|
225
|
+
catch (error) {
|
|
226
|
+
throw new SDKError(`Default stealth profile "${DEFAULT_PROFILE}" cannot be mapped to a wreq-js browser profile.`, { cause: error instanceof Error ? error : undefined });
|
|
227
|
+
}
|
|
228
|
+
const identifier = profile.tlsClientIdentifier?.toLowerCase() ?? "";
|
|
229
|
+
if (!parseProfileIdentifier(identifier)) {
|
|
230
|
+
throw new SDKError(`Default stealth profile "${DEFAULT_PROFILE}" cannot be mapped to a wreq-js browser profile.`);
|
|
231
|
+
}
|
|
232
|
+
return { identifier, os: profile.platform };
|
|
197
233
|
}
|
|
198
|
-
|
|
234
|
+
const DEFAULT_WREQ_PROFILE_MAPPING = resolveDefaultWreqProfileMapping();
|
|
235
|
+
export function resolveWreqProfile(profileName, wreqProfiles) {
|
|
199
236
|
if (REMOVED_CHROME_PROFILE_NAMES.has(profileName)) {
|
|
200
237
|
throw new SDKError(`Unknown stealth profile: ${profileName}`);
|
|
201
238
|
}
|
|
202
|
-
let
|
|
239
|
+
let identifier;
|
|
240
|
+
let os;
|
|
203
241
|
try {
|
|
204
|
-
profile = getStealthProfile(profileName);
|
|
242
|
+
const profile = getStealthProfile(profileName);
|
|
243
|
+
identifier = profile.tlsClientIdentifier?.toLowerCase() ?? "";
|
|
244
|
+
os = profile.platform;
|
|
205
245
|
}
|
|
206
246
|
catch {
|
|
207
247
|
// Preserve the previous ctx.stealth.fetch() compatibility behavior: unknown
|
|
208
248
|
// profile strings still run with the transport default instead of failing
|
|
209
249
|
// before the request starts. Removed built-in profile aliases above remain
|
|
210
250
|
// explicit errors so callers do not accidentally pin retired fingerprints.
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
const identifier = profile.tlsClientIdentifier?.toLowerCase() ?? "";
|
|
214
|
-
const chromeMatch = /^(?:chrome|edge)_(\d+)/.exec(identifier);
|
|
215
|
-
if (chromeMatch?.[1]) {
|
|
216
|
-
return closestImpitBrowser(Number(chromeMatch[1]), CHROME_IMPIT_BY_MAJOR);
|
|
251
|
+
identifier = DEFAULT_WREQ_PROFILE_MAPPING.identifier;
|
|
252
|
+
os = DEFAULT_WREQ_PROFILE_MAPPING.os;
|
|
217
253
|
}
|
|
218
|
-
const
|
|
219
|
-
if (
|
|
220
|
-
|
|
254
|
+
const browser = closestWreqProfile(identifier, wreqProfiles);
|
|
255
|
+
if (!browser) {
|
|
256
|
+
throw new SDKError(`Stealth profile "${profileName}" cannot be mapped to a wreq-js browser profile.`);
|
|
221
257
|
}
|
|
222
|
-
|
|
223
|
-
throw new SDKError(`Stealth profile "${profileName}" uses a Safari stealth fingerprint, but TypeScript ctx.stealth uses impit which currently supports Chrome, Firefox, and OkHttp profiles only. Use a Chrome/Firefox stealth profile for ctx.stealth or ctx.browser for Safari-specific behavior.`);
|
|
224
|
-
}
|
|
225
|
-
throw new SDKError(`Stealth profile "${profileName}" cannot be mapped to an impit browser profile.`);
|
|
258
|
+
return { browser, os };
|
|
226
259
|
}
|
|
227
260
|
function resolveUrl(baseUrl, url) {
|
|
228
261
|
return new URL(url, baseUrl).toString();
|
|
@@ -242,18 +275,6 @@ function normalizeHeaders(headers) {
|
|
|
242
275
|
function hasOwn(object, key) {
|
|
243
276
|
return Object.hasOwn(object, key);
|
|
244
277
|
}
|
|
245
|
-
function toImpitCookieJar(cookieJar) {
|
|
246
|
-
return {
|
|
247
|
-
setCookie(cookie, url, cb) {
|
|
248
|
-
cookieJar.setFromCookieStrings([cookie], url);
|
|
249
|
-
if (typeof cb === "function")
|
|
250
|
-
cb();
|
|
251
|
-
},
|
|
252
|
-
getCookieString(url) {
|
|
253
|
-
return cookieJar.toHeader(url);
|
|
254
|
-
},
|
|
255
|
-
};
|
|
256
|
-
}
|
|
257
278
|
function assertNoUnsupportedFingerprintOverrides(options) {
|
|
258
279
|
if (!isRecord(options))
|
|
259
280
|
return;
|
|
@@ -267,7 +288,7 @@ function assertNoUnsupportedFingerprintOverrides(options) {
|
|
|
267
288
|
unsupported.push("stealth.h2");
|
|
268
289
|
if (unsupported.length === 0)
|
|
269
290
|
return;
|
|
270
|
-
throw new SDKError(`ctx.stealth.fetch uses
|
|
291
|
+
throw new SDKError(`ctx.stealth.fetch uses transport-managed browser fingerprints and no longer accepts low-level stealth overrides: ${unsupported.join(", ")}. Use the profile option instead.`);
|
|
271
292
|
}
|
|
272
293
|
function responseHeadersToRecord(headers) {
|
|
273
294
|
const record = {};
|
|
@@ -348,23 +369,10 @@ function declaredContentLength(headers) {
|
|
|
348
369
|
const parsed = Number(contentLength);
|
|
349
370
|
return Number.isFinite(parsed) ? parsed : undefined;
|
|
350
371
|
}
|
|
351
|
-
function abortTransportResponse(response) {
|
|
352
|
-
if (!response.abort)
|
|
353
|
-
return false;
|
|
354
|
-
try {
|
|
355
|
-
response.abort();
|
|
356
|
-
}
|
|
357
|
-
catch {
|
|
358
|
-
// The size error remains the primary failure if impit has already closed the response.
|
|
359
|
-
}
|
|
360
|
-
return true;
|
|
361
|
-
}
|
|
362
372
|
async function readResponseBodyWithLimit(response, maxBodyBytes) {
|
|
363
373
|
const contentLength = declaredContentLength(response.headers);
|
|
364
374
|
if (contentLength !== undefined && contentLength > maxBodyBytes) {
|
|
365
|
-
|
|
366
|
-
await response.body?.cancel().catch(() => undefined);
|
|
367
|
-
}
|
|
375
|
+
await response.body?.cancel().catch(() => undefined);
|
|
368
376
|
throw responseTooLargeError(maxBodyBytes, contentLength);
|
|
369
377
|
}
|
|
370
378
|
if (!response.body) {
|
|
@@ -382,10 +390,11 @@ async function readResponseBodyWithLimit(response, maxBodyBytes) {
|
|
|
382
390
|
const { done, value } = await reader.read();
|
|
383
391
|
if (done)
|
|
384
392
|
break;
|
|
393
|
+
if (!value)
|
|
394
|
+
continue;
|
|
385
395
|
receivedBytes += value.byteLength;
|
|
386
396
|
if (receivedBytes > maxBodyBytes) {
|
|
387
397
|
await reader.cancel().catch(() => undefined);
|
|
388
|
-
abortTransportResponse(response);
|
|
389
398
|
throw responseTooLargeError(maxBodyBytes, receivedBytes);
|
|
390
399
|
}
|
|
391
400
|
chunks.push(value);
|
|
@@ -566,31 +575,138 @@ function locationHeader(headers) {
|
|
|
566
575
|
}
|
|
567
576
|
return undefined;
|
|
568
577
|
}
|
|
578
|
+
function withoutRedirectBodyHeaders(headers) {
|
|
579
|
+
return Object.fromEntries(Object.entries(headers).filter(([name]) => !REDIRECT_BODY_HEADERS.has(name.toLowerCase())));
|
|
580
|
+
}
|
|
581
|
+
function assertStealthRedirectUrl(url) {
|
|
582
|
+
const protocol = new URL(url).protocol;
|
|
583
|
+
if (protocol !== "http:" && protocol !== "https:") {
|
|
584
|
+
throw new TransportError(`Stealth redirect target scheme "${protocol}" is not allowed`, {
|
|
585
|
+
code: "transport_invalid_url",
|
|
586
|
+
status: 0,
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
function discardStealthRedirectBody(response) {
|
|
591
|
+
try {
|
|
592
|
+
const cancellation = response.body?.cancel();
|
|
593
|
+
if (cancellation)
|
|
594
|
+
void cancellation.catch(() => undefined);
|
|
595
|
+
}
|
|
596
|
+
catch {
|
|
597
|
+
// Redirect handling is decided from status and headers. A cancellation
|
|
598
|
+
// failure must not replace or delay that decision.
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
async function fetchStealthRedirectChain(transport, cookieJar, requestUrl, method, options) {
|
|
602
|
+
let currentUrl = requestUrl;
|
|
603
|
+
let currentMethod = method;
|
|
604
|
+
let currentBody = options.body === undefined ? undefined : normalizeBody(options.body);
|
|
605
|
+
let currentHeaders = { ...(options.headers ?? {}) };
|
|
606
|
+
let followedHops = 0;
|
|
607
|
+
let response;
|
|
608
|
+
const deadline = options.timeout ? performance.now() + options.timeout : undefined;
|
|
609
|
+
while (true) {
|
|
610
|
+
const headers = { ...currentHeaders };
|
|
611
|
+
if (!hasHeader(headers, "Cookie")) {
|
|
612
|
+
const cookieHeader = cookieJar.toHeader(currentUrl);
|
|
613
|
+
if (cookieHeader)
|
|
614
|
+
headers.Cookie = cookieHeader;
|
|
615
|
+
}
|
|
616
|
+
const requestInit = {
|
|
617
|
+
headers: normalizeHeaders(headers),
|
|
618
|
+
method: currentMethod,
|
|
619
|
+
redirect: "manual",
|
|
620
|
+
};
|
|
621
|
+
if (currentBody !== undefined)
|
|
622
|
+
requestInit.body = currentBody;
|
|
623
|
+
await transport.clearCookies();
|
|
624
|
+
const remainingTimeout = deadline === undefined ? undefined : Math.ceil(deadline - performance.now());
|
|
625
|
+
if (remainingTimeout !== undefined && remainingTimeout <= 0) {
|
|
626
|
+
throw new TransportError("Request timed out", {
|
|
627
|
+
code: "transport_timeout",
|
|
628
|
+
status: 0,
|
|
629
|
+
});
|
|
630
|
+
}
|
|
631
|
+
if (remainingTimeout !== undefined)
|
|
632
|
+
requestInit.timeout = remainingTimeout;
|
|
633
|
+
response = await transport.fetch(currentUrl, requestInit);
|
|
634
|
+
cookieJar.setFromCookieStrings(setCookieHeadersFromResponse(response.headers), response.url ?? currentUrl);
|
|
635
|
+
if (!isRedirectStatus(response.status) || options.redirect === "manual")
|
|
636
|
+
break;
|
|
637
|
+
if (options.redirect === "error") {
|
|
638
|
+
discardStealthRedirectBody(response);
|
|
639
|
+
throw new TransportError("Stealth request encountered a redirect", {
|
|
640
|
+
code: "transport_network_error",
|
|
641
|
+
status: 0,
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
const nextUrl = resolveRedirectUrl(response.headers.get("location") ?? undefined, response.url ?? currentUrl);
|
|
645
|
+
if (!nextUrl)
|
|
646
|
+
break;
|
|
647
|
+
if (followedHops >= MAX_STEALTH_REDIRECT_HOPS) {
|
|
648
|
+
discardStealthRedirectBody(response);
|
|
649
|
+
throw new TransportError(`Stealth request exceeded the ${MAX_STEALTH_REDIRECT_HOPS}-redirect limit`, { code: "transport_network_error", status: 0 });
|
|
650
|
+
}
|
|
651
|
+
assertStealthRedirectUrl(nextUrl);
|
|
652
|
+
discardStealthRedirectBody(response);
|
|
653
|
+
const nextMethod = nextRedirectMethod(response.status, currentMethod);
|
|
654
|
+
if (nextMethod !== currentMethod) {
|
|
655
|
+
currentBody = undefined;
|
|
656
|
+
currentHeaders = withoutRedirectBodyHeaders(currentHeaders);
|
|
657
|
+
}
|
|
658
|
+
currentMethod = nextMethod;
|
|
659
|
+
currentUrl = nextUrl;
|
|
660
|
+
followedHops += 1;
|
|
661
|
+
}
|
|
662
|
+
const normalized = await normalizeResponse(response, currentUrl, options.maxBodyBytes);
|
|
663
|
+
if (followedHops > 0)
|
|
664
|
+
normalized.redirected = true;
|
|
665
|
+
return { normalized, response };
|
|
666
|
+
}
|
|
569
667
|
function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
570
668
|
const clients = new Map();
|
|
571
669
|
let closed = false;
|
|
572
670
|
let hasWarnedMissingProxy = false;
|
|
573
671
|
const warn = clientOptions.warn ?? console.warn;
|
|
574
672
|
const cookieJar = new CookieJarImpl([], baseUrl);
|
|
575
|
-
|
|
576
|
-
function getClient(profileName, proxyUrl, ignoreTlsErrors) {
|
|
673
|
+
async function getClientEntry(profileName, proxyUrl, ignoreTlsErrors) {
|
|
577
674
|
if (closed) {
|
|
578
675
|
throw new TransportError("Stealth session is closed", { status: 0 });
|
|
579
676
|
}
|
|
580
|
-
const
|
|
677
|
+
const wreq = await getWreqModule();
|
|
678
|
+
const { browser, os } = resolveWreqProfile(profileName, wreq.getProfiles());
|
|
581
679
|
const cacheKey = JSON.stringify({ browser, proxyUrl, ignoreTlsErrors });
|
|
582
|
-
let
|
|
583
|
-
if (!
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
680
|
+
let entry = clients.get(cacheKey);
|
|
681
|
+
if (!entry) {
|
|
682
|
+
entry = {
|
|
683
|
+
session: wreq.createSession({
|
|
684
|
+
browser,
|
|
685
|
+
os,
|
|
686
|
+
...(proxyUrl ? { proxy: proxyUrl } : {}),
|
|
687
|
+
...(ignoreTlsErrors ? { insecure: true } : {}),
|
|
688
|
+
timeout: 30_000,
|
|
689
|
+
}),
|
|
690
|
+
tail: Promise.resolve(),
|
|
691
|
+
};
|
|
692
|
+
clients.set(cacheKey, entry);
|
|
693
|
+
}
|
|
694
|
+
return entry;
|
|
695
|
+
}
|
|
696
|
+
async function withClient(profileName, proxyUrl, ignoreTlsErrors, operation) {
|
|
697
|
+
const entry = await getClientEntry(profileName, proxyUrl, ignoreTlsErrors);
|
|
698
|
+
const previous = entry.tail;
|
|
699
|
+
let release;
|
|
700
|
+
entry.tail = new Promise((resolve) => {
|
|
701
|
+
release = resolve;
|
|
702
|
+
});
|
|
703
|
+
await previous;
|
|
704
|
+
try {
|
|
705
|
+
return await operation(await entry.session);
|
|
706
|
+
}
|
|
707
|
+
finally {
|
|
708
|
+
release();
|
|
592
709
|
}
|
|
593
|
-
return client;
|
|
594
710
|
}
|
|
595
711
|
async function resolveRequestProxy(options, proxyAttempt, refreshEpoch) {
|
|
596
712
|
const resolvedProxy = await resolveProxyConfigAsync({
|
|
@@ -602,7 +718,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
602
718
|
proxyAttemptOffset: options?.proxyAttemptOffset,
|
|
603
719
|
retryAttemptOffset: proxyAttempt,
|
|
604
720
|
}),
|
|
605
|
-
// The
|
|
721
|
+
// The stealth transport tunnels both HTTP CONNECT and SOCKS5,
|
|
606
722
|
// preserving the client TLS fingerprint end-to-end.
|
|
607
723
|
transportProtocols: ["http", "socks5"],
|
|
608
724
|
...(refreshEpoch === undefined ? {} : { proxyRefreshEpoch: refreshEpoch }),
|
|
@@ -729,24 +845,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
729
845
|
const profileName = options.profile ?? defaultProfile;
|
|
730
846
|
serializedUrl = serializeRequestUrl(resolveUrl(baseUrl, url), options.params, sensitiveParams);
|
|
731
847
|
const { requestUrl } = serializedUrl;
|
|
732
|
-
const
|
|
733
|
-
if (!hasHeader(headers, "Cookie")) {
|
|
734
|
-
const cookieHeader = cookieJar.toHeader(requestUrl);
|
|
735
|
-
if (cookieHeader)
|
|
736
|
-
headers.Cookie = cookieHeader;
|
|
737
|
-
}
|
|
738
|
-
const requestInit = {
|
|
739
|
-
headers: normalizeHeaders(headers),
|
|
740
|
-
method,
|
|
741
|
-
...(options.redirect ? { redirect: options.redirect } : {}),
|
|
742
|
-
...(options.timeout ? { timeout: options.timeout } : {}),
|
|
743
|
-
};
|
|
744
|
-
if (options.body !== undefined) {
|
|
745
|
-
requestInit.body = normalizeBody(options.body);
|
|
746
|
-
}
|
|
747
|
-
const response = await getClient(profileName, proxy, ignoreTlsErrors).fetch(requestUrl, requestInit);
|
|
748
|
-
const normalized = await normalizeResponse(response, requestUrl, options.maxBodyBytes);
|
|
749
|
-
cookieJar.setFromCookieStrings(setCookieHeadersFromResponse(response.headers), response.url ?? requestUrl);
|
|
848
|
+
const { normalized, response } = await withClient(profileName, proxy, ignoreTlsErrors, (transport) => fetchStealthRedirectChain(transport, cookieJar, requestUrl, method, options));
|
|
750
849
|
if (proxy && isProxyConnectFailureResponse(response, normalized.body)) {
|
|
751
850
|
throw createProxyConnectFailureError(normalized.body);
|
|
752
851
|
}
|
|
@@ -1043,18 +1142,29 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
1043
1142
|
},
|
|
1044
1143
|
close() {
|
|
1045
1144
|
closed = true;
|
|
1145
|
+
for (const client of clients.values()) {
|
|
1146
|
+
void client.session
|
|
1147
|
+
.then((session) => session.close())
|
|
1148
|
+
.catch((error) => {
|
|
1149
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1150
|
+
warn(`[provider-sdk] Failed to close stealth transport session: ${message}`);
|
|
1151
|
+
});
|
|
1152
|
+
}
|
|
1046
1153
|
clients.clear();
|
|
1047
1154
|
},
|
|
1048
1155
|
};
|
|
1049
1156
|
return session;
|
|
1050
1157
|
async function classifyProxyAuthDiagnostic(profileName, proxy) {
|
|
1051
1158
|
try {
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1159
|
+
return await withClient(profileName, proxy, false, async (client) => {
|
|
1160
|
+
await client.clearCookies();
|
|
1161
|
+
const response = await client.fetch(PROXY_AUTH_DIAGNOSTIC_URL, {
|
|
1162
|
+
method: "GET",
|
|
1163
|
+
timeout: PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS,
|
|
1164
|
+
});
|
|
1165
|
+
const normalized = await normalizeResponse(response);
|
|
1166
|
+
return classifyProxyAuthDiagnosticMessage(normalized.body);
|
|
1055
1167
|
});
|
|
1056
|
-
const normalized = await normalizeResponse(response);
|
|
1057
|
-
return classifyProxyAuthDiagnosticMessage(normalized.body);
|
|
1058
1168
|
}
|
|
1059
1169
|
catch (error) {
|
|
1060
1170
|
const message = error instanceof Error
|
package/dist/server/serve.js
CHANGED
|
@@ -3,6 +3,7 @@ import { join } from "node:path";
|
|
|
3
3
|
import { Hono } from "hono";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
import { AuthAbortError, createAuthFlowHelpers } from "../auth.js";
|
|
6
|
+
import { validateFailClosedDeclaration } from "../declaration-validation.js";
|
|
6
7
|
import { SDK_OWNED_PROVIDER_ERROR_CODES, SDK_RUNTIME_OWNED_ERROR_CODES, SDK_STATUS_MAPPED_PROVIDER_ERROR_CODES, } from "../error-resolution.js";
|
|
7
8
|
import { AuthError, isProviderError, isSessionExpiredError, isTransportError, isValidationError, ProviderError, } from "../errors.js";
|
|
8
9
|
import { loadProviderLocaleCatalogs, localizeAuthTurn, } from "../i18n/catalog.js";
|
|
@@ -1298,6 +1299,11 @@ function parseStatefulForwardingEnvelope(rawBody) {
|
|
|
1298
1299
|
});
|
|
1299
1300
|
}
|
|
1300
1301
|
export function createServerApp(provider, options = {}) {
|
|
1302
|
+
// Fail-closed validation runs here rather than only in serve(): createServerApp
|
|
1303
|
+
// is a public export, so a cast-bypassed declaration would otherwise reach the
|
|
1304
|
+
// request path unvalidated. serve() calls into this function, so validating
|
|
1305
|
+
// here covers both entry points exactly once per app construction.
|
|
1306
|
+
validateFailClosedDeclaration(provider);
|
|
1301
1307
|
validateStatefulServerConfig(options);
|
|
1302
1308
|
const app = new Hono();
|
|
1303
1309
|
const logger = options.logger ?? defaultProviderServerLogger;
|
|
@@ -1626,6 +1632,8 @@ const DEFAULT_SHUTDOWN_TIMEOUT_MS = 30_000;
|
|
|
1626
1632
|
const DEFAULT_SHUTDOWN_SIGNALS = ["SIGTERM", "SIGINT"];
|
|
1627
1633
|
const processSignalCoordinators = new Map();
|
|
1628
1634
|
export async function serve(provider, options = {}) {
|
|
1635
|
+
// Declaration validation happens inside createServerApp (the shared app
|
|
1636
|
+
// construction path), so serve() does not duplicate the call here.
|
|
1629
1637
|
const bunRuntime = getBunServeRuntime();
|
|
1630
1638
|
if (bunRuntime === undefined) {
|
|
1631
1639
|
throw new ProviderError("Bun runtime is required to start the provider server", {
|
package/dist/stealth/profiles.js
CHANGED
|
@@ -57,7 +57,7 @@ const FIREFOX_H2_SETTINGS = {
|
|
|
57
57
|
MAX_HEADER_LIST_SIZE: 65536,
|
|
58
58
|
};
|
|
59
59
|
const SAFARI_H2_SETTINGS = {
|
|
60
|
-
|
|
60
|
+
// Safari 17 also sends a connection-level WINDOW_UPDATE increment of 10485760.
|
|
61
61
|
ENABLE_PUSH: 0,
|
|
62
62
|
INITIAL_WINDOW_SIZE: 4194304,
|
|
63
63
|
MAX_CONCURRENT_STREAMS: 100,
|
|
@@ -162,8 +162,8 @@ const STEALTH_PROFILES = {
|
|
|
162
162
|
}),
|
|
163
163
|
"firefox-132": createProfile("firefox-132", {
|
|
164
164
|
platform: "macos",
|
|
165
|
-
version: "
|
|
166
|
-
userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:
|
|
165
|
+
version: "133.0",
|
|
166
|
+
userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0",
|
|
167
167
|
tlsClientIdentifier: "firefox_132",
|
|
168
168
|
ja3: FIREFOX_JA3,
|
|
169
169
|
h2Settings: FIREFOX_H2_SETTINGS,
|
|
@@ -178,6 +178,15 @@ const STEALTH_PROFILES = {
|
|
|
178
178
|
h2Settings: SAFARI_H2_SETTINGS,
|
|
179
179
|
headerOrder: SAFARI_HEADER_ORDER,
|
|
180
180
|
}),
|
|
181
|
+
"safari-17": createProfile("safari-17", {
|
|
182
|
+
platform: "macos",
|
|
183
|
+
version: "17.0",
|
|
184
|
+
userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15",
|
|
185
|
+
tlsClientIdentifier: "safari_17_0",
|
|
186
|
+
ja3: SAFARI_JA3,
|
|
187
|
+
h2Settings: SAFARI_H2_SETTINGS,
|
|
188
|
+
headerOrder: SAFARI_HEADER_ORDER,
|
|
189
|
+
}),
|
|
181
190
|
"safari-15": createProfile("safari-15", {
|
|
182
191
|
platform: "macos",
|
|
183
192
|
version: "15.6.1",
|
|
@@ -198,8 +207,8 @@ const STEALTH_PROFILES = {
|
|
|
198
207
|
}),
|
|
199
208
|
"ios-safari-18": createProfile("ios-safari-18", {
|
|
200
209
|
platform: "ios",
|
|
201
|
-
version: "18.
|
|
202
|
-
userAgent: "Mozilla/5.0 (iPhone; CPU iPhone OS
|
|
210
|
+
version: "18.1.1",
|
|
211
|
+
userAgent: "Mozilla/5.0 (iPhone; CPU iPhone OS 18_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1.1 Mobile/15E148 Safari/604.1",
|
|
203
212
|
tlsClientIdentifier: "safari_ios_18_0",
|
|
204
213
|
ja3: SAFARI_JA3,
|
|
205
214
|
h2Settings: SAFARI_H2_SETTINGS,
|
|
@@ -207,8 +216,8 @@ const STEALTH_PROFILES = {
|
|
|
207
216
|
}),
|
|
208
217
|
"ios-safari-17": createProfile("ios-safari-17", {
|
|
209
218
|
platform: "ios",
|
|
210
|
-
version: "17.
|
|
211
|
-
userAgent: "Mozilla/5.0 (iPhone; CPU iPhone OS
|
|
219
|
+
version: "17.2",
|
|
220
|
+
userAgent: "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1",
|
|
212
221
|
tlsClientIdentifier: "safari_ios_17_0",
|
|
213
222
|
ja3: SAFARI_JA3,
|
|
214
223
|
h2Settings: SAFARI_H2_SETTINGS,
|
package/dist/types.d.ts
CHANGED
|
@@ -298,9 +298,27 @@ export type ProviderChallenge = {
|
|
|
298
298
|
} | {
|
|
299
299
|
readonly kind: "aws_waf";
|
|
300
300
|
readonly pageUrl: string;
|
|
301
|
+
/** `window.gokuProps.key`; solver vendors require it, while `"browser"` does not. */
|
|
302
|
+
readonly siteKey?: string;
|
|
301
303
|
readonly captchaScript?: string;
|
|
302
304
|
readonly context?: string;
|
|
303
305
|
readonly iv?: string;
|
|
306
|
+
} | {
|
|
307
|
+
readonly kind: "akamai_sec_cpt";
|
|
308
|
+
readonly pageUrl: string;
|
|
309
|
+
/** The admitted challenge document, needed for tile/context extraction. */
|
|
310
|
+
readonly challengeHtml?: string;
|
|
311
|
+
} | {
|
|
312
|
+
readonly kind: "akamai_sensor";
|
|
313
|
+
readonly pageUrl: string;
|
|
314
|
+
/** Upstream sensor script URL the payload must be POSTed to. */
|
|
315
|
+
readonly scriptUrl: string;
|
|
316
|
+
/** Current `_abck` cookie value, rotates each round. */
|
|
317
|
+
readonly abck?: string;
|
|
318
|
+
/** Current `bm_sz` / `ak_bmsc` value when the upstream set one. */
|
|
319
|
+
readonly bmsz?: string;
|
|
320
|
+
/** Bot Manager major version when known ("3" measured on zozo.jp). */
|
|
321
|
+
readonly version?: string;
|
|
304
322
|
};
|
|
305
323
|
export type ProviderChallengeKind = ProviderChallenge["kind"];
|
|
306
324
|
/**
|
|
@@ -317,12 +335,21 @@ export type ChallengeSolution = {
|
|
|
317
335
|
readonly form: "cookies";
|
|
318
336
|
readonly cookies: Readonly<Record<string, string>>;
|
|
319
337
|
readonly userAgent: string;
|
|
338
|
+
/** Epoch seconds copied from the upstream cookie's own expiry attribute; never a constant. */
|
|
339
|
+
readonly expires?: number;
|
|
320
340
|
};
|
|
321
341
|
export interface ProviderResolverConfig {
|
|
322
342
|
/** Ordered vendor fallback chain, tried first to last. */
|
|
323
343
|
readonly vendors: readonly ProviderResolverVendor[];
|
|
324
344
|
/** Challenge kinds this provider is permitted to request. */
|
|
325
345
|
readonly kinds: readonly ProviderChallengeKind[];
|
|
346
|
+
/**
|
|
347
|
+
* Client fingerprint profile the SDK must use when reaching this upstream.
|
|
348
|
+
* Measured on zozo.jp: Chrome/Firefox profiles are refused 403 before any
|
|
349
|
+
* challenge is served, while a Safari profile is admitted. Provider-declared
|
|
350
|
+
* because only the provider knows its upstream's admission rule.
|
|
351
|
+
*/
|
|
352
|
+
readonly clientProfile?: string;
|
|
326
353
|
}
|
|
327
354
|
export type SttAudioInput = {
|
|
328
355
|
kind: "base64";
|
|
@@ -534,7 +561,13 @@ export interface HealthJourneyDefinition {
|
|
|
534
561
|
requiredSecrets?: readonly string[];
|
|
535
562
|
manualTrigger?: HealthJourneyManualTriggerPolicy;
|
|
536
563
|
steps: readonly [HealthJourneyStep, ...HealthJourneyStep[]];
|
|
537
|
-
|
|
564
|
+
/**
|
|
565
|
+
* Required: a journey always declares `coversOperations`, and the health
|
|
566
|
+
* monitor reports a run-less journey as `journey_run_missing`. Declaration
|
|
567
|
+
* validation rejects a missing `run` (`health-journey-executable`), so this
|
|
568
|
+
* is typed required to fail at compile time rather than at boot.
|
|
569
|
+
*/
|
|
570
|
+
run: (ctx: HealthJourneyRunContext) => Promise<HealthJourneyRunResult | undefined>;
|
|
538
571
|
}
|
|
539
572
|
/**
|
|
540
573
|
* Health-check authoring surface owned by `@apifuse/provider-sdk`.
|
package/package.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"version": "2.2.0-beta.
|
|
2
|
+
"version": "2.2.0-beta.27",
|
|
3
3
|
"name": "@apifuse/provider-sdk",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
@@ -106,7 +106,6 @@
|
|
|
106
106
|
"acorn": "^8.17.0",
|
|
107
107
|
"ajv": "^8.17",
|
|
108
108
|
"hono": "^4.12.25",
|
|
109
|
-
"impit": "0.14.1",
|
|
110
109
|
"ioredis": "^5.11.1",
|
|
111
110
|
"ms": "^2.1.3",
|
|
112
111
|
"playwright": "^1.55.1",
|
|
@@ -116,6 +115,7 @@
|
|
|
116
115
|
"safe-regex": "^2.1",
|
|
117
116
|
"socks": "^2.8.9",
|
|
118
117
|
"tough-cookie": "^6.0.2",
|
|
118
|
+
"wreq-js": "3.0.0",
|
|
119
119
|
"zod": "^4.4.3"
|
|
120
120
|
},
|
|
121
121
|
"repository": {
|