@apifuse/provider-sdk 2.2.0-beta.25 → 2.2.0-beta.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +7 -6
- package/CHANGELOG.md +5 -1
- package/README.md +3 -3
- package/bin/apifuse-check.ts +62 -3
- package/bin/apifuse-pack-check.ts +8 -2
- package/bin/apifuse-pack-smoke.ts +43 -2
- package/bin/apifuse-pack-types.ts +58 -0
- package/dist/auth.js +29 -0
- package/dist/cli/templates/provider/README.md.tpl +4 -4
- package/dist/contract-serialization.d.ts +20 -1
- package/dist/contract-serialization.js +583 -8
- package/dist/contract.d.ts +2 -0
- package/dist/contract.js +9 -5
- package/dist/declaration-validation.d.ts +23 -0
- package/dist/declaration-validation.js +159 -0
- package/dist/define.d.ts +1 -1
- package/dist/define.js +13 -2
- package/dist/index.d.ts +3 -2
- package/dist/index.js +2 -2
- package/dist/lint.js +85 -3
- package/dist/provider.d.ts +1 -1
- package/dist/provider.js +1 -1
- package/dist/runtime/resolver-vendors/bindings.d.ts +42 -2
- package/dist/runtime/resolver-vendors/bindings.js +31 -6
- package/dist/runtime/resolver-vendors/browser.d.ts +3 -7
- package/dist/runtime/resolver-vendors/browser.js +7 -22
- package/dist/runtime/resolver-vendors/hosts.d.ts +2 -0
- package/dist/runtime/resolver-vendors/hosts.js +33 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +23 -0
- package/dist/runtime/resolver-vendors/twocaptcha.js +264 -0
- package/dist/runtime/resolver-vendors/types.d.ts +44 -3
- package/dist/runtime/resolver-vendors/types.js +10 -0
- package/dist/runtime/resolver.d.ts +17 -2
- package/dist/runtime/resolver.js +237 -15
- package/dist/runtime/stealth.d.ts +26 -4
- package/dist/runtime/stealth.js +224 -114
- package/dist/schema.d.ts +63 -0
- package/dist/schema.js +808 -8
- package/dist/server/serve.js +8 -0
- package/dist/stealth/profiles.js +16 -7
- package/dist/types.d.ts +34 -1
- package/package.json +2 -2
- package/src/auth.ts +40 -0
- package/src/cli/templates/provider/README.md.tpl +4 -4
- package/src/contract-serialization.ts +857 -8
- package/src/contract.ts +16 -5
- package/src/declaration-validation.ts +202 -0
- package/src/define.ts +23 -2
- package/src/index.ts +12 -0
- package/src/lint.ts +98 -3
- package/src/provider.ts +10 -0
- package/src/runtime/resolver-vendors/bindings.ts +40 -15
- package/src/runtime/resolver-vendors/browser.ts +9 -31
- package/src/runtime/resolver-vendors/hosts.ts +38 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +366 -0
- package/src/runtime/resolver-vendors/types.ts +54 -0
- package/src/runtime/resolver.ts +304 -24
- package/src/runtime/stealth.ts +317 -136
- package/src/schema.ts +1060 -9
- package/src/server/serve.ts +8 -0
- package/src/stealth/profiles.ts +17 -7
- package/src/types.ts +36 -3
package/dist/runtime/stealth.js
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { Impit } from "impit";
|
|
3
2
|
import { Cookie, CookieJar as ToughCookieJar } from "tough-cookie";
|
|
4
3
|
import { DEFAULT_SMARTPROXY_POOL_SIZE, invalidateProxyResolutionCacheAsync, ProxyResolutionError, policyResolvesRegistryVendorChain, resolvePolicyProxyPoolSpan, resolvePolicyTransportAttemptCap, resolveProxyConfigAsync, vendorFromResolvedSource, } from "../config/loader.js";
|
|
5
4
|
import { SDKError, StealthCookieStoreVersionError, TransportError } from "../errors.js";
|
|
6
5
|
import { getStealthProfile } from "../stealth/profiles.js";
|
|
7
6
|
import { createProxyAuthIpDeniedError, createProxyEdgeAuthRejectedError, createProxyEdgeTlsRejectedError, createProxyPoolExhaustedError, createProxyPoolStaleError, isProxyAuthIpDeniedMessage, isProxyEdgeAuthRejectedMessage, isProxyEdgeTlsRejectedResponse, isProxyPoolRefreshableError, isProxyPoolStaleMessage, isProxyPoolStaleStatus, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_STALE_CODE, } from "./proxy-errors.js";
|
|
8
7
|
import { computeProxyAttemptIndex, computeProxyTransportRetryDelayMs, createDefaultProxyTransportRetryOptions, normalizeProxyTransportRetryOptions, shouldRetryProxyTransportAttempt, validateUnsafeProxyTransportRetryMethods, } from "./proxy-retry-policy.js";
|
|
9
|
-
import { evaluateRedirectHop, isRedirectStatus, resolveRedirectUrl, } from "./redirects.js";
|
|
10
|
-
import { isSensitiveKey, redactSensitiveError, redactSensitiveRequestError, redactSensitiveText, redactUrlQueryParams,
|
|
8
|
+
import { evaluateRedirectHop, isRedirectStatus, nextRedirectMethod, resolveRedirectUrl, } from "./redirects.js";
|
|
9
|
+
import { isSensitiveKey, normalizeSensitiveParams, redactSensitiveError, redactSensitiveRequestError, redactSensitiveText, redactUrlQueryParams, serializeRequestUrl, } from "./request-options.js";
|
|
11
10
|
const DEFAULT_PROFILE = "chrome-146";
|
|
12
11
|
const MISSING_PROXY_WARNING = "[provider-sdk] Provider requested proxy routing, but no proxy URL was configured. Continuing without proxy.";
|
|
13
12
|
const MAX_POLICY_PROXY_POOL_REFRESHES = 1;
|
|
@@ -16,6 +15,13 @@ const PROXY_CONNECT_FAILURE_BODY_PATTERN = /\bproxy\b.*\b(non[\s-]?200|connect|t
|
|
|
16
15
|
const PROXY_AUTH_DIAGNOSTIC_URL = "http://example.com/";
|
|
17
16
|
const PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS = 5_000;
|
|
18
17
|
const STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES = [PROXY_CONNECT_FAILURE_CODE];
|
|
18
|
+
const MAX_STEALTH_REDIRECT_HOPS = 10;
|
|
19
|
+
const REDIRECT_BODY_HEADERS = new Set([
|
|
20
|
+
"content-encoding",
|
|
21
|
+
"content-language",
|
|
22
|
+
"content-location",
|
|
23
|
+
"content-type",
|
|
24
|
+
]);
|
|
19
25
|
function sensitiveQueryParamNames(url) {
|
|
20
26
|
const queryStart = url.indexOf("?");
|
|
21
27
|
if (queryStart === -1)
|
|
@@ -37,25 +43,6 @@ const REMOVED_CHROME_PROFILE_NAMES = new Set([
|
|
|
37
43
|
"chrome-130-psk",
|
|
38
44
|
"edge-131",
|
|
39
45
|
]);
|
|
40
|
-
const CHROME_IMPIT_BY_MAJOR = {
|
|
41
|
-
100: "chrome100",
|
|
42
|
-
101: "chrome101",
|
|
43
|
-
104: "chrome104",
|
|
44
|
-
107: "chrome107",
|
|
45
|
-
110: "chrome110",
|
|
46
|
-
116: "chrome116",
|
|
47
|
-
124: "chrome124",
|
|
48
|
-
125: "chrome125",
|
|
49
|
-
131: "chrome131",
|
|
50
|
-
136: "chrome136",
|
|
51
|
-
142: "chrome142",
|
|
52
|
-
};
|
|
53
|
-
const FIREFOX_IMPIT_BY_MAJOR = {
|
|
54
|
-
128: "firefox128",
|
|
55
|
-
133: "firefox133",
|
|
56
|
-
135: "firefox135",
|
|
57
|
-
144: "firefox144",
|
|
58
|
-
};
|
|
59
46
|
function isRecord(value) {
|
|
60
47
|
return typeof value === "object" && value !== null;
|
|
61
48
|
}
|
|
@@ -182,47 +169,93 @@ class CookieJarImpl {
|
|
|
182
169
|
});
|
|
183
170
|
}
|
|
184
171
|
}
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
172
|
+
let wreqModulePromise;
|
|
173
|
+
function getWreqModule() {
|
|
174
|
+
if (!wreqModulePromise) {
|
|
175
|
+
wreqModulePromise = import("wreq-js").catch((error) => {
|
|
176
|
+
throw new SDKError(`Stealth transport is unavailable on ${process.platform}-${process.arch}: the wreq-js native binary could not be loaded.`, {
|
|
177
|
+
code: "stealth_transport_unavailable",
|
|
178
|
+
cause: error instanceof Error ? error : undefined,
|
|
179
|
+
});
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
return wreqModulePromise;
|
|
183
|
+
}
|
|
184
|
+
function parseProfileIdentifier(identifier) {
|
|
185
|
+
const match = /^(safari_ios|safari_ipad|firefox_android|firefox_private|chrome|edge|firefox|opera|safari|okhttp)_(\d+(?:[._]\d+)*)$/.exec(identifier.toLowerCase());
|
|
186
|
+
if (!match?.[1] || !match[2])
|
|
187
|
+
return null;
|
|
188
|
+
return {
|
|
189
|
+
family: match[1],
|
|
190
|
+
version: match[2].split(/[._]/).map(Number),
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
function compareVersionDistance(target, left, right) {
|
|
194
|
+
const width = Math.max(target.length, left.length, right.length);
|
|
195
|
+
for (let index = 0; index < width; index += 1) {
|
|
196
|
+
const targetPart = target[index] ?? 0;
|
|
197
|
+
const leftDistance = Math.abs((left[index] ?? 0) - targetPart);
|
|
198
|
+
const rightDistance = Math.abs((right[index] ?? 0) - targetPart);
|
|
199
|
+
if (leftDistance !== rightDistance)
|
|
200
|
+
return leftDistance - rightDistance;
|
|
201
|
+
}
|
|
202
|
+
return 0;
|
|
203
|
+
}
|
|
204
|
+
function closestWreqProfile(identifier, wreqProfiles) {
|
|
205
|
+
const requested = parseProfileIdentifier(identifier);
|
|
206
|
+
if (!requested)
|
|
207
|
+
return undefined;
|
|
208
|
+
let closest;
|
|
209
|
+
for (const candidateName of wreqProfiles) {
|
|
210
|
+
const candidate = parseProfileIdentifier(candidateName);
|
|
211
|
+
if (!candidate || candidate.family !== requested.family)
|
|
212
|
+
continue;
|
|
213
|
+
if (!closest ||
|
|
214
|
+
compareVersionDistance(requested.version, candidate.version, closest.version) < 0) {
|
|
215
|
+
closest = { name: candidateName, version: candidate.version };
|
|
194
216
|
}
|
|
195
217
|
}
|
|
196
|
-
return
|
|
218
|
+
return closest?.name;
|
|
219
|
+
}
|
|
220
|
+
function resolveDefaultWreqProfileMapping() {
|
|
221
|
+
let profile;
|
|
222
|
+
try {
|
|
223
|
+
profile = getStealthProfile(DEFAULT_PROFILE);
|
|
224
|
+
}
|
|
225
|
+
catch (error) {
|
|
226
|
+
throw new SDKError(`Default stealth profile "${DEFAULT_PROFILE}" cannot be mapped to a wreq-js browser profile.`, { cause: error instanceof Error ? error : undefined });
|
|
227
|
+
}
|
|
228
|
+
const identifier = profile.tlsClientIdentifier?.toLowerCase() ?? "";
|
|
229
|
+
if (!parseProfileIdentifier(identifier)) {
|
|
230
|
+
throw new SDKError(`Default stealth profile "${DEFAULT_PROFILE}" cannot be mapped to a wreq-js browser profile.`);
|
|
231
|
+
}
|
|
232
|
+
return { identifier, os: profile.platform };
|
|
197
233
|
}
|
|
198
|
-
|
|
234
|
+
const DEFAULT_WREQ_PROFILE_MAPPING = resolveDefaultWreqProfileMapping();
|
|
235
|
+
export function resolveWreqProfile(profileName, wreqProfiles) {
|
|
199
236
|
if (REMOVED_CHROME_PROFILE_NAMES.has(profileName)) {
|
|
200
237
|
throw new SDKError(`Unknown stealth profile: ${profileName}`);
|
|
201
238
|
}
|
|
202
|
-
let
|
|
239
|
+
let identifier;
|
|
240
|
+
let os;
|
|
203
241
|
try {
|
|
204
|
-
profile = getStealthProfile(profileName);
|
|
242
|
+
const profile = getStealthProfile(profileName);
|
|
243
|
+
identifier = profile.tlsClientIdentifier?.toLowerCase() ?? "";
|
|
244
|
+
os = profile.platform;
|
|
205
245
|
}
|
|
206
246
|
catch {
|
|
207
247
|
// Preserve the previous ctx.stealth.fetch() compatibility behavior: unknown
|
|
208
248
|
// profile strings still run with the transport default instead of failing
|
|
209
249
|
// before the request starts. Removed built-in profile aliases above remain
|
|
210
250
|
// explicit errors so callers do not accidentally pin retired fingerprints.
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
const identifier = profile.tlsClientIdentifier?.toLowerCase() ?? "";
|
|
214
|
-
const chromeMatch = /^(?:chrome|edge)_(\d+)/.exec(identifier);
|
|
215
|
-
if (chromeMatch?.[1]) {
|
|
216
|
-
return closestImpitBrowser(Number(chromeMatch[1]), CHROME_IMPIT_BY_MAJOR);
|
|
251
|
+
identifier = DEFAULT_WREQ_PROFILE_MAPPING.identifier;
|
|
252
|
+
os = DEFAULT_WREQ_PROFILE_MAPPING.os;
|
|
217
253
|
}
|
|
218
|
-
const
|
|
219
|
-
if (
|
|
220
|
-
|
|
254
|
+
const browser = closestWreqProfile(identifier, wreqProfiles);
|
|
255
|
+
if (!browser) {
|
|
256
|
+
throw new SDKError(`Stealth profile "${profileName}" cannot be mapped to a wreq-js browser profile.`);
|
|
221
257
|
}
|
|
222
|
-
|
|
223
|
-
throw new SDKError(`Stealth profile "${profileName}" uses a Safari stealth fingerprint, but TypeScript ctx.stealth uses impit which currently supports Chrome, Firefox, and OkHttp profiles only. Use a Chrome/Firefox stealth profile for ctx.stealth or ctx.browser for Safari-specific behavior.`);
|
|
224
|
-
}
|
|
225
|
-
throw new SDKError(`Stealth profile "${profileName}" cannot be mapped to an impit browser profile.`);
|
|
258
|
+
return { browser, os };
|
|
226
259
|
}
|
|
227
260
|
function resolveUrl(baseUrl, url) {
|
|
228
261
|
return new URL(url, baseUrl).toString();
|
|
@@ -242,18 +275,6 @@ function normalizeHeaders(headers) {
|
|
|
242
275
|
function hasOwn(object, key) {
|
|
243
276
|
return Object.hasOwn(object, key);
|
|
244
277
|
}
|
|
245
|
-
function toImpitCookieJar(cookieJar) {
|
|
246
|
-
return {
|
|
247
|
-
setCookie(cookie, url, cb) {
|
|
248
|
-
cookieJar.setFromCookieStrings([cookie], url);
|
|
249
|
-
if (typeof cb === "function")
|
|
250
|
-
cb();
|
|
251
|
-
},
|
|
252
|
-
getCookieString(url) {
|
|
253
|
-
return cookieJar.toHeader(url);
|
|
254
|
-
},
|
|
255
|
-
};
|
|
256
|
-
}
|
|
257
278
|
function assertNoUnsupportedFingerprintOverrides(options) {
|
|
258
279
|
if (!isRecord(options))
|
|
259
280
|
return;
|
|
@@ -267,7 +288,7 @@ function assertNoUnsupportedFingerprintOverrides(options) {
|
|
|
267
288
|
unsupported.push("stealth.h2");
|
|
268
289
|
if (unsupported.length === 0)
|
|
269
290
|
return;
|
|
270
|
-
throw new SDKError(`ctx.stealth.fetch uses
|
|
291
|
+
throw new SDKError(`ctx.stealth.fetch uses transport-managed browser fingerprints and no longer accepts low-level stealth overrides: ${unsupported.join(", ")}. Use the profile option instead.`);
|
|
271
292
|
}
|
|
272
293
|
function responseHeadersToRecord(headers) {
|
|
273
294
|
const record = {};
|
|
@@ -348,23 +369,10 @@ function declaredContentLength(headers) {
|
|
|
348
369
|
const parsed = Number(contentLength);
|
|
349
370
|
return Number.isFinite(parsed) ? parsed : undefined;
|
|
350
371
|
}
|
|
351
|
-
function abortTransportResponse(response) {
|
|
352
|
-
if (!response.abort)
|
|
353
|
-
return false;
|
|
354
|
-
try {
|
|
355
|
-
response.abort();
|
|
356
|
-
}
|
|
357
|
-
catch {
|
|
358
|
-
// The size error remains the primary failure if impit has already closed the response.
|
|
359
|
-
}
|
|
360
|
-
return true;
|
|
361
|
-
}
|
|
362
372
|
async function readResponseBodyWithLimit(response, maxBodyBytes) {
|
|
363
373
|
const contentLength = declaredContentLength(response.headers);
|
|
364
374
|
if (contentLength !== undefined && contentLength > maxBodyBytes) {
|
|
365
|
-
|
|
366
|
-
await response.body?.cancel().catch(() => undefined);
|
|
367
|
-
}
|
|
375
|
+
await response.body?.cancel().catch(() => undefined);
|
|
368
376
|
throw responseTooLargeError(maxBodyBytes, contentLength);
|
|
369
377
|
}
|
|
370
378
|
if (!response.body) {
|
|
@@ -382,10 +390,11 @@ async function readResponseBodyWithLimit(response, maxBodyBytes) {
|
|
|
382
390
|
const { done, value } = await reader.read();
|
|
383
391
|
if (done)
|
|
384
392
|
break;
|
|
393
|
+
if (!value)
|
|
394
|
+
continue;
|
|
385
395
|
receivedBytes += value.byteLength;
|
|
386
396
|
if (receivedBytes > maxBodyBytes) {
|
|
387
397
|
await reader.cancel().catch(() => undefined);
|
|
388
|
-
abortTransportResponse(response);
|
|
389
398
|
throw responseTooLargeError(maxBodyBytes, receivedBytes);
|
|
390
399
|
}
|
|
391
400
|
chunks.push(value);
|
|
@@ -566,31 +575,138 @@ function locationHeader(headers) {
|
|
|
566
575
|
}
|
|
567
576
|
return undefined;
|
|
568
577
|
}
|
|
578
|
+
function withoutRedirectBodyHeaders(headers) {
|
|
579
|
+
return Object.fromEntries(Object.entries(headers).filter(([name]) => !REDIRECT_BODY_HEADERS.has(name.toLowerCase())));
|
|
580
|
+
}
|
|
581
|
+
function assertStealthRedirectUrl(url) {
|
|
582
|
+
const protocol = new URL(url).protocol;
|
|
583
|
+
if (protocol !== "http:" && protocol !== "https:") {
|
|
584
|
+
throw new TransportError(`Stealth redirect target scheme "${protocol}" is not allowed`, {
|
|
585
|
+
code: "transport_invalid_url",
|
|
586
|
+
status: 0,
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
function discardStealthRedirectBody(response) {
|
|
591
|
+
try {
|
|
592
|
+
const cancellation = response.body?.cancel();
|
|
593
|
+
if (cancellation)
|
|
594
|
+
void cancellation.catch(() => undefined);
|
|
595
|
+
}
|
|
596
|
+
catch {
|
|
597
|
+
// Redirect handling is decided from status and headers. A cancellation
|
|
598
|
+
// failure must not replace or delay that decision.
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
async function fetchStealthRedirectChain(transport, cookieJar, requestUrl, method, options) {
|
|
602
|
+
let currentUrl = requestUrl;
|
|
603
|
+
let currentMethod = method;
|
|
604
|
+
let currentBody = options.body === undefined ? undefined : normalizeBody(options.body);
|
|
605
|
+
let currentHeaders = { ...(options.headers ?? {}) };
|
|
606
|
+
let followedHops = 0;
|
|
607
|
+
let response;
|
|
608
|
+
const deadline = options.timeout ? performance.now() + options.timeout : undefined;
|
|
609
|
+
while (true) {
|
|
610
|
+
const headers = { ...currentHeaders };
|
|
611
|
+
if (!hasHeader(headers, "Cookie")) {
|
|
612
|
+
const cookieHeader = cookieJar.toHeader(currentUrl);
|
|
613
|
+
if (cookieHeader)
|
|
614
|
+
headers.Cookie = cookieHeader;
|
|
615
|
+
}
|
|
616
|
+
const requestInit = {
|
|
617
|
+
headers: normalizeHeaders(headers),
|
|
618
|
+
method: currentMethod,
|
|
619
|
+
redirect: "manual",
|
|
620
|
+
};
|
|
621
|
+
if (currentBody !== undefined)
|
|
622
|
+
requestInit.body = currentBody;
|
|
623
|
+
await transport.clearCookies();
|
|
624
|
+
const remainingTimeout = deadline === undefined ? undefined : Math.ceil(deadline - performance.now());
|
|
625
|
+
if (remainingTimeout !== undefined && remainingTimeout <= 0) {
|
|
626
|
+
throw new TransportError("Request timed out", {
|
|
627
|
+
code: "transport_timeout",
|
|
628
|
+
status: 0,
|
|
629
|
+
});
|
|
630
|
+
}
|
|
631
|
+
if (remainingTimeout !== undefined)
|
|
632
|
+
requestInit.timeout = remainingTimeout;
|
|
633
|
+
response = await transport.fetch(currentUrl, requestInit);
|
|
634
|
+
cookieJar.setFromCookieStrings(setCookieHeadersFromResponse(response.headers), response.url ?? currentUrl);
|
|
635
|
+
if (!isRedirectStatus(response.status) || options.redirect === "manual")
|
|
636
|
+
break;
|
|
637
|
+
if (options.redirect === "error") {
|
|
638
|
+
discardStealthRedirectBody(response);
|
|
639
|
+
throw new TransportError("Stealth request encountered a redirect", {
|
|
640
|
+
code: "transport_network_error",
|
|
641
|
+
status: 0,
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
const nextUrl = resolveRedirectUrl(response.headers.get("location") ?? undefined, response.url ?? currentUrl);
|
|
645
|
+
if (!nextUrl)
|
|
646
|
+
break;
|
|
647
|
+
if (followedHops >= MAX_STEALTH_REDIRECT_HOPS) {
|
|
648
|
+
discardStealthRedirectBody(response);
|
|
649
|
+
throw new TransportError(`Stealth request exceeded the ${MAX_STEALTH_REDIRECT_HOPS}-redirect limit`, { code: "transport_network_error", status: 0 });
|
|
650
|
+
}
|
|
651
|
+
assertStealthRedirectUrl(nextUrl);
|
|
652
|
+
discardStealthRedirectBody(response);
|
|
653
|
+
const nextMethod = nextRedirectMethod(response.status, currentMethod);
|
|
654
|
+
if (nextMethod !== currentMethod) {
|
|
655
|
+
currentBody = undefined;
|
|
656
|
+
currentHeaders = withoutRedirectBodyHeaders(currentHeaders);
|
|
657
|
+
}
|
|
658
|
+
currentMethod = nextMethod;
|
|
659
|
+
currentUrl = nextUrl;
|
|
660
|
+
followedHops += 1;
|
|
661
|
+
}
|
|
662
|
+
const normalized = await normalizeResponse(response, currentUrl, options.maxBodyBytes);
|
|
663
|
+
if (followedHops > 0)
|
|
664
|
+
normalized.redirected = true;
|
|
665
|
+
return { normalized, response };
|
|
666
|
+
}
|
|
569
667
|
function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
570
668
|
const clients = new Map();
|
|
571
669
|
let closed = false;
|
|
572
670
|
let hasWarnedMissingProxy = false;
|
|
573
671
|
const warn = clientOptions.warn ?? console.warn;
|
|
574
672
|
const cookieJar = new CookieJarImpl([], baseUrl);
|
|
575
|
-
|
|
576
|
-
function getClient(profileName, proxyUrl, ignoreTlsErrors) {
|
|
673
|
+
async function getClientEntry(profileName, proxyUrl, ignoreTlsErrors) {
|
|
577
674
|
if (closed) {
|
|
578
675
|
throw new TransportError("Stealth session is closed", { status: 0 });
|
|
579
676
|
}
|
|
580
|
-
const
|
|
677
|
+
const wreq = await getWreqModule();
|
|
678
|
+
const { browser, os } = resolveWreqProfile(profileName, wreq.getProfiles());
|
|
581
679
|
const cacheKey = JSON.stringify({ browser, proxyUrl, ignoreTlsErrors });
|
|
582
|
-
let
|
|
583
|
-
if (!
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
680
|
+
let entry = clients.get(cacheKey);
|
|
681
|
+
if (!entry) {
|
|
682
|
+
entry = {
|
|
683
|
+
session: wreq.createSession({
|
|
684
|
+
browser,
|
|
685
|
+
os,
|
|
686
|
+
...(proxyUrl ? { proxy: proxyUrl } : {}),
|
|
687
|
+
...(ignoreTlsErrors ? { insecure: true } : {}),
|
|
688
|
+
timeout: 30_000,
|
|
689
|
+
}),
|
|
690
|
+
tail: Promise.resolve(),
|
|
691
|
+
};
|
|
692
|
+
clients.set(cacheKey, entry);
|
|
693
|
+
}
|
|
694
|
+
return entry;
|
|
695
|
+
}
|
|
696
|
+
async function withClient(profileName, proxyUrl, ignoreTlsErrors, operation) {
|
|
697
|
+
const entry = await getClientEntry(profileName, proxyUrl, ignoreTlsErrors);
|
|
698
|
+
const previous = entry.tail;
|
|
699
|
+
let release;
|
|
700
|
+
entry.tail = new Promise((resolve) => {
|
|
701
|
+
release = resolve;
|
|
702
|
+
});
|
|
703
|
+
await previous;
|
|
704
|
+
try {
|
|
705
|
+
return await operation(await entry.session);
|
|
706
|
+
}
|
|
707
|
+
finally {
|
|
708
|
+
release();
|
|
592
709
|
}
|
|
593
|
-
return client;
|
|
594
710
|
}
|
|
595
711
|
async function resolveRequestProxy(options, proxyAttempt, refreshEpoch) {
|
|
596
712
|
const resolvedProxy = await resolveProxyConfigAsync({
|
|
@@ -602,7 +718,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
602
718
|
proxyAttemptOffset: options?.proxyAttemptOffset,
|
|
603
719
|
retryAttemptOffset: proxyAttempt,
|
|
604
720
|
}),
|
|
605
|
-
// The
|
|
721
|
+
// The stealth transport tunnels both HTTP CONNECT and SOCKS5,
|
|
606
722
|
// preserving the client TLS fingerprint end-to-end.
|
|
607
723
|
transportProtocols: ["http", "socks5"],
|
|
608
724
|
...(refreshEpoch === undefined ? {} : { proxyRefreshEpoch: refreshEpoch }),
|
|
@@ -729,24 +845,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
729
845
|
const profileName = options.profile ?? defaultProfile;
|
|
730
846
|
serializedUrl = serializeRequestUrl(resolveUrl(baseUrl, url), options.params, sensitiveParams);
|
|
731
847
|
const { requestUrl } = serializedUrl;
|
|
732
|
-
const
|
|
733
|
-
if (!hasHeader(headers, "Cookie")) {
|
|
734
|
-
const cookieHeader = cookieJar.toHeader(requestUrl);
|
|
735
|
-
if (cookieHeader)
|
|
736
|
-
headers.Cookie = cookieHeader;
|
|
737
|
-
}
|
|
738
|
-
const requestInit = {
|
|
739
|
-
headers: normalizeHeaders(headers),
|
|
740
|
-
method,
|
|
741
|
-
...(options.redirect ? { redirect: options.redirect } : {}),
|
|
742
|
-
...(options.timeout ? { timeout: options.timeout } : {}),
|
|
743
|
-
};
|
|
744
|
-
if (options.body !== undefined) {
|
|
745
|
-
requestInit.body = normalizeBody(options.body);
|
|
746
|
-
}
|
|
747
|
-
const response = await getClient(profileName, proxy, ignoreTlsErrors).fetch(requestUrl, requestInit);
|
|
748
|
-
const normalized = await normalizeResponse(response, requestUrl, options.maxBodyBytes);
|
|
749
|
-
cookieJar.setFromCookieStrings(setCookieHeadersFromResponse(response.headers), response.url ?? requestUrl);
|
|
848
|
+
const { normalized, response } = await withClient(profileName, proxy, ignoreTlsErrors, (transport) => fetchStealthRedirectChain(transport, cookieJar, requestUrl, method, options));
|
|
750
849
|
if (proxy && isProxyConnectFailureResponse(response, normalized.body)) {
|
|
751
850
|
throw createProxyConnectFailureError(normalized.body);
|
|
752
851
|
}
|
|
@@ -1043,18 +1142,29 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
1043
1142
|
},
|
|
1044
1143
|
close() {
|
|
1045
1144
|
closed = true;
|
|
1145
|
+
for (const client of clients.values()) {
|
|
1146
|
+
void client.session
|
|
1147
|
+
.then((session) => session.close())
|
|
1148
|
+
.catch((error) => {
|
|
1149
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1150
|
+
warn(`[provider-sdk] Failed to close stealth transport session: ${message}`);
|
|
1151
|
+
});
|
|
1152
|
+
}
|
|
1046
1153
|
clients.clear();
|
|
1047
1154
|
},
|
|
1048
1155
|
};
|
|
1049
1156
|
return session;
|
|
1050
1157
|
async function classifyProxyAuthDiagnostic(profileName, proxy) {
|
|
1051
1158
|
try {
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1159
|
+
return await withClient(profileName, proxy, false, async (client) => {
|
|
1160
|
+
await client.clearCookies();
|
|
1161
|
+
const response = await client.fetch(PROXY_AUTH_DIAGNOSTIC_URL, {
|
|
1162
|
+
method: "GET",
|
|
1163
|
+
timeout: PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS,
|
|
1164
|
+
});
|
|
1165
|
+
const normalized = await normalizeResponse(response);
|
|
1166
|
+
return classifyProxyAuthDiagnosticMessage(normalized.body);
|
|
1055
1167
|
});
|
|
1056
|
-
const normalized = await normalizeResponse(response);
|
|
1057
|
-
return classifyProxyAuthDiagnosticMessage(normalized.body);
|
|
1058
1168
|
}
|
|
1059
1169
|
catch (error) {
|
|
1060
1170
|
const message = error instanceof Error
|
package/dist/schema.d.ts
CHANGED
|
@@ -13,7 +13,27 @@ export declare function safeParseSchemaSync<TSchema extends SchemaLike>(schema:
|
|
|
13
13
|
export declare const APIFUSE_SENSITIVE_META_KEY = "x-apifuse-sensitive";
|
|
14
14
|
export declare const APIFUSE_SENSITIVE_KIND_META_KEY = "x-apifuse-sensitive-kind";
|
|
15
15
|
export declare const APIFUSE_DESCRIPTION_KEY_META_KEY = "x-apifuse-description-key";
|
|
16
|
+
export declare const APIFUSE_TEXT_TRUST_META_KEY = "x-apifuse-text-trust";
|
|
16
17
|
export declare const APIFUSE_REDACTION_MARKER = "<redacted>";
|
|
18
|
+
export type TextTrust = "trusted" | "untrusted";
|
|
19
|
+
export interface TextTrustMetadata {
|
|
20
|
+
readonly v: 1;
|
|
21
|
+
readonly trust: TextTrust;
|
|
22
|
+
readonly carrier?: "container";
|
|
23
|
+
}
|
|
24
|
+
export type OutputTextTrustMap = Readonly<Record<string, TextTrust>>;
|
|
25
|
+
/**
|
|
26
|
+
* Auto-trusted output leaves are string-valued `z.literal()` / `z.enum()`,
|
|
27
|
+
* patterns whose every top-level alternative is anchored at both ends and
|
|
28
|
+
* whose bodies use only finite quantifiers, have a maximum match length of 32,
|
|
29
|
+
* and contain only digits and safe punctuation in positive whitespace-free
|
|
30
|
+
* character classes, literals/escapes, and ordinary or non-capturing groups,
|
|
31
|
+
* plus the structurally constrained formats listed here. A brand is trusted
|
|
32
|
+
* only when its underlying schema meets one of those rules. Letter-bearing
|
|
33
|
+
* identifiers, length-only strings, and other formats (including email, URL,
|
|
34
|
+
* base64, and JWT) are not trusted.
|
|
35
|
+
*/
|
|
36
|
+
export declare const AUTO_TRUSTED_ZOD_STRING_FORMATS: readonly ["cidrv4", "cidrv6", "date", "datetime", "duration", "e164", "ipv4", "ipv6", "time"];
|
|
17
37
|
export type SensitivePathSegment = string | "*";
|
|
18
38
|
export type SensitivePath = readonly SensitivePathSegment[];
|
|
19
39
|
export type SensitiveFieldKind = "api_key" | "authorization" | "cookie" | "credential" | "otp" | "password" | "payment_url" | "personal_data" | "phone" | "secret" | "token";
|
|
@@ -32,9 +52,12 @@ export interface SensitiveFieldOptions {
|
|
|
32
52
|
description?: string;
|
|
33
53
|
}
|
|
34
54
|
export declare function describeKey<TSchema extends ZodType>(schema: TSchema, key: ProviderLocaleKey | string): TSchema;
|
|
55
|
+
/** Attach output text-trust authoring metadata without changing validation. */
|
|
56
|
+
export declare function textTrust<TSchema extends ZodType>(schema: TSchema, trust: TextTrust): TSchema;
|
|
35
57
|
declare module "zod" {
|
|
36
58
|
interface ZodType {
|
|
37
59
|
describeKey(key: ProviderLocaleKey | string): this;
|
|
60
|
+
textTrust(trust: TextTrust): this;
|
|
38
61
|
}
|
|
39
62
|
}
|
|
40
63
|
export declare function field<TSchema extends ZodType>(schema: TSchema, options?: SensitiveFieldOptions): TSchema;
|
|
@@ -69,6 +92,46 @@ export declare const fields: {
|
|
|
69
92
|
description?: string;
|
|
70
93
|
}) => ZodString;
|
|
71
94
|
};
|
|
95
|
+
export declare class OutputTextTrustSchemaError extends TypeError {
|
|
96
|
+
readonly code = "invalid_output_text_trust_schema";
|
|
97
|
+
constructor(value: unknown);
|
|
98
|
+
}
|
|
99
|
+
export declare class OutputTextTrustCollectionError extends Error {
|
|
100
|
+
readonly schemaPath: string;
|
|
101
|
+
readonly classification: "untrusted";
|
|
102
|
+
readonly code = "output_text_trust_collection_failed";
|
|
103
|
+
constructor(schemaPath: string, cause: unknown);
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Collect every textual leaf in a Zod output schema as a deterministic
|
|
107
|
+
* schema-path-to-classification object. The path grammar is rooted at `$`:
|
|
108
|
+
* object keys are `["key"]`, arrays and record values are `[*]`, tuple slots
|
|
109
|
+
* are `[n]`, union/intersection alternatives are `<union:n>` / `<intersection:n>`,
|
|
110
|
+
* record keys are `<record-key>`, and one bounded cycle expansion is marked
|
|
111
|
+
* `<recursive>`.
|
|
112
|
+
*/
|
|
113
|
+
export declare function collectOutputTextTrust(schema: ZodType): OutputTextTrustMap;
|
|
114
|
+
/** Report textual output paths that lack a valid explicit or auto-derived classification. */
|
|
115
|
+
export declare function findUnclassifiedOutputTextPaths(schema: ZodType): string[];
|
|
116
|
+
/** @internal Used by contract JSON Schema projection. */
|
|
117
|
+
export declare function resolveOutputTextTrust(schema: z.core.$ZodType, inheritedUntrusted?: boolean): TextTrust | undefined;
|
|
118
|
+
/** @internal Resolve both projection states from one stable schema traversal. */
|
|
119
|
+
export declare function resolveOutputTextTrustProjection(schema: z.core.$ZodType): {
|
|
120
|
+
readonly inherited: TextTrust;
|
|
121
|
+
readonly local: TextTrust;
|
|
122
|
+
} | undefined;
|
|
123
|
+
/** @internal Whether this node can bypass or mutate descendant validation guarantees. */
|
|
124
|
+
export declare function invalidatesDescendantOutputTextAutoTrust(schema: z.core.$ZodType): boolean;
|
|
125
|
+
/** @internal Run output projection without evaluating dynamic fallback callbacks. */
|
|
126
|
+
export declare function suppressDynamicOutputFallbacksDuring<T>(project: () => T): T;
|
|
127
|
+
/** @internal Whether this mutator has a static fallback that cannot contain text. */
|
|
128
|
+
export declare function mutatorReturnIsProvablyNonText(schema: z.core.$ZodType): boolean;
|
|
129
|
+
/** @internal Whether this mutator has a static array fallback. */
|
|
130
|
+
export declare function mutatorUsesStaticArrayFallback(schema: z.core.$ZodType): boolean;
|
|
131
|
+
/** @internal Whether this default or catch was authored with a callback. */
|
|
132
|
+
export declare function hasDynamicOutputFallback(schema: z.core.$ZodType): boolean;
|
|
133
|
+
/** @internal Used by contract JSON Schema projection. */
|
|
134
|
+
export declare function inheritsUntrustedOutputTextTrust(schema: z.core.$ZodType): boolean;
|
|
72
135
|
export declare function isSensitiveSchema(schema: unknown): boolean;
|
|
73
136
|
export declare function collectSensitivePaths(schema: unknown): SensitivePath[];
|
|
74
137
|
export declare function redactPayload(value: unknown, paths?: readonly SensitivePath[]): unknown;
|