@apifuse/provider-sdk 2.1.0-beta.6 → 2.1.0-beta.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/bin/apifuse-perf.ts +18 -9
- package/dist/ceremonies/index.d.ts +41 -0
- package/dist/ceremonies/index.js +490 -0
- package/dist/choice-token.d.ts +24 -0
- package/dist/choice-token.js +74 -0
- package/dist/cli/commands.d.ts +10 -0
- package/dist/cli/commands.js +80 -0
- package/dist/cli/create.d.ts +47 -0
- package/dist/cli/create.js +762 -0
- package/dist/config/loader.d.ts +107 -0
- package/dist/config/loader.js +935 -0
- package/dist/contract-json.d.ts +9 -0
- package/dist/contract-json.js +51 -0
- package/dist/contract-serialization.d.ts +4 -0
- package/dist/contract-serialization.js +78 -0
- package/dist/contract-types.d.ts +49 -0
- package/dist/contract-types.js +1 -0
- package/dist/contract.d.ts +6 -0
- package/dist/contract.js +155 -0
- package/dist/define.d.ts +97 -0
- package/dist/define.js +1320 -0
- package/dist/dev.d.ts +9 -0
- package/dist/dev.js +15 -0
- package/dist/errors.d.ts +59 -0
- package/dist/errors.js +97 -0
- package/dist/i18n/catalog.d.ts +29 -0
- package/dist/i18n/catalog.js +159 -0
- package/dist/i18n/index.d.ts +2 -0
- package/dist/i18n/index.js +2 -0
- package/dist/i18n/keys.d.ts +10 -0
- package/dist/i18n/keys.js +34 -0
- package/dist/index.d.ts +41 -0
- package/dist/index.js +37 -0
- package/dist/lint.d.ts +73 -0
- package/dist/lint.js +702 -0
- package/dist/observability.d.ts +5 -0
- package/dist/observability.js +39 -0
- package/dist/provider.d.ts +9 -0
- package/dist/provider.js +8 -0
- package/dist/public-schema-field-lint.d.ts +2 -0
- package/dist/public-schema-field-lint.js +158 -0
- package/dist/recipes/gov-api.d.ts +19 -0
- package/dist/recipes/gov-api.js +72 -0
- package/dist/recipes/rest-api.d.ts +21 -0
- package/dist/recipes/rest-api.js +115 -0
- package/dist/runtime/auth-flow.d.ts +14 -0
- package/dist/runtime/auth-flow.js +44 -0
- package/dist/runtime/browser.d.ts +25 -0
- package/dist/runtime/browser.js +1034 -0
- package/dist/runtime/cache.d.ts +10 -0
- package/dist/runtime/cache.js +372 -0
- package/dist/runtime/choice.d.ts +15 -0
- package/dist/runtime/choice.js +435 -0
- package/dist/runtime/credential.d.ts +8 -0
- package/dist/runtime/credential.js +61 -0
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +10 -0
- package/dist/runtime/executor.d.ts +16 -0
- package/dist/runtime/executor.js +51 -0
- package/dist/runtime/http.d.ts +8 -0
- package/dist/runtime/http.js +706 -0
- package/dist/runtime/insights.d.ts +9 -0
- package/dist/runtime/insights.js +324 -0
- package/dist/runtime/instrumentation.d.ts +8 -0
- package/dist/runtime/instrumentation.js +269 -0
- package/dist/runtime/key-derivation.d.ts +24 -0
- package/dist/runtime/key-derivation.js +73 -0
- package/dist/runtime/keyring.d.ts +25 -0
- package/dist/runtime/keyring.js +93 -0
- package/dist/runtime/namespace.d.ts +9 -0
- package/dist/runtime/namespace.js +19 -0
- package/dist/runtime/otlp.d.ts +39 -0
- package/dist/runtime/otlp.js +103 -0
- package/dist/runtime/perf.d.ts +12 -0
- package/dist/runtime/perf.js +52 -0
- package/dist/runtime/prevalidate.d.ts +12 -0
- package/dist/runtime/prevalidate.js +173 -0
- package/dist/runtime/provider.d.ts +2 -0
- package/dist/runtime/provider.js +11 -0
- package/dist/runtime/proxy-errors.d.ts +21 -0
- package/dist/runtime/proxy-errors.js +83 -0
- package/dist/runtime/proxy-telemetry.d.ts +8 -0
- package/dist/runtime/proxy-telemetry.js +174 -0
- package/dist/runtime/redis.d.ts +17 -0
- package/dist/runtime/redis.js +82 -0
- package/dist/runtime/request-options.d.ts +3 -0
- package/dist/runtime/request-options.js +42 -0
- package/dist/runtime/state.d.ts +17 -0
- package/dist/runtime/state.js +344 -0
- package/dist/runtime/stealth.d.ts +18 -0
- package/dist/runtime/stealth.js +834 -0
- package/dist/runtime/stt.d.ts +22 -0
- package/dist/runtime/stt.js +480 -0
- package/dist/runtime/trace.d.ts +26 -0
- package/dist/runtime/trace.js +142 -0
- package/dist/runtime/waterfall.d.ts +12 -0
- package/dist/runtime/waterfall.js +147 -0
- package/dist/schema.d.ts +74 -0
- package/dist/schema.js +243 -0
- package/dist/serve.d.ts +1 -0
- package/dist/serve.js +1 -0
- package/dist/server/index.d.ts +3 -0
- package/dist/server/index.js +2 -0
- package/dist/server/serve.d.ts +64 -0
- package/dist/server/serve.js +1110 -0
- package/dist/server/types.d.ts +136 -0
- package/dist/server/types.js +86 -0
- package/dist/stealth/profiles.d.ts +4 -0
- package/dist/stealth/profiles.js +259 -0
- package/dist/stream.d.ts +44 -0
- package/dist/stream.js +151 -0
- package/dist/testing/helpers.d.ts +23 -0
- package/dist/testing/helpers.js +95 -0
- package/dist/testing/index.d.ts +2 -0
- package/dist/testing/index.js +2 -0
- package/dist/testing/run.d.ts +34 -0
- package/dist/testing/run.js +303 -0
- package/dist/types.d.ts +1326 -0
- package/dist/types.js +61 -0
- package/dist/utils/date.d.ts +6 -0
- package/dist/utils/date.js +101 -0
- package/dist/utils/parse.d.ts +16 -0
- package/dist/utils/parse.js +51 -0
- package/dist/utils/text.d.ts +4 -0
- package/dist/utils/text.js +14 -0
- package/dist/utils/transform.d.ts +8 -0
- package/dist/utils/transform.js +48 -0
- package/package.json +109 -107
- package/src/runtime/stealth.ts +8 -1
- package/src/types.ts +2 -0
|
@@ -0,0 +1,834 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { Impit } from "impit";
|
|
3
|
+
import { DEFAULT_SMARTPROXY_POOL_SIZE, invalidateProxyResolutionCacheAsync, ProxyResolutionError, resolveProxyConfigAsync, SMARTPROXY_MAX_POOL_SIZE, } from "../config/loader";
|
|
4
|
+
import { ProviderError, SDKError, TransportError } from "../errors";
|
|
5
|
+
import { getStealthProfile } from "../stealth/profiles";
|
|
6
|
+
import { HttpRetryPreset, HttpRetryUnsafeMethodPolicy } from "../types";
|
|
7
|
+
import { createProxyAuthIpDeniedError, createProxyEdgeAuthRejectedError, createProxyEdgeTlsRejectedError, createProxyPoolExhaustedError, createProxyPoolStaleError, isProxyAuthIpDeniedMessage, isProxyEdgeAuthRejectedMessage, isProxyEdgeTlsRejectedResponse, isProxyPoolRefreshableError, isProxyPoolStaleMessage, isProxyPoolStaleStatus, PROXY_AUTH_IP_DENIED_CODE, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_STALE_CODE, } from "./proxy-errors";
|
|
8
|
+
import { appendQueryParams } from "./request-options";
|
|
9
|
+
const DEFAULT_PROFILE = "chrome-146";
|
|
10
|
+
const MISSING_PROXY_WARNING = "[provider-sdk] Provider requested proxy routing, but no proxy URL was configured. Continuing without proxy.";
|
|
11
|
+
const MAX_POLICY_PROXY_RETRY_ATTEMPTS = SMARTPROXY_MAX_POOL_SIZE;
|
|
12
|
+
const MAX_POLICY_PROXY_POOL_REFRESHES = 1;
|
|
13
|
+
const PROXY_CONNECT_FAILURE_CODE = "proxy_connect_failed";
|
|
14
|
+
const PROXY_CONNECT_FAILURE_BODY_PATTERN = /\bproxy\b.*\b(non[\s-]?200|connect|tunnel)|\bconnect\b.*\bproxy\b|\btunnel\b/i;
|
|
15
|
+
const PROXY_AUTH_DIAGNOSTIC_URL = "http://example.com/";
|
|
16
|
+
const PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS = 5_000;
|
|
17
|
+
const DEFAULT_STEALTH_RETRY_METHODS = ["GET", "HEAD", "OPTIONS"];
|
|
18
|
+
const DEFAULT_STEALTH_RETRY_ERROR_CODES = [
|
|
19
|
+
PROXY_CONNECT_FAILURE_CODE,
|
|
20
|
+
"transport_network_error",
|
|
21
|
+
"transport_timeout",
|
|
22
|
+
];
|
|
23
|
+
const RATE_LIMIT_STEALTH_RETRY_ERROR_CODES = ["transport_timeout"];
|
|
24
|
+
const KNOWN_STEALTH_RETRY_METHODS = new Set([
|
|
25
|
+
"GET",
|
|
26
|
+
"HEAD",
|
|
27
|
+
"POST",
|
|
28
|
+
"PUT",
|
|
29
|
+
"DELETE",
|
|
30
|
+
"OPTIONS",
|
|
31
|
+
"TRACE",
|
|
32
|
+
"PATCH",
|
|
33
|
+
]);
|
|
34
|
+
const UNSAFE_STEALTH_RETRY_METHODS = new Set([
|
|
35
|
+
"POST",
|
|
36
|
+
"PUT",
|
|
37
|
+
"PATCH",
|
|
38
|
+
"DELETE",
|
|
39
|
+
"TRACE",
|
|
40
|
+
]);
|
|
41
|
+
const MAX_STEALTH_RETRY_ATTEMPTS = 8;
|
|
42
|
+
const REMOVED_CHROME_PROFILE_NAMES = new Set([
|
|
43
|
+
"chrome-120",
|
|
44
|
+
"chrome-124",
|
|
45
|
+
"chrome-129",
|
|
46
|
+
"chrome-130",
|
|
47
|
+
"chrome-131",
|
|
48
|
+
"chrome-133",
|
|
49
|
+
"chrome-144",
|
|
50
|
+
"chrome-146-psk",
|
|
51
|
+
"chrome-131-psk",
|
|
52
|
+
"chrome-130-psk",
|
|
53
|
+
"edge-131",
|
|
54
|
+
]);
|
|
55
|
+
const CHROME_IMPIT_BY_MAJOR = {
|
|
56
|
+
100: "chrome100",
|
|
57
|
+
101: "chrome101",
|
|
58
|
+
104: "chrome104",
|
|
59
|
+
107: "chrome107",
|
|
60
|
+
110: "chrome110",
|
|
61
|
+
116: "chrome116",
|
|
62
|
+
124: "chrome124",
|
|
63
|
+
125: "chrome125",
|
|
64
|
+
131: "chrome131",
|
|
65
|
+
136: "chrome136",
|
|
66
|
+
142: "chrome142",
|
|
67
|
+
};
|
|
68
|
+
const FIREFOX_IMPIT_BY_MAJOR = {
|
|
69
|
+
128: "firefox128",
|
|
70
|
+
133: "firefox133",
|
|
71
|
+
135: "firefox135",
|
|
72
|
+
144: "firefox144",
|
|
73
|
+
};
|
|
74
|
+
function isRecord(value) {
|
|
75
|
+
return typeof value === "object" && value !== null;
|
|
76
|
+
}
|
|
77
|
+
class CookieJarImpl {
|
|
78
|
+
cookies;
|
|
79
|
+
constructor(cookieStrings) {
|
|
80
|
+
this.cookies = {};
|
|
81
|
+
this.setFromCookieStrings(cookieStrings);
|
|
82
|
+
}
|
|
83
|
+
setFromCookieStrings(cookieStrings) {
|
|
84
|
+
for (const cookieString of cookieStrings) {
|
|
85
|
+
const [nameValue] = cookieString.split(";");
|
|
86
|
+
if (!nameValue) {
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
const separatorIndex = nameValue.indexOf("=");
|
|
90
|
+
if (separatorIndex === -1) {
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
const name = nameValue.slice(0, separatorIndex).trim();
|
|
94
|
+
const value = nameValue.slice(separatorIndex + 1).trim();
|
|
95
|
+
if (name)
|
|
96
|
+
this.cookies[name] = value;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
get(name) {
|
|
100
|
+
return this.cookies[name];
|
|
101
|
+
}
|
|
102
|
+
getAll() {
|
|
103
|
+
return { ...this.cookies };
|
|
104
|
+
}
|
|
105
|
+
toString() {
|
|
106
|
+
return Object.entries(this.cookies)
|
|
107
|
+
.map(([name, value]) => `${name}=${value}`)
|
|
108
|
+
.join("; ");
|
|
109
|
+
}
|
|
110
|
+
find(predicate) {
|
|
111
|
+
for (const [name, value] of Object.entries(this.cookies)) {
|
|
112
|
+
const cookie = `${name}=${value}`;
|
|
113
|
+
if (predicate(cookie)) {
|
|
114
|
+
return cookie;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return undefined;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
function closestImpitBrowser(major, candidates) {
|
|
121
|
+
let closestMajor;
|
|
122
|
+
let closestBrowser;
|
|
123
|
+
for (const [candidateMajorText, browser] of Object.entries(candidates)) {
|
|
124
|
+
const candidateMajor = Number(candidateMajorText);
|
|
125
|
+
if (closestMajor === undefined ||
|
|
126
|
+
Math.abs(candidateMajor - major) < Math.abs(closestMajor - major)) {
|
|
127
|
+
closestMajor = candidateMajor;
|
|
128
|
+
closestBrowser = browser;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return closestBrowser ?? "chrome142";
|
|
132
|
+
}
|
|
133
|
+
function resolveImpitBrowser(profileName) {
|
|
134
|
+
if (REMOVED_CHROME_PROFILE_NAMES.has(profileName)) {
|
|
135
|
+
throw new SDKError(`Unknown stealth profile: ${profileName}`);
|
|
136
|
+
}
|
|
137
|
+
let profile;
|
|
138
|
+
try {
|
|
139
|
+
profile = getStealthProfile(profileName);
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
// Preserve the previous ctx.stealth.fetch() compatibility behavior: unknown
|
|
143
|
+
// profile strings still run with the transport default instead of failing
|
|
144
|
+
// before the request starts. Removed built-in profile aliases above remain
|
|
145
|
+
// explicit errors so callers do not accidentally pin retired fingerprints.
|
|
146
|
+
return "chrome142";
|
|
147
|
+
}
|
|
148
|
+
const identifier = profile.tlsClientIdentifier?.toLowerCase() ?? "";
|
|
149
|
+
const chromeMatch = /^(?:chrome|edge)_(\d+)/.exec(identifier);
|
|
150
|
+
if (chromeMatch?.[1]) {
|
|
151
|
+
return closestImpitBrowser(Number(chromeMatch[1]), CHROME_IMPIT_BY_MAJOR);
|
|
152
|
+
}
|
|
153
|
+
const firefoxMatch = /^firefox_(\d+)/.exec(identifier);
|
|
154
|
+
if (firefoxMatch?.[1]) {
|
|
155
|
+
return closestImpitBrowser(Number(firefoxMatch[1]), FIREFOX_IMPIT_BY_MAJOR);
|
|
156
|
+
}
|
|
157
|
+
if (identifier.startsWith("safari_")) {
|
|
158
|
+
throw new SDKError(`Stealth profile "${profileName}" uses a Safari stealth fingerprint, but TypeScript ctx.stealth uses impit which currently supports Chrome, Firefox, and OkHttp profiles only. Use a Chrome/Firefox stealth profile for ctx.stealth or ctx.browser for Safari-specific behavior.`);
|
|
159
|
+
}
|
|
160
|
+
throw new SDKError(`Stealth profile "${profileName}" cannot be mapped to an impit browser profile.`);
|
|
161
|
+
}
|
|
162
|
+
function resolveUrl(baseUrl, url) {
|
|
163
|
+
return new URL(url, baseUrl).toString();
|
|
164
|
+
}
|
|
165
|
+
function headerEntriesFromHeaders(headers) {
|
|
166
|
+
return Array.from(headers.entries());
|
|
167
|
+
}
|
|
168
|
+
function normalizeHeaders(headers) {
|
|
169
|
+
const normalized = {};
|
|
170
|
+
for (const [name, value] of Object.entries(headers)) {
|
|
171
|
+
if (value === undefined)
|
|
172
|
+
continue;
|
|
173
|
+
normalized[name] = Array.isArray(value) ? value.join(", ") : value;
|
|
174
|
+
}
|
|
175
|
+
return normalized;
|
|
176
|
+
}
|
|
177
|
+
function hasOwn(object, key) {
|
|
178
|
+
return Object.hasOwn(object, key);
|
|
179
|
+
}
|
|
180
|
+
function toImpitCookieJar(cookieJar) {
|
|
181
|
+
return {
|
|
182
|
+
setCookie(cookie, _url, cb) {
|
|
183
|
+
cookieJar.setFromCookieStrings([cookie]);
|
|
184
|
+
if (typeof cb === "function")
|
|
185
|
+
cb();
|
|
186
|
+
},
|
|
187
|
+
getCookieString(_url) {
|
|
188
|
+
return cookieJar.toString();
|
|
189
|
+
},
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
function assertNoUnsupportedFingerprintOverrides(options) {
|
|
193
|
+
if (!isRecord(options))
|
|
194
|
+
return;
|
|
195
|
+
const unsupported = [];
|
|
196
|
+
if (hasOwn(options, "headerOrder"))
|
|
197
|
+
unsupported.push("headerOrder");
|
|
198
|
+
const stealth = options.stealth;
|
|
199
|
+
if (isRecord(stealth) && hasOwn(stealth, "ja3"))
|
|
200
|
+
unsupported.push("stealth.ja3");
|
|
201
|
+
if (isRecord(stealth) && hasOwn(stealth, "h2"))
|
|
202
|
+
unsupported.push("stealth.h2");
|
|
203
|
+
if (unsupported.length === 0)
|
|
204
|
+
return;
|
|
205
|
+
throw new SDKError(`ctx.stealth.fetch uses impit-managed browser fingerprints and no longer accepts low-level stealth overrides: ${unsupported.join(", ")}. Use the profile option instead.`);
|
|
206
|
+
}
|
|
207
|
+
function responseHeadersToRecord(headers) {
|
|
208
|
+
const record = {};
|
|
209
|
+
for (const [name, value] of headers.entries())
|
|
210
|
+
record[name] = value;
|
|
211
|
+
return record;
|
|
212
|
+
}
|
|
213
|
+
function setCookieHeadersFromResponse(headers) {
|
|
214
|
+
const getSetCookie = headers.getSetCookie;
|
|
215
|
+
if (typeof getSetCookie === "function")
|
|
216
|
+
return getSetCookie.call(headers);
|
|
217
|
+
const setCookie = headers.get("set-cookie");
|
|
218
|
+
return setCookie ? splitCombinedSetCookieHeader(setCookie) : [];
|
|
219
|
+
}
|
|
220
|
+
function splitCombinedSetCookieHeader(headerValue) {
|
|
221
|
+
const cookieNamePattern = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+=/;
|
|
222
|
+
const cookieStrings = [];
|
|
223
|
+
let start = 0;
|
|
224
|
+
for (let index = 0; index < headerValue.length; index += 1) {
|
|
225
|
+
if (headerValue[index] !== ",")
|
|
226
|
+
continue;
|
|
227
|
+
const next = headerValue.slice(index + 1).trimStart();
|
|
228
|
+
if (!cookieNamePattern.test(next))
|
|
229
|
+
continue;
|
|
230
|
+
const cookie = headerValue.slice(start, index).trim();
|
|
231
|
+
if (cookie)
|
|
232
|
+
cookieStrings.push(cookie);
|
|
233
|
+
start = index + 1;
|
|
234
|
+
}
|
|
235
|
+
const finalCookie = headerValue.slice(start).trim();
|
|
236
|
+
if (finalCookie)
|
|
237
|
+
cookieStrings.push(finalCookie);
|
|
238
|
+
return cookieStrings;
|
|
239
|
+
}
|
|
240
|
+
export async function normalizeResponse(response) {
|
|
241
|
+
const headers = Object.fromEntries(response.headers.entries());
|
|
242
|
+
const cookies = new CookieJarImpl(setCookieHeadersFromResponse(response.headers));
|
|
243
|
+
const bodyBytes = await response.arrayBuffer();
|
|
244
|
+
const body = new TextDecoder().decode(bodyBytes);
|
|
245
|
+
return {
|
|
246
|
+
status: response.status,
|
|
247
|
+
ok: response.status >= 200 && response.status < 300,
|
|
248
|
+
headers,
|
|
249
|
+
rawHeaders: headerEntriesFromHeaders(response.headers),
|
|
250
|
+
body,
|
|
251
|
+
cookies,
|
|
252
|
+
json() {
|
|
253
|
+
return Promise.resolve(JSON.parse(body));
|
|
254
|
+
},
|
|
255
|
+
arrayBuffer() {
|
|
256
|
+
return Promise.resolve(bodyBytes.slice(0));
|
|
257
|
+
},
|
|
258
|
+
bytes() {
|
|
259
|
+
return Promise.resolve(new Uint8Array(bodyBytes.slice(0)));
|
|
260
|
+
},
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
function normalizeBody(body) {
|
|
264
|
+
if (body === undefined) {
|
|
265
|
+
return "";
|
|
266
|
+
}
|
|
267
|
+
if (typeof body === "string") {
|
|
268
|
+
return body;
|
|
269
|
+
}
|
|
270
|
+
if (Buffer.isBuffer(body)) {
|
|
271
|
+
return body.toString();
|
|
272
|
+
}
|
|
273
|
+
return String(body);
|
|
274
|
+
}
|
|
275
|
+
function isPolicyManagedProxy(options) {
|
|
276
|
+
const policy = options.proxyPolicy ?? options.upstream?.proxy;
|
|
277
|
+
return Boolean(policy && typeof policy === "object");
|
|
278
|
+
}
|
|
279
|
+
function isRetrySafeStealthMethod(method) {
|
|
280
|
+
return method === "GET" || method === "HEAD" || method === "OPTIONS";
|
|
281
|
+
}
|
|
282
|
+
function createStealthRetryOptions(preset) {
|
|
283
|
+
switch (preset) {
|
|
284
|
+
case HttpRetryPreset.Off:
|
|
285
|
+
return {
|
|
286
|
+
attempts: 1,
|
|
287
|
+
methods: DEFAULT_STEALTH_RETRY_METHODS,
|
|
288
|
+
errorCodes: DEFAULT_STEALTH_RETRY_ERROR_CODES,
|
|
289
|
+
unsafeMethodPolicy: HttpRetryUnsafeMethodPolicy.Reject,
|
|
290
|
+
};
|
|
291
|
+
case HttpRetryPreset.AggressiveRead:
|
|
292
|
+
return {
|
|
293
|
+
attempts: 4,
|
|
294
|
+
methods: DEFAULT_STEALTH_RETRY_METHODS,
|
|
295
|
+
errorCodes: DEFAULT_STEALTH_RETRY_ERROR_CODES,
|
|
296
|
+
unsafeMethodPolicy: HttpRetryUnsafeMethodPolicy.Reject,
|
|
297
|
+
};
|
|
298
|
+
case HttpRetryPreset.RateLimitAware:
|
|
299
|
+
return {
|
|
300
|
+
attempts: 3,
|
|
301
|
+
methods: DEFAULT_STEALTH_RETRY_METHODS,
|
|
302
|
+
errorCodes: RATE_LIMIT_STEALTH_RETRY_ERROR_CODES,
|
|
303
|
+
unsafeMethodPolicy: HttpRetryUnsafeMethodPolicy.Reject,
|
|
304
|
+
};
|
|
305
|
+
case HttpRetryPreset.SafeRead:
|
|
306
|
+
case HttpRetryPreset.TransportTransient:
|
|
307
|
+
return {
|
|
308
|
+
attempts: 3,
|
|
309
|
+
methods: DEFAULT_STEALTH_RETRY_METHODS,
|
|
310
|
+
errorCodes: DEFAULT_STEALTH_RETRY_ERROR_CODES,
|
|
311
|
+
unsafeMethodPolicy: HttpRetryUnsafeMethodPolicy.Reject,
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
throw new ProviderError(`Unknown stealth retry preset: ${preset}`, {
|
|
315
|
+
code: "retry_invalid_policy",
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
function normalizeStealthRetryOptions(retry) {
|
|
319
|
+
if (retry === undefined)
|
|
320
|
+
return undefined;
|
|
321
|
+
if (retry === false)
|
|
322
|
+
return createStealthRetryOptions(HttpRetryPreset.Off);
|
|
323
|
+
if (retry === true)
|
|
324
|
+
return createStealthRetryOptions(HttpRetryPreset.TransportTransient);
|
|
325
|
+
if (typeof retry === "string") {
|
|
326
|
+
if (!Object.values(HttpRetryPreset).includes(retry)) {
|
|
327
|
+
throw new ProviderError(`Unknown stealth retry preset: ${retry}`, {
|
|
328
|
+
code: "retry_invalid_policy",
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
return createStealthRetryOptions(retry);
|
|
332
|
+
}
|
|
333
|
+
if (typeof retry !== "object" || retry === null || Array.isArray(retry)) {
|
|
334
|
+
throw new ProviderError("Stealth retry policy must be a plain object", {
|
|
335
|
+
code: "retry_invalid_policy",
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
if (retry.unsafeMethodPolicy !== undefined &&
|
|
339
|
+
!Object.values(HttpRetryUnsafeMethodPolicy).includes(retry.unsafeMethodPolicy)) {
|
|
340
|
+
throw new ProviderError(`Unknown stealth retry unsafe method policy: ${String(retry.unsafeMethodPolicy)}`, { code: "retry_invalid_policy" });
|
|
341
|
+
}
|
|
342
|
+
if (retry.methods !== undefined) {
|
|
343
|
+
if (!Array.isArray(retry.methods)) {
|
|
344
|
+
throw new ProviderError("Stealth retry methods must be an array", {
|
|
345
|
+
code: "retry_invalid_policy",
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
const unknownMethods = retry.methods
|
|
349
|
+
.map((method) => (typeof method === "string" ? method.toUpperCase() : ""))
|
|
350
|
+
.filter((method) => !KNOWN_STEALTH_RETRY_METHODS.has(method));
|
|
351
|
+
if (unknownMethods.length > 0) {
|
|
352
|
+
throw new ProviderError(`Unknown stealth retry method(s): ${unknownMethods.join(", ")}`, { code: "retry_invalid_policy" });
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
if (retry.errorCodes !== undefined) {
|
|
356
|
+
if (!Array.isArray(retry.errorCodes) ||
|
|
357
|
+
retry.errorCodes.some((errorCode) => typeof errorCode !== "string")) {
|
|
358
|
+
throw new ProviderError("Stealth retry errorCodes must contain only strings", { code: "retry_invalid_policy" });
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
const base = createStealthRetryOptions(retry.preset ?? HttpRetryPreset.TransportTransient);
|
|
362
|
+
const attempts = retry.attempts === undefined || !Number.isFinite(retry.attempts)
|
|
363
|
+
? base.attempts
|
|
364
|
+
: Math.max(1, Math.min(MAX_STEALTH_RETRY_ATTEMPTS, Math.floor(retry.attempts)));
|
|
365
|
+
const normalized = {
|
|
366
|
+
attempts,
|
|
367
|
+
methods: retry.methods?.map((method) => method.toUpperCase()) ?? base.methods,
|
|
368
|
+
errorCodes: retry.errorCodes ?? base.errorCodes,
|
|
369
|
+
unsafeMethodPolicy: retry.unsafeMethodPolicy ?? base.unsafeMethodPolicy,
|
|
370
|
+
};
|
|
371
|
+
if (normalized.unsafeMethodPolicy !==
|
|
372
|
+
HttpRetryUnsafeMethodPolicy.AllowExplicitUnsafe) {
|
|
373
|
+
const unsafeMethods = normalized.methods.filter((method) => UNSAFE_STEALTH_RETRY_METHODS.has(method.toUpperCase()));
|
|
374
|
+
if (unsafeMethods.length > 0) {
|
|
375
|
+
throw new ProviderError(`Stealth retry methods include unsafe method(s): ${unsafeMethods.join(", ")}`, { code: "retry_unsafe_method" });
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
return normalized;
|
|
379
|
+
}
|
|
380
|
+
function isExplicitStealthRetryAllowed(method, error, retryOptions) {
|
|
381
|
+
if (!retryOptions || retryOptions.attempts <= 1)
|
|
382
|
+
return false;
|
|
383
|
+
return (retryOptions.methods.includes(method.toUpperCase()) &&
|
|
384
|
+
retryOptions.errorCodes.includes(proxyAttemptErrorCode(error)));
|
|
385
|
+
}
|
|
386
|
+
function isRetryableProxyTransportError(error) {
|
|
387
|
+
if (error instanceof TransportError) {
|
|
388
|
+
if (error.code === PROXY_AUTH_IP_DENIED_CODE) {
|
|
389
|
+
return false;
|
|
390
|
+
}
|
|
391
|
+
return (error.code === PROXY_CONNECT_FAILURE_CODE ||
|
|
392
|
+
error.code === "transport_network_error" ||
|
|
393
|
+
error.code === "transport_timeout");
|
|
394
|
+
}
|
|
395
|
+
if (error instanceof SDKError) {
|
|
396
|
+
return false;
|
|
397
|
+
}
|
|
398
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
399
|
+
return /\bproxy\b|\bnon[\s-]?200\b|\bconnect\b|\btunnel\b/i.test(message);
|
|
400
|
+
}
|
|
401
|
+
function isProxyConnectFailureResponse(response, body) {
|
|
402
|
+
return (response.status === 0 && PROXY_CONNECT_FAILURE_BODY_PATTERN.test(body ?? ""));
|
|
403
|
+
}
|
|
404
|
+
function createProxyConnectFailureError(body, cause) {
|
|
405
|
+
const bodyExcerpt = (body ?? "").trim().slice(0, 1_000);
|
|
406
|
+
if (isProxyAuthIpDeniedMessage(bodyExcerpt)) {
|
|
407
|
+
return createProxyAuthIpDeniedError(cause);
|
|
408
|
+
}
|
|
409
|
+
if (isProxyEdgeAuthRejectedMessage(bodyExcerpt)) {
|
|
410
|
+
return createProxyEdgeAuthRejectedError(cause);
|
|
411
|
+
}
|
|
412
|
+
if (isProxyPoolStaleMessage(bodyExcerpt)) {
|
|
413
|
+
return createProxyPoolStaleError(bodyExcerpt.includes("512") ? 512 : 509, cause);
|
|
414
|
+
}
|
|
415
|
+
return new TransportError(bodyExcerpt || "Proxy CONNECT failed", {
|
|
416
|
+
code: PROXY_CONNECT_FAILURE_CODE,
|
|
417
|
+
status: 0,
|
|
418
|
+
cause,
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
function shouldRunProxyAuthDiagnostic(error) {
|
|
422
|
+
if (!(error instanceof TransportError)) {
|
|
423
|
+
return false;
|
|
424
|
+
}
|
|
425
|
+
if (error.code !== PROXY_POOL_STALE_CODE || error.status !== 512) {
|
|
426
|
+
return false;
|
|
427
|
+
}
|
|
428
|
+
return error.cause instanceof Error;
|
|
429
|
+
}
|
|
430
|
+
function proxyPoolIndexFromDiagnostics(diagnostics) {
|
|
431
|
+
const value = diagnostics?.poolIndex;
|
|
432
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
433
|
+
return undefined;
|
|
434
|
+
}
|
|
435
|
+
return Math.floor(value);
|
|
436
|
+
}
|
|
437
|
+
function proxyEndpointHash(proxyUrl) {
|
|
438
|
+
if (!proxyUrl)
|
|
439
|
+
return undefined;
|
|
440
|
+
try {
|
|
441
|
+
const parsed = new URL(proxyUrl);
|
|
442
|
+
return createHash("sha256")
|
|
443
|
+
.update(`${parsed.protocol}//${parsed.host}`)
|
|
444
|
+
.digest("hex")
|
|
445
|
+
.slice(0, 12);
|
|
446
|
+
}
|
|
447
|
+
catch {
|
|
448
|
+
return createHash("sha256").update(proxyUrl).digest("hex").slice(0, 12);
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
function getProxyTunnelStatus(error) {
|
|
452
|
+
if (isRecord(error)) {
|
|
453
|
+
const status = error.status;
|
|
454
|
+
if (typeof status === "number" && Number.isFinite(status)) {
|
|
455
|
+
return status;
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
const cause = error instanceof Error ? error.cause : undefined;
|
|
459
|
+
if (cause && cause !== error) {
|
|
460
|
+
return getProxyTunnelStatus(cause);
|
|
461
|
+
}
|
|
462
|
+
return undefined;
|
|
463
|
+
}
|
|
464
|
+
function isTimeoutError(error, message) {
|
|
465
|
+
if (error instanceof Error) {
|
|
466
|
+
if (error.name === "AbortError" || error.name === "TimeoutError") {
|
|
467
|
+
return true;
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
return /\b(timed out|timeout|deadline exceeded)\b/i.test(message);
|
|
471
|
+
}
|
|
472
|
+
function normalizeStealthTransportError(error) {
|
|
473
|
+
if (error instanceof ProxyResolutionError) {
|
|
474
|
+
return new TransportError(error.message, {
|
|
475
|
+
code: error.code,
|
|
476
|
+
status: 0,
|
|
477
|
+
cause: error,
|
|
478
|
+
});
|
|
479
|
+
}
|
|
480
|
+
if (error instanceof TransportError) {
|
|
481
|
+
return error;
|
|
482
|
+
}
|
|
483
|
+
if (error instanceof SDKError) {
|
|
484
|
+
throw error;
|
|
485
|
+
}
|
|
486
|
+
const message = error instanceof Error
|
|
487
|
+
? [error.message, error.cause instanceof Error ? error.cause.message : ""]
|
|
488
|
+
.filter(Boolean)
|
|
489
|
+
.join(" ")
|
|
490
|
+
: String(error);
|
|
491
|
+
if (isTimeoutError(error, message)) {
|
|
492
|
+
return new TransportError("Request timed out", {
|
|
493
|
+
code: "transport_timeout",
|
|
494
|
+
status: 0,
|
|
495
|
+
cause: error instanceof Error ? error : undefined,
|
|
496
|
+
});
|
|
497
|
+
}
|
|
498
|
+
if (isProxyAuthIpDeniedMessage(message)) {
|
|
499
|
+
return createProxyAuthIpDeniedError(error instanceof Error ? error : undefined);
|
|
500
|
+
}
|
|
501
|
+
if (isProxyEdgeAuthRejectedMessage(message)) {
|
|
502
|
+
return createProxyEdgeAuthRejectedError(error instanceof Error ? error : undefined);
|
|
503
|
+
}
|
|
504
|
+
const proxyTunnelStatus = getProxyTunnelStatus(error);
|
|
505
|
+
if (proxyTunnelStatus !== undefined &&
|
|
506
|
+
isProxyPoolStaleStatus(proxyTunnelStatus)) {
|
|
507
|
+
return createProxyPoolStaleError(proxyTunnelStatus, error instanceof Error ? error : undefined);
|
|
508
|
+
}
|
|
509
|
+
if (PROXY_CONNECT_FAILURE_BODY_PATTERN.test(message)) {
|
|
510
|
+
return createProxyConnectFailureError(message, error instanceof Error ? error : undefined);
|
|
511
|
+
}
|
|
512
|
+
return new TransportError("Network error", {
|
|
513
|
+
code: "transport_network_error",
|
|
514
|
+
status: 0,
|
|
515
|
+
cause: error instanceof Error ? error : undefined,
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
function normalizeMethod(method) {
|
|
519
|
+
switch (method.toUpperCase()) {
|
|
520
|
+
case "HEAD":
|
|
521
|
+
return "HEAD";
|
|
522
|
+
case "GET":
|
|
523
|
+
return "GET";
|
|
524
|
+
case "POST":
|
|
525
|
+
return "POST";
|
|
526
|
+
case "PUT":
|
|
527
|
+
return "PUT";
|
|
528
|
+
case "DELETE":
|
|
529
|
+
return "DELETE";
|
|
530
|
+
case "OPTIONS":
|
|
531
|
+
return "OPTIONS";
|
|
532
|
+
case "TRACE":
|
|
533
|
+
return "TRACE";
|
|
534
|
+
case "PATCH":
|
|
535
|
+
return "PATCH";
|
|
536
|
+
default:
|
|
537
|
+
throw new SDKError(`Unsupported stealth method: ${method}`);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
541
|
+
const clients = new Map();
|
|
542
|
+
let closed = false;
|
|
543
|
+
let hasWarnedMissingProxy = false;
|
|
544
|
+
const warn = clientOptions.warn ?? console.warn;
|
|
545
|
+
const cookieJar = new CookieJarImpl([]);
|
|
546
|
+
const impitCookieJar = toImpitCookieJar(cookieJar);
|
|
547
|
+
function getClient(profileName, proxyUrl, ignoreTlsErrors) {
|
|
548
|
+
if (closed) {
|
|
549
|
+
throw new TransportError("Stealth session is closed", { status: 0 });
|
|
550
|
+
}
|
|
551
|
+
const browser = resolveImpitBrowser(profileName);
|
|
552
|
+
const cacheKey = JSON.stringify({ browser, proxyUrl, ignoreTlsErrors });
|
|
553
|
+
let client = clients.get(cacheKey);
|
|
554
|
+
if (!client) {
|
|
555
|
+
client = new Impit({
|
|
556
|
+
browser,
|
|
557
|
+
cookieJar: impitCookieJar,
|
|
558
|
+
...(proxyUrl ? { proxyUrl } : {}),
|
|
559
|
+
...(ignoreTlsErrors ? { ignoreTlsErrors: true } : {}),
|
|
560
|
+
timeout: 30_000,
|
|
561
|
+
});
|
|
562
|
+
clients.set(cacheKey, client);
|
|
563
|
+
}
|
|
564
|
+
return client;
|
|
565
|
+
}
|
|
566
|
+
async function resolveRequestProxy(options, proxyAttempt) {
|
|
567
|
+
const rawProxyAttemptOffset = options?.proxyAttemptOffset ?? 0;
|
|
568
|
+
const proxyAttemptOffset = Number.isFinite(rawProxyAttemptOffset)
|
|
569
|
+
? Math.max(0, Math.floor(rawProxyAttemptOffset))
|
|
570
|
+
: 0;
|
|
571
|
+
const resolvedProxy = await resolveProxyConfigAsync({
|
|
572
|
+
proxy: options?.proxy ?? clientOptions.proxy,
|
|
573
|
+
upstream: clientOptions.upstream,
|
|
574
|
+
apifuseConfig: clientOptions.apifuseConfig,
|
|
575
|
+
affinityKey: clientOptions.affinityKey,
|
|
576
|
+
proxyAttempt: proxyAttempt === undefined
|
|
577
|
+
? proxyAttemptOffset
|
|
578
|
+
: proxyAttemptOffset + proxyAttempt,
|
|
579
|
+
telemetry: clientOptions.telemetry,
|
|
580
|
+
});
|
|
581
|
+
if (resolvedProxy.shouldWarn && !hasWarnedMissingProxy) {
|
|
582
|
+
hasWarnedMissingProxy = true;
|
|
583
|
+
warn(MISSING_PROXY_WARNING);
|
|
584
|
+
}
|
|
585
|
+
return {
|
|
586
|
+
url: resolvedProxy.url,
|
|
587
|
+
poolIndex: proxyPoolIndexFromDiagnostics(resolvedProxy.diagnostics),
|
|
588
|
+
proxyHash: proxyEndpointHash(resolvedProxy.url),
|
|
589
|
+
};
|
|
590
|
+
}
|
|
591
|
+
return {
|
|
592
|
+
async fetch(url, options = {}) {
|
|
593
|
+
const method = normalizeMethod(options.method ?? "GET");
|
|
594
|
+
const hasExplicitRetryPolicy = options.retry !== undefined;
|
|
595
|
+
const stealthRetryOptions = normalizeStealthRetryOptions(options.retry);
|
|
596
|
+
const hasPolicyProxy = isPolicyManagedProxy(clientOptions);
|
|
597
|
+
const usesPolicyAllocator = hasPolicyProxy && !options.proxy && !clientOptions.proxy;
|
|
598
|
+
const maxAttempts = usesPolicyAllocator
|
|
599
|
+
? Math.max(1, Math.min(MAX_POLICY_PROXY_RETRY_ATTEMPTS, clientOptions.proxyPolicy?.session?.poolSize ??
|
|
600
|
+
(typeof clientOptions.upstream?.proxy === "object"
|
|
601
|
+
? clientOptions.upstream.proxy.session?.poolSize
|
|
602
|
+
: undefined) ??
|
|
603
|
+
DEFAULT_SMARTPROXY_POOL_SIZE))
|
|
604
|
+
: 1;
|
|
605
|
+
let lastError;
|
|
606
|
+
for (let refreshAttempt = 0; refreshAttempt <= MAX_POLICY_PROXY_POOL_REFRESHES; refreshAttempt += 1) {
|
|
607
|
+
let stalePoolError;
|
|
608
|
+
let stalePoolDiagnosticProxy;
|
|
609
|
+
const attemptedProxies = new Set();
|
|
610
|
+
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
611
|
+
let proxy;
|
|
612
|
+
let attemptProxy;
|
|
613
|
+
const attemptStartedAt = Date.now();
|
|
614
|
+
let attemptRecorded = false;
|
|
615
|
+
const recordProxyAttempt = (outcome, errorCode, status) => {
|
|
616
|
+
if (attemptRecorded || !proxy)
|
|
617
|
+
return;
|
|
618
|
+
attemptRecorded = true;
|
|
619
|
+
clientOptions.telemetry?.recordProxyAttempt?.({
|
|
620
|
+
provider: "smartproxy",
|
|
621
|
+
attempt: attempt + 1,
|
|
622
|
+
...(attemptProxy?.poolIndex === undefined
|
|
623
|
+
? {}
|
|
624
|
+
: { poolIndex: attemptProxy.poolIndex }),
|
|
625
|
+
...(attemptProxy?.proxyHash
|
|
626
|
+
? { proxyHash: attemptProxy.proxyHash }
|
|
627
|
+
: {}),
|
|
628
|
+
outcome,
|
|
629
|
+
...(errorCode ? { errorCode } : {}),
|
|
630
|
+
...(status === undefined ? {} : { status }),
|
|
631
|
+
durationMs: Date.now() - attemptStartedAt,
|
|
632
|
+
});
|
|
633
|
+
};
|
|
634
|
+
try {
|
|
635
|
+
assertNoUnsupportedFingerprintOverrides(options);
|
|
636
|
+
attemptProxy = await resolveRequestProxy(options, attempt);
|
|
637
|
+
proxy = attemptProxy.url;
|
|
638
|
+
if (proxy) {
|
|
639
|
+
if (attemptedProxies.has(proxy)) {
|
|
640
|
+
break;
|
|
641
|
+
}
|
|
642
|
+
attemptedProxies.add(proxy);
|
|
643
|
+
}
|
|
644
|
+
const ignoreTlsErrors = Boolean(options.stealth?.insecureSkipVerify ??
|
|
645
|
+
(!hasPolicyProxy &&
|
|
646
|
+
proxy &&
|
|
647
|
+
clientOptions.proxyStealth?.insecureSkipVerify));
|
|
648
|
+
const profileName = options.profile ?? defaultProfile;
|
|
649
|
+
const requestUrl = appendQueryParams(resolveUrl(baseUrl, url), options.params);
|
|
650
|
+
const headers = { ...(options.headers ?? {}) };
|
|
651
|
+
if (!hasHeader(headers, "Cookie")) {
|
|
652
|
+
const cookieHeader = cookieJar.toString();
|
|
653
|
+
if (cookieHeader)
|
|
654
|
+
headers.Cookie = cookieHeader;
|
|
655
|
+
}
|
|
656
|
+
const requestInit = {
|
|
657
|
+
headers: normalizeHeaders(headers),
|
|
658
|
+
method,
|
|
659
|
+
...(options.timeout ? { timeout: options.timeout } : {}),
|
|
660
|
+
};
|
|
661
|
+
if (options.body !== undefined) {
|
|
662
|
+
requestInit.body = normalizeBody(options.body);
|
|
663
|
+
}
|
|
664
|
+
const response = await getClient(profileName, proxy, ignoreTlsErrors).fetch(requestUrl, requestInit);
|
|
665
|
+
const normalized = await normalizeResponse(response);
|
|
666
|
+
cookieJar.setFromCookieStrings(setCookieHeadersFromResponse(response.headers));
|
|
667
|
+
if (proxy &&
|
|
668
|
+
isProxyConnectFailureResponse(response, normalized.body)) {
|
|
669
|
+
throw createProxyConnectFailureError(normalized.body);
|
|
670
|
+
}
|
|
671
|
+
if (response.status >= 400) {
|
|
672
|
+
if (proxy &&
|
|
673
|
+
usesPolicyAllocator &&
|
|
674
|
+
isProxyEdgeTlsRejectedResponse(response.status, [
|
|
675
|
+
JSON.stringify(responseHeadersToRecord(response.headers)),
|
|
676
|
+
normalized.body,
|
|
677
|
+
].join("\n"))) {
|
|
678
|
+
throw createProxyEdgeTlsRejectedError(response.status);
|
|
679
|
+
}
|
|
680
|
+
if (proxy && isProxyAuthIpDeniedMessage(normalized.body)) {
|
|
681
|
+
throw createProxyAuthIpDeniedError();
|
|
682
|
+
}
|
|
683
|
+
if (proxy && isProxyEdgeAuthRejectedMessage(normalized.body)) {
|
|
684
|
+
throw createProxyEdgeAuthRejectedError();
|
|
685
|
+
}
|
|
686
|
+
if (proxy &&
|
|
687
|
+
isProxyPoolStaleStatus(response.status) &&
|
|
688
|
+
isProxyPoolStaleMessage(normalized.body)) {
|
|
689
|
+
throw createProxyPoolStaleError(response.status);
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
if (response.status >= 400 && options.throwOnHttpError !== false) {
|
|
693
|
+
throw new TransportError(`Upstream request failed with status ${response.status}`, {
|
|
694
|
+
code: "upstream_http_error",
|
|
695
|
+
status: response.status,
|
|
696
|
+
});
|
|
697
|
+
}
|
|
698
|
+
recordProxyAttempt("ok", undefined, response.status);
|
|
699
|
+
return normalized;
|
|
700
|
+
}
|
|
701
|
+
catch (error) {
|
|
702
|
+
const normalizedError = normalizeStealthTransportError(error);
|
|
703
|
+
recordProxyAttempt("error", proxyAttemptErrorCode(normalizedError), proxyAttemptStatus(normalizedError));
|
|
704
|
+
lastError = normalizedError;
|
|
705
|
+
if (proxy &&
|
|
706
|
+
usesPolicyAllocator &&
|
|
707
|
+
isProxyPoolRefreshableError(normalizedError)) {
|
|
708
|
+
stalePoolError = normalizedError;
|
|
709
|
+
if (shouldRunProxyAuthDiagnostic(normalizedError)) {
|
|
710
|
+
stalePoolDiagnosticProxy = proxy;
|
|
711
|
+
}
|
|
712
|
+
if (attempt + 1 < maxAttempts) {
|
|
713
|
+
continue;
|
|
714
|
+
}
|
|
715
|
+
break;
|
|
716
|
+
}
|
|
717
|
+
if (proxy &&
|
|
718
|
+
attempt + 1 <
|
|
719
|
+
(stealthRetryOptions
|
|
720
|
+
? Math.min(maxAttempts, stealthRetryOptions.attempts)
|
|
721
|
+
: maxAttempts) &&
|
|
722
|
+
(!hasExplicitRetryPolicy
|
|
723
|
+
? isRetrySafeStealthMethod(method)
|
|
724
|
+
: isExplicitStealthRetryAllowed(method, normalizedError, stealthRetryOptions)) &&
|
|
725
|
+
isRetryableProxyTransportError(normalizedError)) {
|
|
726
|
+
continue;
|
|
727
|
+
}
|
|
728
|
+
throw normalizedError;
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
if (usesPolicyAllocator &&
|
|
732
|
+
stalePoolError &&
|
|
733
|
+
refreshAttempt < MAX_POLICY_PROXY_POOL_REFRESHES) {
|
|
734
|
+
await invalidateProxyResolutionCacheAsync({
|
|
735
|
+
proxyPolicy: clientOptions.proxyPolicy,
|
|
736
|
+
upstream: clientOptions.upstream,
|
|
737
|
+
affinityKey: clientOptions.affinityKey,
|
|
738
|
+
});
|
|
739
|
+
continue;
|
|
740
|
+
}
|
|
741
|
+
const proxyAuthDiagnostic = stalePoolError && stalePoolDiagnosticProxy
|
|
742
|
+
? await classifyProxyAuthDiagnostic(options.profile ?? defaultProfile, stalePoolDiagnosticProxy)
|
|
743
|
+
: undefined;
|
|
744
|
+
if (proxyAuthDiagnostic === "source_ip_denied") {
|
|
745
|
+
throw createProxyAuthIpDeniedError(stalePoolError instanceof Error ? stalePoolError : undefined);
|
|
746
|
+
}
|
|
747
|
+
if (proxyAuthDiagnostic === "edge_auth_rejected") {
|
|
748
|
+
throw createProxyEdgeAuthRejectedError(stalePoolError instanceof Error ? stalePoolError : undefined);
|
|
749
|
+
}
|
|
750
|
+
if (stalePoolError) {
|
|
751
|
+
if (stalePoolError instanceof TransportError &&
|
|
752
|
+
stalePoolError.code === PROXY_EDGE_AUTH_REJECTED_CODE) {
|
|
753
|
+
throw stalePoolError;
|
|
754
|
+
}
|
|
755
|
+
throw createProxyPoolExhaustedError(stalePoolError instanceof Error ? stalePoolError : undefined);
|
|
756
|
+
}
|
|
757
|
+
break;
|
|
758
|
+
}
|
|
759
|
+
throw normalizeStealthTransportError(lastError);
|
|
760
|
+
},
|
|
761
|
+
close() {
|
|
762
|
+
closed = true;
|
|
763
|
+
clients.clear();
|
|
764
|
+
},
|
|
765
|
+
};
|
|
766
|
+
async function classifyProxyAuthDiagnostic(profileName, proxy) {
|
|
767
|
+
try {
|
|
768
|
+
const response = await getClient(profileName, proxy, false).fetch(PROXY_AUTH_DIAGNOSTIC_URL, {
|
|
769
|
+
method: "GET",
|
|
770
|
+
timeout: PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS,
|
|
771
|
+
});
|
|
772
|
+
const normalized = await normalizeResponse(response);
|
|
773
|
+
return classifyProxyAuthDiagnosticMessage(normalized.body);
|
|
774
|
+
}
|
|
775
|
+
catch (error) {
|
|
776
|
+
const message = error instanceof Error
|
|
777
|
+
? [
|
|
778
|
+
error.message,
|
|
779
|
+
error.cause instanceof Error ? error.cause.message : "",
|
|
780
|
+
]
|
|
781
|
+
.filter(Boolean)
|
|
782
|
+
.join(" ")
|
|
783
|
+
: String(error);
|
|
784
|
+
return classifyProxyAuthDiagnosticMessage(message);
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
function classifyProxyAuthDiagnosticMessage(message) {
|
|
789
|
+
if (isProxyAuthIpDeniedMessage(message)) {
|
|
790
|
+
return "source_ip_denied";
|
|
791
|
+
}
|
|
792
|
+
if (isProxyEdgeAuthRejectedMessage(message)) {
|
|
793
|
+
return "edge_auth_rejected";
|
|
794
|
+
}
|
|
795
|
+
return undefined;
|
|
796
|
+
}
|
|
797
|
+
function proxyAttemptErrorCode(error) {
|
|
798
|
+
return error.code ?? error.name ?? "transport_error";
|
|
799
|
+
}
|
|
800
|
+
function proxyAttemptStatus(error) {
|
|
801
|
+
return error.status ?? error.upstreamStatus;
|
|
802
|
+
}
|
|
803
|
+
function hasHeader(headers, name) {
|
|
804
|
+
const needle = name.toLowerCase();
|
|
805
|
+
return Object.keys(headers).some((key) => key.toLowerCase() === needle);
|
|
806
|
+
}
|
|
807
|
+
export function createStealthClient(baseUrl, defaultProfileOrOptions = DEFAULT_PROFILE, clientOptions = {}) {
|
|
808
|
+
const defaultProfile = typeof defaultProfileOrOptions === "string"
|
|
809
|
+
? defaultProfileOrOptions
|
|
810
|
+
: DEFAULT_PROFILE;
|
|
811
|
+
const resolvedClientOptions = typeof defaultProfileOrOptions === "string"
|
|
812
|
+
? clientOptions
|
|
813
|
+
: defaultProfileOrOptions;
|
|
814
|
+
let sharedSession = null;
|
|
815
|
+
function getSharedSession() {
|
|
816
|
+
if (!sharedSession) {
|
|
817
|
+
sharedSession = createSessionFetcher(baseUrl, defaultProfile, resolvedClientOptions);
|
|
818
|
+
}
|
|
819
|
+
return sharedSession;
|
|
820
|
+
}
|
|
821
|
+
return {
|
|
822
|
+
fetch(url, options) {
|
|
823
|
+
return getSharedSession().fetch(url, options);
|
|
824
|
+
},
|
|
825
|
+
createSession(opts) {
|
|
826
|
+
const sessionProfile = opts?.profile ?? defaultProfile;
|
|
827
|
+
return createSessionFetcher(baseUrl, sessionProfile, resolvedClientOptions);
|
|
828
|
+
},
|
|
829
|
+
close() {
|
|
830
|
+
sharedSession?.close();
|
|
831
|
+
sharedSession = null;
|
|
832
|
+
},
|
|
833
|
+
};
|
|
834
|
+
}
|