@apifuse/provider-sdk 2.2.0-beta.48 → 2.2.0-beta.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +91 -36
- package/CHANGELOG.md +4 -0
- package/README.md +11 -9
- package/SUBMISSION.md +1 -1
- package/bin/apifuse-dev.ts +24 -13
- package/bin/apifuse-migrate-operation-declaration.ts +55 -0
- package/bin/apifuse-pack-smoke.ts +1 -1
- package/bin/apifuse-pack-types.ts +2 -1
- package/bin/apifuse-record.ts +30 -16
- package/bin/apifuse-submit-check.ts +20 -35
- package/dist/cli/commands.d.ts +1 -1
- package/dist/cli/commands.js +11 -0
- package/dist/cli/migrate-operation-declaration.d.ts +59 -0
- package/dist/cli/migrate-operation-declaration.js +1178 -0
- package/dist/cli/templates/provider/README.md.tpl +3 -3
- package/dist/cli/templates/provider/operations/ping.ts.tpl +2 -0
- package/dist/config/loader.d.ts +2 -0
- package/dist/config/loader.js +18 -7
- package/dist/contract-types.d.ts +11 -5
- package/dist/contract.js +21 -10
- package/dist/define.d.ts +25 -22
- package/dist/define.js +49 -75
- package/dist/dev.d.ts +3 -0
- package/dist/dev.js +1 -1
- package/dist/engine.d.ts +78 -0
- package/dist/engine.js +133 -0
- package/dist/index.d.ts +3 -2
- package/dist/index.js +2 -1
- package/dist/lint.d.ts +7 -15
- package/dist/lint.js +45 -70
- package/dist/provider.d.ts +3 -1
- package/dist/provider.js +1 -0
- package/dist/runtime/chrome149-header-order.d.ts +58 -0
- package/dist/runtime/chrome149-header-order.js +289 -0
- package/dist/runtime/env.js +12 -0
- package/dist/runtime/executor.d.ts +2 -1
- package/dist/runtime/executor.js +3 -36
- package/dist/runtime/insights.js +2 -2
- package/dist/runtime/otlp.d.ts +71 -2
- package/dist/runtime/otlp.js +397 -16
- package/dist/runtime/resolver-vendors/capsolver.js +3 -3
- package/dist/runtime/resolver.js +3 -3
- package/dist/runtime/stealth.d.ts +13 -4
- package/dist/runtime/stealth.js +362 -85
- package/dist/runtime/trace-config.js +2 -1
- package/dist/runtime/trace.d.ts +5 -0
- package/dist/runtime/trace.js +43 -10
- package/dist/server/self-test.d.ts +1 -3
- package/dist/server/self-test.js +2 -12
- package/dist/server/serve-implementation.d.ts +6 -1
- package/dist/server/serve-implementation.js +55 -40
- package/dist/server/trace-output.d.ts +3 -1
- package/dist/server/trace-output.js +61 -2
- package/dist/stealth/profiles.d.ts +9 -8
- package/dist/stealth/profiles.js +123 -286
- package/dist/types.d.ts +111 -108
- package/package.json +2 -1
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/approval-override.ts.txt +6 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/codemod-syntax.ts.txt +3 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/connection-precedence.ts.txt +10 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/docs-conflict.ts.txt +8 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/examples-map.ts.txt +5 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/examples-operation.ts.txt +16 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/factory-map.ts.txt +3 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/hoist-all.ts.txt +31 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/hoisted-const.ts.txt +11 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/imported-spread.ts.txt +11 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/inline-map.ts.txt +11 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/inline-spread-cast-tail.ts.txt +21 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/inline-spread-ekitan.ts.txt +11 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/inline-spread-override.ts.txt +14 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/missing-english-locale.ts.txt +7 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/no-safety.ts.txt +6 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/non-literal.ts.txt +7 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/redundant-approval.ts.txt +6 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/safety-conflict.ts.txt +7 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/stream.ts.txt +7 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/tool-router-spread.ts.txt +15 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/unparseable.ts.txt +4 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/verbatim-template.ts.txt +12 -0
- package/src/cli/commands.ts +13 -0
- package/src/cli/migrate-operation-declaration.ts +1654 -0
- package/src/cli/templates/provider/README.md.tpl +3 -3
- package/src/cli/templates/provider/operations/ping.ts.tpl +2 -0
- package/src/config/loader.ts +31 -6
- package/src/contract-types.ts +11 -5
- package/src/contract.ts +21 -10
- package/src/define.ts +107 -119
- package/src/dev.ts +4 -1
- package/src/engine.ts +279 -0
- package/src/index.ts +13 -5
- package/src/lint.ts +58 -92
- package/src/provider.ts +25 -3
- package/src/runtime/chrome149-header-order.ts +330 -0
- package/src/runtime/env.ts +13 -0
- package/src/runtime/executor.ts +7 -40
- package/src/runtime/insights.ts +2 -2
- package/src/runtime/otlp.ts +467 -21
- package/src/runtime/resolver-vendors/capsolver.ts +4 -3
- package/src/runtime/resolver.ts +3 -3
- package/src/runtime/stealth.ts +435 -103
- package/src/runtime/trace-config.ts +3 -2
- package/src/runtime/trace.ts +57 -17
- package/src/server/self-test.ts +2 -9
- package/src/server/serve-implementation.ts +89 -72
- package/src/server/trace-output.ts +99 -2
- package/src/stealth/profiles.ts +169 -327
- package/src/types.ts +109 -137
package/dist/runtime/stealth.js
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { DEFAULT_SMARTPROXY_POOL_SIZE, invalidateProxyResolutionCacheAsync, ProxyResolutionError, policyResolvesRegistryVendorChain, resolvePolicyProxyPoolSpan, resolvePolicyTransportAttemptCap, resolveProxyConfigAsync, vendorFromResolvedSource, } from "../config/loader.js";
|
|
3
3
|
import { SDKError, TransportError } from "../errors.js";
|
|
4
|
-
import { getStealthProfile,
|
|
5
|
-
import {
|
|
4
|
+
import { DEFAULT_STEALTH_BROWSER, DEFAULT_STEALTH_OS, getStealthProfile, resolveStealthProfileSelection, } from "../stealth/profiles.js";
|
|
5
|
+
import { chrome149HeaderOrder } from "./chrome149-header-order.js";
|
|
6
6
|
import { createProxyAuthIpDeniedError, createProxyEdgeAuthRejectedError, createProxyEdgeTlsRejectedError, createProxyPoolExhaustedError, createProxyPoolStaleError, isProxyAuthIpDeniedMessage, isProxyEdgeAuthRejectedMessage, isProxyEdgeTlsRejectedResponse, isProxyPoolRefreshableError, isProxyPoolStaleMessage, isProxyPoolStaleStatus, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_STALE_CODE, } from "./proxy-errors.js";
|
|
7
7
|
import { computeProxyAttemptIndex, computeProxyTransportRetryDelayMs, createDefaultProxyTransportRetryOptions, normalizeProxyTransportRetryOptions, shouldRetryProxyTransportAttempt, validateUnsafeProxyTransportRetryMethods, } from "./proxy-retry-policy.js";
|
|
8
8
|
import { evaluateRedirectHop, isRedirectStatus, nextRedirectMethod, resolveRedirectUrl, } from "./redirects.js";
|
|
9
9
|
import { isSensitiveKey, normalizeSensitiveParams, redactSensitiveError, redactSensitiveRequestError, redactSensitiveText, redactUrlQueryParams, serializeRequestUrl, } from "./request-options.js";
|
|
10
|
-
|
|
10
|
+
import { StealthCookieJar } from "./stealth-cookies.js";
|
|
11
|
+
export const DEFAULT_STEALTH_PROFILE = Object.freeze({
|
|
12
|
+
browser: DEFAULT_STEALTH_BROWSER,
|
|
13
|
+
os: DEFAULT_STEALTH_OS,
|
|
14
|
+
});
|
|
11
15
|
const MISSING_PROXY_WARNING = "[provider-sdk] Provider requested proxy routing, but no proxy URL was configured. Continuing without proxy.";
|
|
12
16
|
const MAX_POLICY_PROXY_POOL_REFRESHES = 1;
|
|
13
17
|
const PROXY_CONNECT_FAILURE_CODE = "proxy_connect_failed";
|
|
@@ -33,19 +37,6 @@ function sensitiveQueryParamNames(url) {
|
|
|
33
37
|
const query = url.slice(queryStart + 1, fragmentStart === -1 ? undefined : fragmentStart);
|
|
34
38
|
return [...new URLSearchParams(query).keys()].filter(isSensitiveKey);
|
|
35
39
|
}
|
|
36
|
-
const REMOVED_CHROME_PROFILE_NAMES = new Set([
|
|
37
|
-
"chrome-120",
|
|
38
|
-
"chrome-124",
|
|
39
|
-
"chrome-129",
|
|
40
|
-
"chrome-130",
|
|
41
|
-
"chrome-131",
|
|
42
|
-
"chrome-133",
|
|
43
|
-
"chrome-144",
|
|
44
|
-
"chrome-146-psk",
|
|
45
|
-
"chrome-131-psk",
|
|
46
|
-
"chrome-130-psk",
|
|
47
|
-
"edge-131",
|
|
48
|
-
]);
|
|
49
40
|
let wreqModulePromise;
|
|
50
41
|
function getWreqModule() {
|
|
51
42
|
if (!wreqModulePromise) {
|
|
@@ -94,48 +85,13 @@ function closestWreqProfile(identifier, wreqProfiles) {
|
|
|
94
85
|
}
|
|
95
86
|
return closest?.name;
|
|
96
87
|
}
|
|
97
|
-
function
|
|
98
|
-
|
|
99
|
-
try {
|
|
100
|
-
profile = getStealthProfile(DEFAULT_PROFILE);
|
|
101
|
-
}
|
|
102
|
-
catch (error) {
|
|
103
|
-
throw new SDKError(`Default stealth profile "${DEFAULT_PROFILE}" cannot be mapped to a wreq-js browser profile.`, { cause: error instanceof Error ? error : undefined });
|
|
104
|
-
}
|
|
88
|
+
export function resolveWreqProfile(selection, wreqProfiles) {
|
|
89
|
+
const profile = getStealthProfile(selection);
|
|
105
90
|
const identifier = profile.tlsClientIdentifier?.toLowerCase() ?? "";
|
|
106
|
-
|
|
107
|
-
throw new SDKError(`Default stealth profile "${DEFAULT_PROFILE}" cannot be mapped to a wreq-js browser profile.`);
|
|
108
|
-
}
|
|
109
|
-
return { identifier, os: profile.platform };
|
|
110
|
-
}
|
|
111
|
-
let defaultWreqProfileMapping;
|
|
112
|
-
function getDefaultWreqProfileMapping() {
|
|
113
|
-
defaultWreqProfileMapping ??= resolveDefaultWreqProfileMapping();
|
|
114
|
-
return defaultWreqProfileMapping;
|
|
115
|
-
}
|
|
116
|
-
export function resolveWreqProfile(profileName, wreqProfiles) {
|
|
117
|
-
if (REMOVED_CHROME_PROFILE_NAMES.has(profileName)) {
|
|
118
|
-
throw new SDKError(`Unknown stealth profile: ${profileName}`);
|
|
119
|
-
}
|
|
120
|
-
let identifier;
|
|
121
|
-
let os;
|
|
122
|
-
try {
|
|
123
|
-
const profile = getStealthProfile(profileName);
|
|
124
|
-
identifier = profile.tlsClientIdentifier?.toLowerCase() ?? "";
|
|
125
|
-
os = profile.platform;
|
|
126
|
-
}
|
|
127
|
-
catch {
|
|
128
|
-
// Preserve the previous ctx.stealth.fetch() compatibility behavior: unknown
|
|
129
|
-
// profile strings still run with the transport default instead of failing
|
|
130
|
-
// before the request starts. Removed built-in profile aliases above remain
|
|
131
|
-
// explicit errors so callers do not accidentally pin retired fingerprints.
|
|
132
|
-
const defaultMapping = getDefaultWreqProfileMapping();
|
|
133
|
-
identifier = defaultMapping.identifier;
|
|
134
|
-
os = defaultMapping.os;
|
|
135
|
-
}
|
|
91
|
+
const os = profile.os;
|
|
136
92
|
const browser = closestWreqProfile(identifier, wreqProfiles);
|
|
137
93
|
if (!browser) {
|
|
138
|
-
throw new SDKError(`Stealth profile
|
|
94
|
+
throw new SDKError(`Stealth profile ${profile.browser}/${profile.os} cannot be mapped to a wreq-js browser profile.`);
|
|
139
95
|
}
|
|
140
96
|
return { browser, os };
|
|
141
97
|
}
|
|
@@ -146,13 +102,302 @@ function headerEntriesFromHeaders(headers) {
|
|
|
146
102
|
return Array.from(headers.entries());
|
|
147
103
|
}
|
|
148
104
|
function normalizeHeaders(headers) {
|
|
149
|
-
|
|
150
|
-
|
|
105
|
+
return Object.fromEntries(normalizedHeaderEntries(headers));
|
|
106
|
+
}
|
|
107
|
+
function trimOuterHttpWhitespace(value) {
|
|
108
|
+
return value.replace(/^[\t\n\r ]+|[\t\n\r ]+$/g, "");
|
|
109
|
+
}
|
|
110
|
+
function normalizedHeaderEntries(headers) {
|
|
111
|
+
const entries = [];
|
|
112
|
+
const indices = new Map();
|
|
113
|
+
for (const [originalName, originalValue] of Object.entries(headers)) {
|
|
114
|
+
if (originalValue === undefined)
|
|
115
|
+
continue;
|
|
116
|
+
const name = originalName.toLowerCase();
|
|
117
|
+
const value = (Array.isArray(originalValue) ? originalValue : [originalValue])
|
|
118
|
+
.map(trimOuterHttpWhitespace)
|
|
119
|
+
.join(", ");
|
|
120
|
+
const existingIndex = indices.get(name);
|
|
121
|
+
if (existingIndex === undefined) {
|
|
122
|
+
indices.set(name, entries.length);
|
|
123
|
+
entries.push([name, value]);
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
const existing = entries[existingIndex];
|
|
127
|
+
if (existing)
|
|
128
|
+
existing[1] = `${existing[1]}, ${value}`;
|
|
129
|
+
}
|
|
130
|
+
return entries;
|
|
131
|
+
}
|
|
132
|
+
const SDK_OWNED_EXACT_CHROME_HEADERS = new Set([
|
|
133
|
+
"host",
|
|
134
|
+
"connection",
|
|
135
|
+
"user-agent",
|
|
136
|
+
"sec-ch-ua",
|
|
137
|
+
"sec-ch-ua-mobile",
|
|
138
|
+
"sec-ch-ua-platform",
|
|
139
|
+
"accept-encoding",
|
|
140
|
+
]);
|
|
141
|
+
// Non-pseudo-header order of real Chrome 149. The fixture captures
|
|
142
|
+
// (chrome-ground-truth-capture.json, chrome-extended-capture.json,
|
|
143
|
+
// h1-casing-capture.json) were taken through Playwright's `locale` option, which
|
|
144
|
+
// installs Accept-Language via DevTools next to User-Agent; real Chrome only
|
|
145
|
+
// receives it from //net (URLRequestHttpJob::AddExtraHeaders), after
|
|
146
|
+
// Accept-Encoding and before Cookie, as al-placement-capture.json B/C show.
|
|
147
|
+
const CHROME_HEADER_ORDERS = {
|
|
148
|
+
navigation: [
|
|
149
|
+
"sec-ch-ua",
|
|
150
|
+
"sec-ch-ua-mobile",
|
|
151
|
+
"sec-ch-ua-platform",
|
|
152
|
+
"upgrade-insecure-requests",
|
|
153
|
+
"user-agent",
|
|
154
|
+
"accept",
|
|
155
|
+
"sec-fetch-site",
|
|
156
|
+
"sec-fetch-mode",
|
|
157
|
+
"sec-fetch-user",
|
|
158
|
+
"sec-fetch-dest",
|
|
159
|
+
"accept-encoding",
|
|
160
|
+
"accept-language",
|
|
161
|
+
"cookie",
|
|
162
|
+
"priority",
|
|
163
|
+
],
|
|
164
|
+
// xhr has no fixed table: it is emulated by chrome149HeaderOrder.
|
|
165
|
+
post: [
|
|
166
|
+
"content-length",
|
|
167
|
+
"sec-ch-ua-platform",
|
|
168
|
+
"user-agent",
|
|
169
|
+
"sec-ch-ua",
|
|
170
|
+
"content-type",
|
|
171
|
+
"sec-ch-ua-mobile",
|
|
172
|
+
"accept",
|
|
173
|
+
"origin",
|
|
174
|
+
"sec-fetch-site",
|
|
175
|
+
"sec-fetch-mode",
|
|
176
|
+
"sec-fetch-dest",
|
|
177
|
+
"referer",
|
|
178
|
+
"accept-encoding",
|
|
179
|
+
"accept-language",
|
|
180
|
+
"cookie",
|
|
181
|
+
"priority",
|
|
182
|
+
],
|
|
183
|
+
};
|
|
184
|
+
const CHROME_H1_HEADER_ORDERS = {
|
|
185
|
+
navigation: [
|
|
186
|
+
"host",
|
|
187
|
+
"connection",
|
|
188
|
+
"sec-ch-ua",
|
|
189
|
+
"sec-ch-ua-mobile",
|
|
190
|
+
"sec-ch-ua-platform",
|
|
191
|
+
"upgrade-insecure-requests",
|
|
192
|
+
"user-agent",
|
|
193
|
+
"accept",
|
|
194
|
+
"sec-fetch-site",
|
|
195
|
+
"sec-fetch-mode",
|
|
196
|
+
"sec-fetch-user",
|
|
197
|
+
"sec-fetch-dest",
|
|
198
|
+
"accept-encoding",
|
|
199
|
+
"accept-language",
|
|
200
|
+
"cookie",
|
|
201
|
+
],
|
|
202
|
+
xhr: [
|
|
203
|
+
"host",
|
|
204
|
+
"connection",
|
|
205
|
+
"sec-ch-ua-platform",
|
|
206
|
+
"cache-control",
|
|
207
|
+
"x-requested-with",
|
|
208
|
+
"user-agent",
|
|
209
|
+
"sec-ch-ua",
|
|
210
|
+
"sec-ch-ua-mobile",
|
|
211
|
+
"accept",
|
|
212
|
+
"sec-fetch-site",
|
|
213
|
+
"sec-fetch-mode",
|
|
214
|
+
"sec-fetch-dest",
|
|
215
|
+
"referer",
|
|
216
|
+
"accept-encoding",
|
|
217
|
+
"accept-language",
|
|
218
|
+
"cookie",
|
|
219
|
+
],
|
|
220
|
+
post: [
|
|
221
|
+
"host",
|
|
222
|
+
"connection",
|
|
223
|
+
"content-length",
|
|
224
|
+
"sec-ch-ua-platform",
|
|
225
|
+
"user-agent",
|
|
226
|
+
"sec-ch-ua",
|
|
227
|
+
"content-type",
|
|
228
|
+
"sec-ch-ua-mobile",
|
|
229
|
+
"accept",
|
|
230
|
+
"origin",
|
|
231
|
+
"sec-fetch-site",
|
|
232
|
+
"sec-fetch-mode",
|
|
233
|
+
"sec-fetch-dest",
|
|
234
|
+
"referer",
|
|
235
|
+
"accept-encoding",
|
|
236
|
+
"accept-language",
|
|
237
|
+
"cookie",
|
|
238
|
+
],
|
|
239
|
+
};
|
|
240
|
+
const CHROME_H1_HEADER_NAMES = {
|
|
241
|
+
host: "Host",
|
|
242
|
+
connection: "Connection",
|
|
243
|
+
"upgrade-insecure-requests": "Upgrade-Insecure-Requests",
|
|
244
|
+
"user-agent": "User-Agent",
|
|
245
|
+
"accept-language": "Accept-Language",
|
|
246
|
+
accept: "Accept",
|
|
247
|
+
"sec-fetch-site": "Sec-Fetch-Site",
|
|
248
|
+
"sec-fetch-mode": "Sec-Fetch-Mode",
|
|
249
|
+
"sec-fetch-user": "Sec-Fetch-User",
|
|
250
|
+
"sec-fetch-dest": "Sec-Fetch-Dest",
|
|
251
|
+
"accept-encoding": "Accept-Encoding",
|
|
252
|
+
cookie: "Cookie",
|
|
253
|
+
"cache-control": "Cache-Control",
|
|
254
|
+
"x-requested-with": "X-Requested-With",
|
|
255
|
+
referer: "Referer",
|
|
256
|
+
"content-length": "Content-Length",
|
|
257
|
+
"content-type": "Content-Type",
|
|
258
|
+
origin: "Origin",
|
|
259
|
+
range: "Range",
|
|
260
|
+
};
|
|
261
|
+
function chromeHeaderOrder(requestClass, isHttp1, caller) {
|
|
262
|
+
if (isHttp1)
|
|
263
|
+
return CHROME_H1_HEADER_ORDERS[requestClass];
|
|
264
|
+
if (requestClass !== "xhr")
|
|
265
|
+
return CHROME_HEADER_ORDERS[requestClass];
|
|
266
|
+
// Cookie and Referer are forbidden Fetch headers, so they never enter the page
|
|
267
|
+
// Fetch Headers map the emulator models. Range does occupy a map bucket
|
|
268
|
+
// (m1-capture.json range_*), but HttpCache::Transaction removes it from the
|
|
269
|
+
// request headers and PartialData re-adds it at the tail.
|
|
270
|
+
const order = chrome149HeaderOrder([...caller.keys()].filter((name) => name !== "cookie" && name !== "referer"));
|
|
271
|
+
if (caller.has("range"))
|
|
272
|
+
order.splice(order.indexOf("range"), 1);
|
|
273
|
+
// //net appends Accept-Encoding and, unless the caller supplied one,
|
|
274
|
+
// Accept-Language (URLRequestHttpJob::AddExtraHeaders), then writes the Cookie
|
|
275
|
+
// header (SetCookieHeaderAndStart); the HTTP cache re-adds a caller Range after
|
|
276
|
+
// that, and the HTTP/2 Priority header stays last.
|
|
277
|
+
let insertAt = order.indexOf("accept-encoding") + 1;
|
|
278
|
+
if (order[insertAt] === "accept-language")
|
|
279
|
+
insertAt += 1;
|
|
280
|
+
for (const name of ["cookie", "range"]) {
|
|
281
|
+
if (!caller.has(name))
|
|
282
|
+
continue;
|
|
283
|
+
order.splice(insertAt, 0, name);
|
|
284
|
+
insertAt += 1;
|
|
285
|
+
}
|
|
286
|
+
return order;
|
|
287
|
+
}
|
|
288
|
+
function normalizedCallerHeaderEntries(headers) {
|
|
289
|
+
const entries = normalizedHeaderEntries(headers);
|
|
290
|
+
assertCallerHeadersSupported(entries);
|
|
291
|
+
return entries;
|
|
292
|
+
}
|
|
293
|
+
function normalizedCallerHeaderEntriesFromRecord(headers) {
|
|
294
|
+
// The record has been through normalizeHeaders (lowercase, merged names); the
|
|
295
|
+
// lowercase here only guards later insertions such as the cookie jar's header.
|
|
296
|
+
const entries = Object.entries(headers).map(([name, value]) => [
|
|
297
|
+
name.toLowerCase(),
|
|
298
|
+
value,
|
|
299
|
+
]);
|
|
300
|
+
assertCallerHeadersSupported(entries);
|
|
301
|
+
return entries;
|
|
302
|
+
}
|
|
303
|
+
function assertCallerHeadersSupported(entries) {
|
|
304
|
+
for (const [name] of entries) {
|
|
305
|
+
if (SDK_OWNED_EXACT_CHROME_HEADERS.has(name) || name.startsWith("sec-fetch-")) {
|
|
306
|
+
throw new SDKError(`Stealth transport owns the "${name}" header; remove it from headers.`, {
|
|
307
|
+
code: "STEALTH_HEADER_OVERRIDE_UNSUPPORTED",
|
|
308
|
+
});
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
function chromeRequestClass(method, requestedClass) {
|
|
313
|
+
if (requestedClass)
|
|
314
|
+
return requestedClass;
|
|
315
|
+
if (method === "POST")
|
|
316
|
+
return "post";
|
|
317
|
+
return "navigation";
|
|
318
|
+
}
|
|
319
|
+
function secFetchSite(requestUrl, referer) {
|
|
320
|
+
if (!referer)
|
|
321
|
+
return "none";
|
|
322
|
+
try {
|
|
323
|
+
return new URL(referer).origin === new URL(requestUrl).origin ? "same-origin" : "cross-site";
|
|
324
|
+
}
|
|
325
|
+
catch {
|
|
326
|
+
return "cross-site";
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
function requiredEmulationHeader(headers, name) {
|
|
330
|
+
const value = headers.get(name);
|
|
331
|
+
if (value !== undefined)
|
|
332
|
+
return value;
|
|
333
|
+
throw new SDKError(`wreq-js Chrome emulation exposes no ${name} header.`, {
|
|
334
|
+
code: "STEALTH_PROFILE_UNAVAILABLE",
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
function buildChromeHeaderTuples(options) {
|
|
338
|
+
const callerEntries = normalizedCallerHeaderEntriesFromRecord(options.headers);
|
|
339
|
+
const caller = new Map(callerEntries);
|
|
340
|
+
const emulation = new Map(Array.from(options.emulationHeaders, ([name, value]) => [name.toLowerCase(), value]));
|
|
341
|
+
const requestClass = chromeRequestClass(options.method, options.requestClass);
|
|
342
|
+
const referer = caller.get("referer");
|
|
343
|
+
const fetchSite = secFetchSite(options.requestUrl, referer);
|
|
344
|
+
const isNavigation = requestClass === "navigation";
|
|
345
|
+
const values = new Map([
|
|
346
|
+
["sec-ch-ua", requiredEmulationHeader(emulation, "sec-ch-ua")],
|
|
347
|
+
["sec-ch-ua-mobile", requiredEmulationHeader(emulation, "sec-ch-ua-mobile")],
|
|
348
|
+
["sec-ch-ua-platform", requiredEmulationHeader(emulation, "sec-ch-ua-platform")],
|
|
349
|
+
["user-agent", requiredEmulationHeader(emulation, "user-agent")],
|
|
350
|
+
[
|
|
351
|
+
"accept-language",
|
|
352
|
+
caller.get("accept-language") ??
|
|
353
|
+
options.acceptLanguage ??
|
|
354
|
+
requiredEmulationHeader(emulation, "accept-language"),
|
|
355
|
+
],
|
|
356
|
+
["accept", isNavigation ? requiredEmulationHeader(emulation, "accept") : "*/*"],
|
|
357
|
+
[
|
|
358
|
+
"accept-encoding",
|
|
359
|
+
caller.has("range") ? "identity" : requiredEmulationHeader(emulation, "accept-encoding"),
|
|
360
|
+
],
|
|
361
|
+
["priority", isNavigation ? requiredEmulationHeader(emulation, "priority") : "u=1, i"],
|
|
362
|
+
["sec-fetch-site", fetchSite],
|
|
363
|
+
["sec-fetch-mode", isNavigation ? "navigate" : "cors"],
|
|
364
|
+
["sec-fetch-dest", isNavigation ? "document" : "empty"],
|
|
365
|
+
]);
|
|
366
|
+
if (isNavigation) {
|
|
367
|
+
values.set("upgrade-insecure-requests", "1");
|
|
368
|
+
values.set("sec-fetch-user", "?1");
|
|
369
|
+
}
|
|
370
|
+
for (const [name, value] of callerEntries)
|
|
371
|
+
values.set(name, value);
|
|
372
|
+
for (const name of ["content-type", "origin", "referer"]) {
|
|
373
|
+
const value = caller.get(name);
|
|
374
|
+
if (value !== undefined)
|
|
375
|
+
values.set(name, value);
|
|
376
|
+
}
|
|
377
|
+
if (requestClass === "post") {
|
|
378
|
+
values.set("content-length", caller.get("content-length") ?? String(Buffer.byteLength(options.body ?? "")));
|
|
379
|
+
}
|
|
380
|
+
const isHttp1 = new URL(options.requestUrl).protocol === "http:";
|
|
381
|
+
if (isHttp1) {
|
|
382
|
+
values.set("host", new URL(options.requestUrl).host);
|
|
383
|
+
values.set("connection", "keep-alive");
|
|
384
|
+
}
|
|
385
|
+
const order = chromeHeaderOrder(requestClass, isHttp1, caller);
|
|
386
|
+
const tuples = [];
|
|
387
|
+
const placed = new Set();
|
|
388
|
+
for (const name of order) {
|
|
389
|
+
const value = values.get(name);
|
|
151
390
|
if (value === undefined)
|
|
152
391
|
continue;
|
|
153
|
-
|
|
392
|
+
tuples.push([isHttp1 ? (CHROME_H1_HEADER_NAMES[name] ?? name) : name, value]);
|
|
393
|
+
placed.add(name);
|
|
154
394
|
}
|
|
155
|
-
|
|
395
|
+
for (const [name, value] of callerEntries) {
|
|
396
|
+
if (placed.has(name))
|
|
397
|
+
continue;
|
|
398
|
+
tuples.push([isHttp1 ? (CHROME_H1_HEADER_NAMES[name] ?? name) : name, value]);
|
|
399
|
+
}
|
|
400
|
+
return tuples;
|
|
156
401
|
}
|
|
157
402
|
function hasOwn(object, key) {
|
|
158
403
|
return Object.hasOwn(object, key);
|
|
@@ -170,7 +415,7 @@ function assertNoUnsupportedFingerprintOverrides(options) {
|
|
|
170
415
|
unsupported.push("stealth.h2");
|
|
171
416
|
if (unsupported.length === 0)
|
|
172
417
|
return;
|
|
173
|
-
throw new SDKError(`ctx.stealth.fetch uses transport-managed browser fingerprints and no longer accepts low-level stealth overrides: ${unsupported.join(", ")}. Use
|
|
418
|
+
throw new SDKError(`ctx.stealth.fetch uses transport-managed browser fingerprints and no longer accepts low-level stealth overrides: ${unsupported.join(", ")}. Use stealth.browser and stealth.os instead.`);
|
|
174
419
|
}
|
|
175
420
|
function responseHeadersToRecord(headers) {
|
|
176
421
|
const record = {};
|
|
@@ -575,26 +820,29 @@ function discardStealthRedirectBody(response) {
|
|
|
575
820
|
// failure must not replace or delay that decision.
|
|
576
821
|
}
|
|
577
822
|
}
|
|
578
|
-
async function fetchStealthRedirectChain(transport, cookieJar, requestUrl, method, options, signal) {
|
|
823
|
+
async function fetchStealthRedirectChain(transport, cookieJar, requestUrl, method, options, signal, buildHeaders) {
|
|
579
824
|
let currentUrl = requestUrl;
|
|
580
825
|
let currentMethod = method;
|
|
581
826
|
let currentBody = options.body === undefined ? undefined : normalizeBody(options.body);
|
|
582
|
-
let currentHeaders = { ...(options.headers ?? {}) };
|
|
827
|
+
let currentHeaders = normalizeHeaders({ ...(options.headers ?? {}) });
|
|
583
828
|
let followedHops = 0;
|
|
584
829
|
let response;
|
|
585
830
|
const deadline = options.timeout ? performance.now() + options.timeout : undefined;
|
|
586
831
|
while (true) {
|
|
587
832
|
throwIfAmbientAborted(signal);
|
|
588
833
|
const headers = { ...currentHeaders };
|
|
589
|
-
if (!hasHeader(headers, "
|
|
834
|
+
if (!hasHeader(headers, "cookie")) {
|
|
590
835
|
const cookieHeader = cookieJar.toHeader(currentUrl);
|
|
591
836
|
if (cookieHeader)
|
|
592
|
-
headers.
|
|
837
|
+
headers.cookie = cookieHeader;
|
|
593
838
|
}
|
|
594
839
|
const requestInit = {
|
|
595
|
-
headers:
|
|
840
|
+
headers: buildHeaders
|
|
841
|
+
? buildHeaders(currentUrl, currentMethod, currentBody, headers)
|
|
842
|
+
: headers,
|
|
596
843
|
method: currentMethod,
|
|
597
844
|
redirect: "manual",
|
|
845
|
+
...(new URL(currentUrl).protocol === "http:" ? { disableDefaultHeaders: true } : {}),
|
|
598
846
|
...(signal ? { signal } : {}),
|
|
599
847
|
};
|
|
600
848
|
if (currentBody !== undefined)
|
|
@@ -653,24 +901,27 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
653
901
|
let closed = false;
|
|
654
902
|
let hasWarnedMissingProxy = false;
|
|
655
903
|
const warn = clientOptions.warn ?? console.warn;
|
|
656
|
-
const intentAlias = getStealthProfileIntentAlias(defaultProfile);
|
|
657
|
-
if (intentAlias) {
|
|
658
|
-
warn(`[provider-sdk] Stealth profile "${defaultProfile}" pins a browser version and is deprecated. Use the intent profile "${intentAlias}" so TLS, headers, and User-Agent stay aligned with SDK updates; derive an explicit User-Agent with getStealthProfile("${intentAlias}").userAgent instead of hardcoding one.`);
|
|
659
|
-
}
|
|
660
904
|
const cookieJar = new StealthCookieJar([], baseUrl);
|
|
661
|
-
async function getClientEntry(
|
|
905
|
+
async function getClientEntry(profile, proxyUrl, ignoreTlsErrors, defaultHeaders) {
|
|
662
906
|
if (closed) {
|
|
663
907
|
throw new TransportError("Stealth session is closed", { status: 0 });
|
|
664
908
|
}
|
|
665
909
|
const wreq = await getWreqModule();
|
|
666
|
-
const { browser, os } = resolveWreqProfile(
|
|
667
|
-
const cacheKey = JSON.stringify({
|
|
910
|
+
const { browser, os } = resolveWreqProfile(profile, wreq.getProfiles());
|
|
911
|
+
const cacheKey = JSON.stringify({
|
|
912
|
+
browser,
|
|
913
|
+
os,
|
|
914
|
+
proxyUrl,
|
|
915
|
+
ignoreTlsErrors,
|
|
916
|
+
headerOrder: defaultHeaders?.map(([name]) => name),
|
|
917
|
+
});
|
|
668
918
|
let entry = clients.get(cacheKey);
|
|
669
919
|
if (!entry) {
|
|
670
920
|
entry = {
|
|
671
921
|
session: wreq.createSession({
|
|
672
922
|
browser,
|
|
673
923
|
os,
|
|
924
|
+
...(defaultHeaders ? { defaultHeaders } : {}),
|
|
674
925
|
...(proxyUrl ? { proxy: proxyUrl } : {}),
|
|
675
926
|
...(ignoreTlsErrors ? { insecure: true } : {}),
|
|
676
927
|
timeout: 30_000,
|
|
@@ -681,9 +932,9 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
681
932
|
}
|
|
682
933
|
return entry;
|
|
683
934
|
}
|
|
684
|
-
async function withClient(
|
|
935
|
+
async function withClient(profile, proxyUrl, ignoreTlsErrors, operation, signal, defaultHeaders) {
|
|
685
936
|
throwIfAmbientAborted(signal);
|
|
686
|
-
const entry = await getClientEntry(
|
|
937
|
+
const entry = await getClientEntry(profile, proxyUrl, ignoreTlsErrors, defaultHeaders);
|
|
687
938
|
throwIfAmbientAborted(signal);
|
|
688
939
|
const previous = entry.tail;
|
|
689
940
|
let release;
|
|
@@ -761,9 +1012,11 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
761
1012
|
}
|
|
762
1013
|
const session = {
|
|
763
1014
|
async fetch(url, options = {}) {
|
|
1015
|
+
const requestProfile = resolveStealthProfileSelection(options.stealth, defaultProfile);
|
|
764
1016
|
const { hasExplicitRetryPolicy, method, stealthRetryOptions } = (() => {
|
|
765
1017
|
try {
|
|
766
1018
|
const method = normalizeMethod(options.method ?? "GET");
|
|
1019
|
+
normalizedCallerHeaderEntries(options.headers ?? {});
|
|
767
1020
|
const hasExplicitRetryPolicy = options.retry !== undefined;
|
|
768
1021
|
const stealthRetryOptions = normalizeProxyTransportRetryOptions(options.retry, {
|
|
769
1022
|
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
@@ -870,10 +1123,32 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
870
1123
|
}
|
|
871
1124
|
const ignoreTlsErrors = Boolean(options.stealth?.insecureSkipVerify ??
|
|
872
1125
|
(!hasPolicyProxy && proxy && clientOptions.proxyStealth?.insecureSkipVerify));
|
|
873
|
-
const profileName = options.profile ?? defaultProfile;
|
|
874
1126
|
serializedUrl = serializeRequestUrl(resolveUrl(baseUrl, url), options.params, sensitiveParams);
|
|
875
1127
|
const { requestUrl } = serializedUrl;
|
|
876
|
-
const
|
|
1128
|
+
const wreq = await getWreqModule();
|
|
1129
|
+
const mapping = resolveWreqProfile(requestProfile, wreq.getProfiles());
|
|
1130
|
+
const chromeEmulationHeaders = mapping.browser.startsWith("chrome_")
|
|
1131
|
+
? Array.from(wreq.getEmulationHeaders(mapping.browser, mapping.os), ([name, value]) => [String(name), String(value)])
|
|
1132
|
+
: undefined;
|
|
1133
|
+
const buildOrderedHeaders = chromeEmulationHeaders
|
|
1134
|
+
? (currentUrl, currentMethod, currentBody, currentHeaders) => buildChromeHeaderTuples({
|
|
1135
|
+
emulationHeaders: chromeEmulationHeaders,
|
|
1136
|
+
method: currentMethod,
|
|
1137
|
+
body: currentBody,
|
|
1138
|
+
headers: currentHeaders,
|
|
1139
|
+
requestUrl: currentUrl,
|
|
1140
|
+
acceptLanguage: clientOptions.stealth?.acceptLanguage,
|
|
1141
|
+
requestClass: options.stealth?.requestClass,
|
|
1142
|
+
})
|
|
1143
|
+
: undefined;
|
|
1144
|
+
const initialHeaders = normalizeHeaders({ ...(options.headers ?? {}) });
|
|
1145
|
+
if (!hasHeader(initialHeaders, "cookie")) {
|
|
1146
|
+
const cookieHeader = cookieJar.toHeader(requestUrl);
|
|
1147
|
+
if (cookieHeader)
|
|
1148
|
+
initialHeaders.cookie = cookieHeader;
|
|
1149
|
+
}
|
|
1150
|
+
const defaultHeaders = buildOrderedHeaders?.(requestUrl, method, options.body === undefined ? undefined : normalizeBody(options.body), initialHeaders);
|
|
1151
|
+
const { normalized, response } = await withClient(requestProfile, proxy, ignoreTlsErrors, (transport) => fetchStealthRedirectChain(transport, cookieJar, requestUrl, method, options, clientOptions.signal, buildOrderedHeaders), clientOptions.signal, defaultHeaders);
|
|
877
1152
|
if (proxy && isProxyConnectFailureResponse(response, normalized.body)) {
|
|
878
1153
|
throw createProxyConnectFailureError(normalized.body);
|
|
879
1154
|
}
|
|
@@ -985,7 +1260,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
985
1260
|
continue;
|
|
986
1261
|
}
|
|
987
1262
|
const proxyAuthDiagnostic = stalePoolError && stalePoolDiagnosticProxy
|
|
988
|
-
? await classifyProxyAuthDiagnostic(
|
|
1263
|
+
? await classifyProxyAuthDiagnostic(requestProfile, stalePoolDiagnosticProxy)
|
|
989
1264
|
: undefined;
|
|
990
1265
|
if (proxyAuthDiagnostic === "source_ip_denied") {
|
|
991
1266
|
throw createProxyAuthIpDeniedError(stalePoolError instanceof Error ? stalePoolError : undefined);
|
|
@@ -1192,9 +1467,9 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
1192
1467
|
},
|
|
1193
1468
|
};
|
|
1194
1469
|
return session;
|
|
1195
|
-
async function classifyProxyAuthDiagnostic(
|
|
1470
|
+
async function classifyProxyAuthDiagnostic(profile, proxy) {
|
|
1196
1471
|
try {
|
|
1197
|
-
return await withClient(
|
|
1472
|
+
return await withClient(profile, proxy, false, async (client) => {
|
|
1198
1473
|
throwIfAmbientAborted(clientOptions.signal);
|
|
1199
1474
|
await client.clearCookies();
|
|
1200
1475
|
throwIfAmbientAborted(clientOptions.signal);
|
|
@@ -1237,13 +1512,15 @@ function hasHeader(headers, name) {
|
|
|
1237
1512
|
const needle = name.toLowerCase();
|
|
1238
1513
|
return Object.keys(headers).some((key) => key.toLowerCase() === needle);
|
|
1239
1514
|
}
|
|
1240
|
-
export function createStealthClient(baseUrl,
|
|
1241
|
-
|
|
1242
|
-
|
|
1515
|
+
export function createStealthClient(baseUrl, clientOptions = {}) {
|
|
1516
|
+
if (typeof clientOptions === "string") {
|
|
1517
|
+
resolveStealthProfileSelection(clientOptions);
|
|
1518
|
+
}
|
|
1519
|
+
const defaultProfile = resolveStealthProfileSelection(clientOptions.stealth);
|
|
1243
1520
|
let sharedSession = null;
|
|
1244
1521
|
function getSharedSession() {
|
|
1245
1522
|
if (!sharedSession) {
|
|
1246
|
-
sharedSession = createSessionFetcher(baseUrl, defaultProfile,
|
|
1523
|
+
sharedSession = createSessionFetcher(baseUrl, defaultProfile, clientOptions);
|
|
1247
1524
|
}
|
|
1248
1525
|
return sharedSession;
|
|
1249
1526
|
}
|
|
@@ -1252,8 +1529,8 @@ export function createStealthClient(baseUrl, defaultProfileOrOptions = DEFAULT_P
|
|
|
1252
1529
|
return getSharedSession().fetch(url, options);
|
|
1253
1530
|
},
|
|
1254
1531
|
createSession(opts) {
|
|
1255
|
-
const sessionProfile = opts?.
|
|
1256
|
-
return createSessionFetcher(baseUrl, sessionProfile,
|
|
1532
|
+
const sessionProfile = resolveStealthProfileSelection(opts?.stealth, defaultProfile);
|
|
1533
|
+
return createSessionFetcher(baseUrl, sessionProfile, clientOptions);
|
|
1257
1534
|
},
|
|
1258
1535
|
close() {
|
|
1259
1536
|
sharedSession?.close();
|
|
@@ -4,6 +4,7 @@ export const APIFUSE__TRACE__EXPORTER = "APIFUSE__TRACE__EXPORTER";
|
|
|
4
4
|
const TRACE_EXPORTER_LOOKUP = {
|
|
5
5
|
console: true,
|
|
6
6
|
json: true,
|
|
7
|
+
otlp: true,
|
|
7
8
|
none: true,
|
|
8
9
|
};
|
|
9
10
|
const TRACE_EXPORTERS = new Set(Object.keys(TRACE_EXPORTER_LOOKUP));
|
|
@@ -52,7 +53,7 @@ export function resolveTraceConfigFromEnv(env = process.env) {
|
|
|
52
53
|
? undefined
|
|
53
54
|
: "none";
|
|
54
55
|
if (exporterRaw !== undefined && exporter === "none" && normalizedExporter !== "none") {
|
|
55
|
-
warnOnce(APIFUSE__TRACE__EXPORTER, `[apifuse] Invalid APIFUSE__TRACE__EXPORTER value "${normalizedExporter ?? exporterRaw}"
|
|
56
|
+
warnOnce(APIFUSE__TRACE__EXPORTER, `[apifuse] Invalid APIFUSE__TRACE__EXPORTER value "${normalizedExporter ?? exporterRaw}"; supported exporters are "console", "json", "otlp", and "none"; falling back to exporter "none".`);
|
|
56
57
|
}
|
|
57
58
|
return {
|
|
58
59
|
...(enabled !== undefined ? { enabled } : {}),
|
package/dist/runtime/trace.d.ts
CHANGED
|
@@ -10,6 +10,11 @@ export interface CreateTraceContextOptions {
|
|
|
10
10
|
onSpan?: (span: Span) => void;
|
|
11
11
|
exportOptions?: OTLPExportOptions;
|
|
12
12
|
resourceAttributes?: Record<string, string>;
|
|
13
|
+
/**
|
|
14
|
+
* Applied to a detached copy of each span immediately before OTLP export; never touches
|
|
15
|
+
* getSpans() or onSpan. Returning nothing (or throwing) drops that export batch.
|
|
16
|
+
*/
|
|
17
|
+
sanitizeSpanForExport?: (span: Span) => Span | undefined;
|
|
13
18
|
}
|
|
14
19
|
type SpanHookOptions<T> = {
|
|
15
20
|
attributes?: Record<string, unknown>;
|