@apifuse/provider-sdk 2.2.0-beta.68 → 2.2.0-beta.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +6 -1
- package/CHANGELOG.md +4 -0
- package/README.md +10 -3
- package/dist/fixture-sanitization.js +159 -15
- package/dist/lint.js +56 -9
- package/dist/runtime/diagnostic-redactor.d.ts +6 -1
- package/dist/runtime/diagnostic-redactor.js +16 -14
- package/dist/runtime/http.js +15 -102
- package/dist/runtime/instrumentation.js +4 -1
- package/dist/runtime/redirect-hop-headers.d.ts +19 -0
- package/dist/runtime/redirect-hop-headers.js +58 -0
- package/dist/runtime/redirects.d.ts +33 -1
- package/dist/runtime/redirects.js +113 -0
- package/dist/runtime/resolver-span-attributes.d.ts +34 -0
- package/dist/runtime/resolver-span-attributes.js +93 -0
- package/dist/runtime/resolver-usage.d.ts +8 -0
- package/dist/runtime/resolver-usage.js +5 -11
- package/dist/runtime/resolver-vendors/browser.js +5 -4
- package/dist/runtime/resolver-vendors/capsolver.js +11 -10
- package/dist/runtime/resolver-vendors/twocaptcha.js +9 -8
- package/dist/runtime/resolver.js +7 -6
- package/dist/runtime/stealth-owned-headers.d.ts +21 -8
- package/dist/runtime/stealth-owned-headers.js +28 -8
- package/dist/runtime/stealth-telemetry.d.ts +1 -1
- package/dist/runtime/stealth.js +158 -46
- package/dist/runtime/trace.js +2 -2
- package/dist/server/serve-implementation.js +2 -1
- package/dist/testing/run.js +14 -7
- package/dist/trace-sanitization.js +1 -1
- package/dist/types.d.ts +30 -6
- package/package.json +2 -2
- package/src/fixture-sanitization.ts +175 -17
- package/src/lint.ts +54 -9
- package/src/runtime/diagnostic-redactor.ts +19 -13
- package/src/runtime/http.ts +22 -111
- package/src/runtime/instrumentation.ts +4 -1
- package/src/runtime/redirect-hop-headers.ts +71 -0
- package/src/runtime/redirects.ts +150 -1
- package/src/runtime/resolver-span-attributes.ts +139 -0
- package/src/runtime/resolver-usage.ts +8 -5
- package/src/runtime/resolver-vendors/browser.ts +5 -4
- package/src/runtime/resolver-vendors/capsolver.ts +11 -10
- package/src/runtime/resolver-vendors/twocaptcha.ts +9 -8
- package/src/runtime/resolver.ts +7 -6
- package/src/runtime/stealth-owned-headers.ts +34 -8
- package/src/runtime/stealth-telemetry.ts +6 -1
- package/src/runtime/stealth.ts +180 -58
- package/src/runtime/trace.ts +6 -2
- package/src/server/serve-implementation.ts +2 -1
- package/src/testing/run.ts +15 -2
- package/src/trace-sanitization.ts +1 -1
- package/src/types.ts +30 -6
|
@@ -1,20 +1,27 @@
|
|
|
1
|
+
import { SDKError } from "../errors.js";
|
|
1
2
|
/**
|
|
2
3
|
* Header names the stealth transport owns.
|
|
3
4
|
*
|
|
4
5
|
* `ctx.stealth` composes these from the Chrome emulation and the request class,
|
|
5
6
|
* so a caller value is rejected at request time with `SDKError` code
|
|
6
|
-
* `STEALTH_HEADER_OVERRIDE_UNSUPPORTED` (
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
7
|
+
* `STEALTH_HEADER_OVERRIDE_UNSUPPORTED` ({@link assertCallerHeadersSupported}).
|
|
8
|
+
* The offline test harness (`runStandardTests`) calls the same function on
|
|
9
|
+
* every `ctx.stealth` request it serves, so a provider test fails exactly where
|
|
10
|
+
* production would, whichever file or variable the header came from. That is
|
|
11
|
+
* the cross-file enforcement.
|
|
12
|
+
*
|
|
13
|
+
* The `browser-version-literal` lint reports a subset of these names — the
|
|
14
|
+
* `sec-fetch-*` and `sec-ch-ua*` families — in files that use `ctx.stealth`
|
|
15
|
+
* (`isReportedOwnedHeaderName` in lint.ts). Both read this module, so the lint
|
|
16
|
+
* cannot claim a rejection the runtime does not perform; it reports
|
|
17
|
+
* deliberately less than the runtime rejects (host, connection,
|
|
12
18
|
* accept-encoding and a non-literal user-agent stay silent — see that
|
|
13
19
|
* predicate for why), so a green `apifuse check` is not proof that no owned
|
|
14
20
|
* header is set.
|
|
15
21
|
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
22
|
+
* Kept off the stealth runtime: the only import is `errors.js`, whose own
|
|
23
|
+
* imports are type-only, so the lazily loaded CLI lint path does not pull the
|
|
24
|
+
* stealth runtime in.
|
|
18
25
|
*/
|
|
19
26
|
export const SDK_OWNED_EXACT_CHROME_HEADERS = new Set([
|
|
20
27
|
"host",
|
|
@@ -35,3 +42,16 @@ export function isStealthOwnedHeaderName(name) {
|
|
|
35
42
|
const lower = name.toLowerCase();
|
|
36
43
|
return (SDK_OWNED_EXACT_CHROME_HEADERS.has(lower) || lower.startsWith(SDK_OWNED_CHROME_HEADER_PREFIX));
|
|
37
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* Throws `SDKError` `STEALTH_HEADER_OVERRIDE_UNSUPPORTED` for the first caller
|
|
47
|
+
* header `ctx.stealth` owns. An `undefined` value is skipped, as the transport
|
|
48
|
+
* drops it before sending. The stealth runtime and the offline test harness
|
|
49
|
+
* both call this, so the two cannot disagree about what is rejected.
|
|
50
|
+
*/
|
|
51
|
+
export function assertCallerHeadersSupported(headers) {
|
|
52
|
+
for (const [name, value] of Object.entries(headers)) {
|
|
53
|
+
if (value === undefined || !isStealthOwnedHeaderName(name))
|
|
54
|
+
continue;
|
|
55
|
+
throw new SDKError(`Stealth transport owns the "${name.toLowerCase()}" header; remove it from headers.`, { code: "STEALTH_HEADER_OVERRIDE_UNSUPPORTED" });
|
|
56
|
+
}
|
|
57
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { StealthBrowser, StealthChallengeClassification, StealthOS, StealthProfileDescriptor } from "../types.js";
|
|
2
2
|
import { type ClosedEnum, type TelemetryContributor } from "./request-telemetry.js";
|
|
3
3
|
export type StealthTelemetryErrorCode = "transport_network_error" | "transport_timeout" | "transport_cancelled" | "upstream_http_error" | "response_too_large" | "proxy_connect_failed" | "PROXY_POOL_STALE" | "PROXY_EDGE_AUTH_REJECTED" | "PROXY_AUTH_IP_DENIED" | "PROXY_EDGE_TLS_REJECTED" | "PROXY_REQUIRED" | "other";
|
|
4
|
-
export type StealthTelemetryRequestClass = "navigation" | "script_navigation" | "xhr" | "post";
|
|
4
|
+
export type StealthTelemetryRequestClass = "navigation" | "script_navigation" | "form_post" | "xhr" | "post";
|
|
5
5
|
export type StealthTelemetryAttemptKind = "request" | "resolver" | "proxy_diagnostic";
|
|
6
6
|
export type StealthTelemetrySbsdOutcome = StealthChallengeClassification["outcome"] | "detected" | "refetch_clear";
|
|
7
7
|
export type StealthTelemetryDiagnostics = {
|
package/dist/runtime/stealth.js
CHANGED
|
@@ -5,11 +5,12 @@ import { SDKError, TransportError } from "../errors.js";
|
|
|
5
5
|
import { DEFAULT_STEALTH_BROWSER, DEFAULT_STEALTH_OS, getStealthProfile, resolverClientProfileFamily, resolveStealthProfileSelection, } from "../stealth/profiles.js";
|
|
6
6
|
import { chrome149HeaderOrder } from "./chrome149-header-order.js";
|
|
7
7
|
import { registerDiagnosticValue } from "./diagnostic-env.js";
|
|
8
|
-
import {
|
|
8
|
+
import { assertCallerHeadersSupported } from "./stealth-owned-headers.js";
|
|
9
9
|
import { ENGINE_CEREMONY_EGRESS_LEASE, } from "./egress-lease.js";
|
|
10
10
|
import { createProxyAuthIpDeniedError, createProxyEdgeAuthRejectedError, createProxyEdgeTlsRejectedError, createProxyPoolExhaustedError, createProxyPoolStaleError, isProxyAuthIpDeniedMessage, isProxyEdgeAuthRejectedMessage, isProxyEdgeTlsRejectedResponse, isProxyPoolRefreshableError, isProxyPoolStaleMessage, isProxyPoolStaleStatus, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_STALE_CODE, } from "./proxy-errors.js";
|
|
11
11
|
import { computeProxyAttemptIndex, computeProxyTransportRetryDelayMs, createDefaultProxyTransportRetryOptions, isProxyTransportRetryMethod, normalizeProxyTransportRetryOptions, shouldRetryProxyTransportAttempt, validateUnsafeProxyTransportRetryMethods, } from "./proxy-retry-policy.js";
|
|
12
|
-
import {
|
|
12
|
+
import { redirectHopHeaders } from "./redirect-hop-headers.js";
|
|
13
|
+
import { evaluateRedirectHop, invalidRedirectPolicy, isRedirectStatus, nextRedirectMethod, nextSameOriginRedirectHop, normalizeRedirectPolicy, resolveRedirectUrl, } from "./redirects.js";
|
|
13
14
|
import { isSensitiveKey, normalizeSensitiveParams, redactSensitiveError, redactSensitiveRequestError, redactSensitiveText, redactUrlQueryParams, serializeRequestUrl, } from "./request-options.js";
|
|
14
15
|
import { akamaiSbsdChallengeKey, detectAkamaiSbsdChallenge, } from "./stealth-akamai-sbsd.js";
|
|
15
16
|
import { StealthCookieJar } from "./stealth-cookies.js";
|
|
@@ -37,12 +38,10 @@ function isBoundEgressTransportFailure(error) {
|
|
|
37
38
|
(error.code !== undefined && BOUND_EGRESS_TRANSPORT_FAILURE_CODES.has(error.code)));
|
|
38
39
|
}
|
|
39
40
|
const MAX_STEALTH_REDIRECT_HOPS = 10;
|
|
40
|
-
const
|
|
41
|
-
"
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
"content-type",
|
|
45
|
-
]);
|
|
41
|
+
const STEALTH_REDIRECT_POLICY_TRANSPORT = {
|
|
42
|
+
label: "ctx.stealth",
|
|
43
|
+
maxHops: MAX_STEALTH_REDIRECT_HOPS,
|
|
44
|
+
};
|
|
46
45
|
function isRecord(value) {
|
|
47
46
|
return typeof value === "object" && value !== null;
|
|
48
47
|
}
|
|
@@ -179,6 +178,8 @@ function normalizedHeaderEntries(headers) {
|
|
|
179
178
|
// installs Accept-Language via DevTools next to User-Agent; real Chrome only
|
|
180
179
|
// receives it from //net (URLRequestHttpJob::AddExtraHeaders), after
|
|
181
180
|
// Accept-Encoding and before Cookie, as al-placement-capture.json B/C show.
|
|
181
|
+
// chrome-form-post-capture.json was taken without that option and needs no
|
|
182
|
+
// adjustment.
|
|
182
183
|
const CHROME_HEADER_ORDERS = {
|
|
183
184
|
navigation: [
|
|
184
185
|
"sec-ch-ua",
|
|
@@ -215,6 +216,27 @@ const CHROME_HEADER_ORDERS = {
|
|
|
215
216
|
"cookie",
|
|
216
217
|
"priority",
|
|
217
218
|
],
|
|
219
|
+
"form-post": [
|
|
220
|
+
"content-length",
|
|
221
|
+
"cache-control",
|
|
222
|
+
"sec-ch-ua",
|
|
223
|
+
"sec-ch-ua-mobile",
|
|
224
|
+
"sec-ch-ua-platform",
|
|
225
|
+
"upgrade-insecure-requests",
|
|
226
|
+
"content-type",
|
|
227
|
+
"user-agent",
|
|
228
|
+
"origin",
|
|
229
|
+
"accept",
|
|
230
|
+
"sec-fetch-site",
|
|
231
|
+
"sec-fetch-mode",
|
|
232
|
+
"sec-fetch-user",
|
|
233
|
+
"sec-fetch-dest",
|
|
234
|
+
"referer",
|
|
235
|
+
"accept-encoding",
|
|
236
|
+
"accept-language",
|
|
237
|
+
"cookie",
|
|
238
|
+
"priority",
|
|
239
|
+
],
|
|
218
240
|
};
|
|
219
241
|
const CHROME_H1_HEADER_ORDERS = {
|
|
220
242
|
navigation: [
|
|
@@ -271,6 +293,28 @@ const CHROME_H1_HEADER_ORDERS = {
|
|
|
271
293
|
"accept-language",
|
|
272
294
|
"cookie",
|
|
273
295
|
],
|
|
296
|
+
"form-post": [
|
|
297
|
+
"host",
|
|
298
|
+
"connection",
|
|
299
|
+
"content-length",
|
|
300
|
+
"cache-control",
|
|
301
|
+
"sec-ch-ua",
|
|
302
|
+
"sec-ch-ua-mobile",
|
|
303
|
+
"sec-ch-ua-platform",
|
|
304
|
+
"upgrade-insecure-requests",
|
|
305
|
+
"content-type",
|
|
306
|
+
"user-agent",
|
|
307
|
+
"origin",
|
|
308
|
+
"accept",
|
|
309
|
+
"sec-fetch-site",
|
|
310
|
+
"sec-fetch-mode",
|
|
311
|
+
"sec-fetch-user",
|
|
312
|
+
"sec-fetch-dest",
|
|
313
|
+
"referer",
|
|
314
|
+
"accept-encoding",
|
|
315
|
+
"accept-language",
|
|
316
|
+
"cookie",
|
|
317
|
+
],
|
|
274
318
|
};
|
|
275
319
|
const CHROME_H1_HEADER_NAMES = {
|
|
276
320
|
host: "Host",
|
|
@@ -293,6 +337,25 @@ const CHROME_H1_HEADER_NAMES = {
|
|
|
293
337
|
origin: "Origin",
|
|
294
338
|
range: "Range",
|
|
295
339
|
};
|
|
340
|
+
/**
|
|
341
|
+
* What each request class sends besides its header order, one row per public
|
|
342
|
+
* `stealth.requestClass`:
|
|
343
|
+
*
|
|
344
|
+
* - `fetchMetadata: "navigate"` is a document navigation: `Sec-Fetch-Mode:
|
|
345
|
+
* navigate`, `Sec-Fetch-Dest: document`, `Sec-Fetch-User: ?1` unless
|
|
346
|
+
* `userActivation` is false, `Upgrade-Insecure-Requests: 1`, and the
|
|
347
|
+
* profile's `Accept` and `Priority`. `"cors"` is a script fetch: `cors` /
|
|
348
|
+
* `empty`, `Accept: *\/*`, `Priority: u=1, i`.
|
|
349
|
+
* - `bodyful` sends `Content-Length`.
|
|
350
|
+
* - `validateCache` sends `Cache-Control: max-age=0`, which Chrome adds to a
|
|
351
|
+
* form submission (chrome-form-post-capture.json).
|
|
352
|
+
*/
|
|
353
|
+
const CHROME_REQUEST_CLASSES = {
|
|
354
|
+
navigation: { fetchMetadata: "navigate", bodyful: false, validateCache: false },
|
|
355
|
+
"form-post": { fetchMetadata: "navigate", bodyful: true, validateCache: true },
|
|
356
|
+
xhr: { fetchMetadata: "cors", bodyful: false, validateCache: false },
|
|
357
|
+
post: { fetchMetadata: "cors", bodyful: true, validateCache: false },
|
|
358
|
+
};
|
|
296
359
|
function chromeHeaderOrder(requestClass, isHttp1, caller) {
|
|
297
360
|
if (isHttp1)
|
|
298
361
|
return CHROME_H1_HEADER_ORDERS[requestClass];
|
|
@@ -320,29 +383,11 @@ function chromeHeaderOrder(requestClass, isHttp1, caller) {
|
|
|
320
383
|
}
|
|
321
384
|
return order;
|
|
322
385
|
}
|
|
323
|
-
function normalizedCallerHeaderEntries(headers) {
|
|
324
|
-
const entries = normalizedHeaderEntries(headers);
|
|
325
|
-
assertCallerHeadersSupported(entries);
|
|
326
|
-
return entries;
|
|
327
|
-
}
|
|
328
386
|
function normalizedCallerHeaderEntriesFromRecord(headers) {
|
|
329
387
|
// The record has been through normalizeHeaders (lowercase, merged names); the
|
|
330
388
|
// lowercase here only guards later insertions such as the cookie jar's header.
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
value,
|
|
334
|
-
]);
|
|
335
|
-
assertCallerHeadersSupported(entries);
|
|
336
|
-
return entries;
|
|
337
|
-
}
|
|
338
|
-
function assertCallerHeadersSupported(entries) {
|
|
339
|
-
for (const [name] of entries) {
|
|
340
|
-
if (isStealthOwnedHeaderName(name)) {
|
|
341
|
-
throw new SDKError(`Stealth transport owns the "${name}" header; remove it from headers.`, {
|
|
342
|
-
code: "STEALTH_HEADER_OVERRIDE_UNSUPPORTED",
|
|
343
|
-
});
|
|
344
|
-
}
|
|
345
|
-
}
|
|
389
|
+
assertCallerHeadersSupported(headers);
|
|
390
|
+
return Object.entries(headers).map(([name, value]) => [name.toLowerCase(), value]);
|
|
346
391
|
}
|
|
347
392
|
function chromeRequestClass(method, requestedClass) {
|
|
348
393
|
if (requestedClass)
|
|
@@ -374,9 +419,10 @@ function buildChromeHeaderTuples(options) {
|
|
|
374
419
|
const caller = new Map(callerEntries);
|
|
375
420
|
const emulation = new Map(Array.from(options.emulationHeaders, ([name, value]) => [name.toLowerCase(), value]));
|
|
376
421
|
const requestClass = chromeRequestClass(options.method, options.requestClass);
|
|
422
|
+
const shape = CHROME_REQUEST_CLASSES[requestClass];
|
|
377
423
|
const referer = caller.get("referer");
|
|
378
424
|
const fetchSite = secFetchSite(options.requestUrl, referer);
|
|
379
|
-
const isNavigation =
|
|
425
|
+
const isNavigation = shape.fetchMetadata === "navigate";
|
|
380
426
|
const values = new Map([
|
|
381
427
|
["sec-ch-ua", requiredEmulationHeader(emulation, "sec-ch-ua")],
|
|
382
428
|
["sec-ch-ua-mobile", requiredEmulationHeader(emulation, "sec-ch-ua-mobile")],
|
|
@@ -403,6 +449,8 @@ function buildChromeHeaderTuples(options) {
|
|
|
403
449
|
if (options.userActivation !== false)
|
|
404
450
|
values.set("sec-fetch-user", "?1");
|
|
405
451
|
}
|
|
452
|
+
if (shape.validateCache)
|
|
453
|
+
values.set("cache-control", "max-age=0");
|
|
406
454
|
for (const [name, value] of callerEntries)
|
|
407
455
|
values.set(name, value);
|
|
408
456
|
for (const name of ["content-type", "origin", "referer"]) {
|
|
@@ -410,7 +458,10 @@ function buildChromeHeaderTuples(options) {
|
|
|
410
458
|
if (value !== undefined)
|
|
411
459
|
values.set(name, value);
|
|
412
460
|
}
|
|
413
|
-
|
|
461
|
+
// A bodyful class keeps its class across a 301/302/303 that rewrites the method
|
|
462
|
+
// to GET (a form submission's redirect stays a validated navigation), but the
|
|
463
|
+
// rewritten request has no body and Chrome sends no Content-Length for it.
|
|
464
|
+
if (shape.bodyful && options.method !== "GET" && options.method !== "HEAD") {
|
|
414
465
|
values.set("content-length", caller.get("content-length") ?? String(Buffer.byteLength(options.body ?? "")));
|
|
415
466
|
}
|
|
416
467
|
const isHttp1 = new URL(options.requestUrl).protocol === "http:";
|
|
@@ -865,8 +916,25 @@ function locationHeader(headers) {
|
|
|
865
916
|
}
|
|
866
917
|
return undefined;
|
|
867
918
|
}
|
|
868
|
-
|
|
869
|
-
|
|
919
|
+
/**
|
|
920
|
+
* Snapshot a caller's `redirectPolicy` before the first send. The policy acts
|
|
921
|
+
* on redirects the transport follows, so it requires the default
|
|
922
|
+
* `redirect: "follow"`.
|
|
923
|
+
*/
|
|
924
|
+
function snapshotStealthRedirectPolicy(options) {
|
|
925
|
+
let policy;
|
|
926
|
+
try {
|
|
927
|
+
policy = normalizeRedirectPolicy(options.redirectPolicy, STEALTH_REDIRECT_POLICY_TRANSPORT);
|
|
928
|
+
}
|
|
929
|
+
catch (error) {
|
|
930
|
+
if (error instanceof TransportError)
|
|
931
|
+
throw error;
|
|
932
|
+
throw invalidRedirectPolicy(STEALTH_REDIRECT_POLICY_TRANSPORT, "could not be read safely", error instanceof Error ? error : undefined);
|
|
933
|
+
}
|
|
934
|
+
if (policy && options.redirect !== undefined && options.redirect !== "follow") {
|
|
935
|
+
throw invalidRedirectPolicy(STEALTH_REDIRECT_POLICY_TRANSPORT, `it applies to followed redirects and cannot be combined with redirect: "${options.redirect}"`);
|
|
936
|
+
}
|
|
937
|
+
return policy;
|
|
870
938
|
}
|
|
871
939
|
function assertStealthRedirectUrl(url) {
|
|
872
940
|
const protocol = new URL(url).protocol;
|
|
@@ -962,7 +1030,9 @@ function startStealthExchange(telemetry, method, options, signal) {
|
|
|
962
1030
|
kind: telemetry.kind,
|
|
963
1031
|
requestClass: requestClass === "navigation" && options.stealth?.userActivation === false
|
|
964
1032
|
? "script_navigation"
|
|
965
|
-
: requestClass
|
|
1033
|
+
: requestClass === "form-post"
|
|
1034
|
+
? "form_post"
|
|
1035
|
+
: requestClass,
|
|
966
1036
|
...(status === undefined ? {} : { status }),
|
|
967
1037
|
...(errorCode ? { errorCode } : {}),
|
|
968
1038
|
...(error === undefined
|
|
@@ -988,6 +1058,10 @@ async function fetchStealthRedirectChain(transport, cookieJar, requestUrl, metho
|
|
|
988
1058
|
let followedHops = 0;
|
|
989
1059
|
let response;
|
|
990
1060
|
const deadline = options.timeout ? performance.now() + options.timeout : undefined;
|
|
1061
|
+
// Snapshotted and validated by session.fetch before the first send.
|
|
1062
|
+
const redirectPolicy = options.redirectPolicy;
|
|
1063
|
+
const initialOrigin = new URL(requestUrl).origin;
|
|
1064
|
+
const visitedRequests = new Set([`${method} ${requestUrl}`]);
|
|
991
1065
|
let finishExchange;
|
|
992
1066
|
let status;
|
|
993
1067
|
let exchangeError;
|
|
@@ -1040,23 +1114,42 @@ async function fetchStealthRedirectChain(transport, cookieJar, requestUrl, metho
|
|
|
1040
1114
|
status: 0,
|
|
1041
1115
|
});
|
|
1042
1116
|
}
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1117
|
+
let nextUrl;
|
|
1118
|
+
let nextMethod;
|
|
1119
|
+
if (redirectPolicy) {
|
|
1120
|
+
discardStealthRedirectBody(response);
|
|
1121
|
+
({ nextUrl, nextMethod } = nextSameOriginRedirectHop({
|
|
1122
|
+
policy: redirectPolicy,
|
|
1123
|
+
initialOrigin,
|
|
1124
|
+
currentUrl,
|
|
1125
|
+
responseUrl: response.url ?? currentUrl,
|
|
1126
|
+
status: response.status,
|
|
1127
|
+
method: currentMethod,
|
|
1128
|
+
location: response.headers.get("location") ?? undefined,
|
|
1129
|
+
followedHops,
|
|
1130
|
+
visitedRequests,
|
|
1131
|
+
}));
|
|
1132
|
+
}
|
|
1133
|
+
else {
|
|
1134
|
+
const location = resolveRedirectUrl(response.headers.get("location") ?? undefined, response.url ?? currentUrl);
|
|
1135
|
+
if (!location)
|
|
1136
|
+
break;
|
|
1137
|
+
if (followedHops >= MAX_STEALTH_REDIRECT_HOPS) {
|
|
1138
|
+
discardStealthRedirectBody(response);
|
|
1139
|
+
throw new TransportError(`Stealth request exceeded the ${MAX_STEALTH_REDIRECT_HOPS}-redirect limit`, { code: "transport_network_error", status: 0 });
|
|
1140
|
+
}
|
|
1047
1141
|
discardStealthRedirectBody(response);
|
|
1048
|
-
|
|
1142
|
+
nextUrl = location;
|
|
1143
|
+
nextMethod = nextRedirectMethod(response.status, currentMethod);
|
|
1049
1144
|
}
|
|
1050
1145
|
assertStealthRedirectUrl(nextUrl);
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
if (nextMethod !== currentMethod) {
|
|
1146
|
+
currentHeaders = redirectHopHeaders({ from: currentUrl, to: nextUrl, methodChanged: nextMethod !== currentMethod }, currentHeaders);
|
|
1147
|
+
if (nextMethod !== currentMethod)
|
|
1054
1148
|
currentBody = undefined;
|
|
1055
|
-
currentHeaders = withoutRedirectBodyHeaders(currentHeaders);
|
|
1056
|
-
}
|
|
1057
1149
|
currentMethod = nextMethod;
|
|
1058
1150
|
currentUrl = nextUrl;
|
|
1059
1151
|
followedHops += 1;
|
|
1152
|
+
visitedRequests.add(`${currentMethod} ${currentUrl}`);
|
|
1060
1153
|
emitStealthTelemetry(telemetry?.sink, (sink) => sink.recordRedirectHop());
|
|
1061
1154
|
finishExchange(status);
|
|
1062
1155
|
finishExchange = undefined;
|
|
@@ -1155,10 +1248,15 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
1155
1248
|
let entry = clients.get(cacheKey);
|
|
1156
1249
|
if (!entry) {
|
|
1157
1250
|
entry = {
|
|
1251
|
+
// The header shape only partitions sessions; it is never handed to wreq as
|
|
1252
|
+
// session `defaultHeaders`. wreq merges those into every request that lacks
|
|
1253
|
+
// one of their names, so a header the SDK left off on purpose — a credential
|
|
1254
|
+
// dropped on a cross-origin redirect hop, the body headers after POST -> GET,
|
|
1255
|
+
// a cookie for another host — would come back with the first request's value.
|
|
1256
|
+
// Every request already sends its complete ordered header list.
|
|
1158
1257
|
session: wreq.createSession({
|
|
1159
1258
|
browser,
|
|
1160
1259
|
os,
|
|
1161
|
-
...(defaultHeaders ? { defaultHeaders } : {}),
|
|
1162
1260
|
...(proxyUrl ? { proxy: proxyUrl } : {}),
|
|
1163
1261
|
...(ignoreTlsErrors ? { insecure: true } : {}),
|
|
1164
1262
|
timeout: 30_000,
|
|
@@ -1332,7 +1430,12 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
1332
1430
|
const { hasExplicitRetryPolicy, method, stealthRetryOptions } = (() => {
|
|
1333
1431
|
try {
|
|
1334
1432
|
const method = normalizeMethod(options.method ?? "GET");
|
|
1335
|
-
|
|
1433
|
+
assertCallerHeadersSupported(options.headers ?? {});
|
|
1434
|
+
// Replace the caller's object with the validated copy: the walker, a
|
|
1435
|
+
// retry and an explicit replay all read this snapshot.
|
|
1436
|
+
const redirectPolicy = snapshotStealthRedirectPolicy(options);
|
|
1437
|
+
if (redirectPolicy)
|
|
1438
|
+
options.redirectPolicy = redirectPolicy;
|
|
1336
1439
|
const hasExplicitRetryPolicy = options.retry !== undefined;
|
|
1337
1440
|
const stealthRetryOptions = normalizeProxyTransportRetryOptions(options.retry, {
|
|
1338
1441
|
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
@@ -1900,10 +2003,11 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
1900
2003
|
const maxHops = options.maxHops === undefined || !Number.isFinite(options.maxHops)
|
|
1901
2004
|
? 10
|
|
1902
2005
|
: Math.max(0, Math.floor(options.maxHops));
|
|
1903
|
-
const { url: _url, maxHops: _maxHops, stopWhen, params, sensitiveParams, ...fetchOptions } = options;
|
|
2006
|
+
const { url: _url, maxHops: _maxHops, stopWhen, params, sensitiveParams, headers: initialHeaders, ...fetchOptions } = options;
|
|
1904
2007
|
const hops = [];
|
|
1905
2008
|
let method = normalizeMethod(options.method ?? "GET");
|
|
1906
2009
|
let body = options.body;
|
|
2010
|
+
let headers = initialHeaders;
|
|
1907
2011
|
let response;
|
|
1908
2012
|
const visitedRequests = new Set();
|
|
1909
2013
|
const initialParams = params
|
|
@@ -1956,6 +2060,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
1956
2060
|
try {
|
|
1957
2061
|
response = await session.fetch(currentUrl, {
|
|
1958
2062
|
...fetchOptions,
|
|
2063
|
+
...(headers ? { headers } : {}),
|
|
1959
2064
|
body,
|
|
1960
2065
|
method,
|
|
1961
2066
|
...(hopIndex === 0 && initialParams ? { params: initialParams } : {}),
|
|
@@ -2042,6 +2147,13 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
2042
2147
|
cookieStore: cookieJar.serialize(),
|
|
2043
2148
|
};
|
|
2044
2149
|
}
|
|
2150
|
+
if (headers) {
|
|
2151
|
+
headers = redirectHopHeaders({
|
|
2152
|
+
from: currentUrl,
|
|
2153
|
+
to: decision.nextUrl,
|
|
2154
|
+
methodChanged: decision.nextMethod !== method,
|
|
2155
|
+
}, headers);
|
|
2156
|
+
}
|
|
2045
2157
|
if (decision.nextMethod !== method) {
|
|
2046
2158
|
body = undefined;
|
|
2047
2159
|
}
|
package/dist/runtime/trace.js
CHANGED
|
@@ -76,7 +76,7 @@ function normalizeAttributes(attributes, redact, retainBigints = false) {
|
|
|
76
76
|
? redactedKey === REDACTION_FAILED
|
|
77
77
|
? REDACTION_FAILED
|
|
78
78
|
: REDACTED_QUERY_VALUE
|
|
79
|
-
: normalizeAttributeValue(value, diagnosticAttributeRedactor(key, redact), retainBigints);
|
|
79
|
+
: normalizeAttributeValue(value, diagnosticAttributeRedactor(key, redact, value), retainBigints);
|
|
80
80
|
return [normalizedKey, normalizedValue];
|
|
81
81
|
})
|
|
82
82
|
.filter((entry) => entry[1] !== undefined);
|
|
@@ -103,7 +103,7 @@ function prepareResourceAttributesForExport(attributes, redact, sanitize) {
|
|
|
103
103
|
? redactedKey === REDACTION_FAILED
|
|
104
104
|
? REDACTION_FAILED
|
|
105
105
|
: REDACTED_QUERY_VALUE
|
|
106
|
-
: sanitizeSpanNameForOutput(value, diagnosticAttributeRedactor(key, redact)),
|
|
106
|
+
: sanitizeSpanNameForOutput(value, diagnosticAttributeRedactor(key, redact, value)),
|
|
107
107
|
];
|
|
108
108
|
}));
|
|
109
109
|
}
|
|
@@ -41,6 +41,7 @@ import { getProviderBaseUrl, getProviderStealthBaseUrl } from "../runtime/provid
|
|
|
41
41
|
import { PROXY_AUTH_IP_DENIED_CODE, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_EXHAUSTED_CODE, } from "../runtime/proxy-errors.js";
|
|
42
42
|
import { PROVIDER_TELEMETRY_HEADER, ProxyTelemetryCollector, } from "../runtime/proxy-telemetry.js";
|
|
43
43
|
import { closedEnum, RequestTelemetry, tenantOpaqueKeys, } from "../runtime/request-telemetry.js";
|
|
44
|
+
import { resolverSpanAttributes } from "../runtime/resolver-span-attributes.js";
|
|
44
45
|
import { bindResolverTelemetry, createUnsupportedResolverClient, inheritResolverTelemetryBinding, } from "../runtime/resolver-shared.js";
|
|
45
46
|
import { ResolverTelemetryCollector, } from "../runtime/resolver-telemetry.js";
|
|
46
47
|
import { createResolverRuntimeOptions, resolveNativeProxyPolicy, } from "../runtime/resolver-runtime-options.js";
|
|
@@ -385,7 +386,7 @@ function createStealthChallengeDetection(provider, resolverRuntime, resolverOver
|
|
|
385
386
|
const solve = () => resolver.solve(challenge, solveSignal, recorder ?? undefined);
|
|
386
387
|
return recorder
|
|
387
388
|
? recorder.runSpan("resolver.solve", solve, {
|
|
388
|
-
attributes: { challenge_kind: challenge.kind },
|
|
389
|
+
attributes: resolverSpanAttributes({ challenge_kind: challenge.kind }),
|
|
389
390
|
})
|
|
390
391
|
: solve();
|
|
391
392
|
},
|
package/dist/testing/run.js
CHANGED
|
@@ -8,6 +8,7 @@ import { createTestHandleContext } from "../runtime/handle.js";
|
|
|
8
8
|
*/
|
|
9
9
|
const STANDARD_TEST_CONNECTION_ID = "standard-test-connection";
|
|
10
10
|
import { createMemoryProviderRuntimeState } from "../runtime/state.js";
|
|
11
|
+
import { assertCallerHeadersSupported } from "../runtime/stealth-owned-headers.js";
|
|
11
12
|
import { createUnsupportedOcrClient } from "../runtime/ocr.js";
|
|
12
13
|
import { createUnsupportedSttClient } from "../runtime/stt.js";
|
|
13
14
|
import { createNativeEgressAuthorization, NativeNetworkError, snapshotNativeConnectInput, snapshotNativeGrantInput, } from "../runtime/native-network.js";
|
|
@@ -213,13 +214,19 @@ function createUpstreamContext(provider, operationName, upstreamStub) {
|
|
|
213
214
|
return normalizeUpstreamResponse(canned);
|
|
214
215
|
};
|
|
215
216
|
const httpCall = async (method, url, body, options) => toHttpResponse(await dispatch({ transport: "http", method, url, body, options }));
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
217
|
+
// The production transport rejects owned headers (User-Agent, Sec-Fetch-*,
|
|
218
|
+
// sec-ch-ua*, ...) before any byte is sent; the fake applies the same
|
|
219
|
+
// function, so a provider test fails exactly where production would.
|
|
220
|
+
const stealthCall = async (url, options) => {
|
|
221
|
+
assertCallerHeadersSupported(options?.headers ?? {});
|
|
222
|
+
return toStealthResponse(await dispatch({
|
|
223
|
+
transport: "stealth",
|
|
224
|
+
method: options?.method?.toUpperCase() ?? "GET",
|
|
225
|
+
url,
|
|
226
|
+
body: options?.body,
|
|
227
|
+
options,
|
|
228
|
+
}), url);
|
|
229
|
+
};
|
|
223
230
|
const createBrowserPage = () => {
|
|
224
231
|
let currentUrl = "about:blank";
|
|
225
232
|
let currentResponse;
|
|
@@ -22,7 +22,7 @@ export function sanitizeTraceAttributes(attributes, redact, reservedKeys = []) {
|
|
|
22
22
|
const sanitizedKey = keyChanged ? nextKey() : sanitizeTraceText(key);
|
|
23
23
|
const valueRedactor = typeof value === "bigint" || isDiagnosticBigintAttribute(attributes, key)
|
|
24
24
|
? diagnosticStructuredRedactor(redact)
|
|
25
|
-
: diagnosticAttributeRedactor(key, redact);
|
|
25
|
+
: diagnosticAttributeRedactor(key, redact, value);
|
|
26
26
|
const sanitizedValue = keyChanged
|
|
27
27
|
? redactedKey === REDACTION_FAILED
|
|
28
28
|
? REDACTION_FAILED
|
package/dist/types.d.ts
CHANGED
|
@@ -1193,7 +1193,23 @@ export interface StealthFetchOptions extends Omit<RequestOptions, "redirectPolic
|
|
|
1193
1193
|
headers?: Record<string, string | string[]>;
|
|
1194
1194
|
method?: HttpMethod;
|
|
1195
1195
|
body?: string | Buffer;
|
|
1196
|
+
/**
|
|
1197
|
+
* Defaults to `"follow"`. Every followed hop drops the request-body headers
|
|
1198
|
+
* when the method changes, `Authorization` and `Proxy-Authorization` when it
|
|
1199
|
+
* leaves the current origin, and an explicit `Cookie` when it leaves the
|
|
1200
|
+
* current site (scheme plus registrable domain); after that the session
|
|
1201
|
+
* cookie jar supplies cookies for the new site.
|
|
1202
|
+
*/
|
|
1196
1203
|
redirect?: "follow" | "manual" | "error";
|
|
1204
|
+
/**
|
|
1205
|
+
* Opt-in same-origin redirect enforcement, the ctx.http `redirectPolicy`
|
|
1206
|
+
* for the stealth transport. A redirect whose target leaves the initial
|
|
1207
|
+
* origin, exceeds `maxHops` (at most 10, the stealth redirect limit),
|
|
1208
|
+
* loops, or has no usable `Location` throws `HttpRedirectError` before any
|
|
1209
|
+
* request is sent to that target. Requires the default
|
|
1210
|
+
* `redirect: "follow"`.
|
|
1211
|
+
*/
|
|
1212
|
+
redirectPolicy?: HttpRedirectPolicy;
|
|
1197
1213
|
/**
|
|
1198
1214
|
* Maximum decoded response-body bytes to buffer. When set, the stealth
|
|
1199
1215
|
* transport aborts the response and throws `response_too_large` if the
|
|
@@ -1219,13 +1235,21 @@ export interface StealthFetchOptions extends Omit<RequestOptions, "redirectPolic
|
|
|
1219
1235
|
* replacement for hand-set `Sec-Fetch-*` headers, which the transport
|
|
1220
1236
|
* rejects; it drives `Sec-Fetch-Mode`/`Dest`/`User`, `Accept`, `Priority`,
|
|
1221
1237
|
* and `Upgrade-Insecure-Requests`.
|
|
1238
|
+
*
|
|
1239
|
+
* `form-post` is an HTML `<form method="post">` submission: a navigation
|
|
1240
|
+
* (`navigate`/`document`, the document `Accept`, `Upgrade-Insecure-Requests`)
|
|
1241
|
+
* that carries a body, in the header order Chrome uses for it, with
|
|
1242
|
+
* `Cache-Control: max-age=0` and the caller's `Content-Type`, `Origin` and
|
|
1243
|
+
* `Referer` in place. It is never inferred, since a form submission and a
|
|
1244
|
+
* script `fetch()` POST look the same to the transport.
|
|
1222
1245
|
*/
|
|
1223
|
-
requestClass?: "navigation" | "xhr" | "post";
|
|
1246
|
+
requestClass?: "navigation" | "form-post" | "xhr" | "post";
|
|
1224
1247
|
/**
|
|
1225
|
-
* Whether a `navigation` request carries a user gesture and
|
|
1226
|
-
* `Sec-Fetch-User: ?1` (default `true`). Set `false` for
|
|
1227
|
-
* navigations such as `location.replace
|
|
1228
|
-
* Ignored for the `xhr` and
|
|
1248
|
+
* Whether a `navigation` or `form-post` request carries a user gesture and
|
|
1249
|
+
* therefore `Sec-Fetch-User: ?1` (default `true`). Set `false` for
|
|
1250
|
+
* script-driven navigations such as `location.replace` or
|
|
1251
|
+
* `form.submit()`, which Chrome sends without it. Ignored for the `xhr` and
|
|
1252
|
+
* `post` request classes.
|
|
1229
1253
|
*/
|
|
1230
1254
|
userActivation?: boolean;
|
|
1231
1255
|
/**
|
|
@@ -1321,7 +1345,7 @@ export interface StealthRedirectHop {
|
|
|
1321
1345
|
location?: string;
|
|
1322
1346
|
nextUrl?: string;
|
|
1323
1347
|
}
|
|
1324
|
-
export interface StealthRedirectRunOptions extends Omit<StealthFetchOptions, "redirect"> {
|
|
1348
|
+
export interface StealthRedirectRunOptions extends Omit<StealthFetchOptions, "redirect" | "redirectPolicy"> {
|
|
1325
1349
|
url: string;
|
|
1326
1350
|
maxHops?: number;
|
|
1327
1351
|
stopWhen?: (hop: StealthRedirectHop) => boolean | Promise<boolean>;
|
package/package.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"version": "2.2.0-beta.
|
|
2
|
+
"version": "2.2.0-beta.69",
|
|
3
3
|
"name": "@apifuse/provider-sdk",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
@@ -140,7 +140,7 @@
|
|
|
140
140
|
},
|
|
141
141
|
"dependencies": {
|
|
142
142
|
"@clack/prompts": "^1.5.1",
|
|
143
|
-
"@rgrove/parse-xml": "4.2.
|
|
143
|
+
"@rgrove/parse-xml": "4.2.3",
|
|
144
144
|
"@types/ms": "^2.1.0",
|
|
145
145
|
"acorn": "^8.17.0",
|
|
146
146
|
"hono": "^4.12.25",
|