@apifuse/provider-sdk 2.2.0-beta.68 → 2.2.0-beta.69

package/src/runtime/stealth.ts

@@ -38,7 +38,7 @@ import type {
 } from "../types.js";
 import { chrome149HeaderOrder } from "./chrome149-header-order.js";
 import { registerDiagnosticValue } from "./diagnostic-env.js";
-import { isStealthOwnedHeaderName } from "./stealth-owned-headers.js";
+import { assertCallerHeadersSupported } from "./stealth-owned-headers.js";
 import {
 	ENGINE_CEREMONY_EGRESS_LEASE,
 	type CeremonyEgressBinding,
@@ -68,10 +68,15 @@ import {
 	shouldRetryProxyTransportAttempt,
 	validateUnsafeProxyTransportRetryMethods,
 } from "./proxy-retry-policy.js";
+import { redirectHopHeaders } from "./redirect-hop-headers.js";
 import {
 	evaluateRedirectHop,
+	invalidRedirectPolicy,
 	isRedirectStatus,
 	nextRedirectMethod,
+	nextSameOriginRedirectHop,
+	normalizeRedirectPolicy,
+	type RedirectPolicyTransport,
 	resolveRedirectUrl,
 } from "./redirects.js";
 import {
@@ -139,12 +144,10 @@ function isBoundEgressTransportFailure(error: TransportError): boolean {
 	);
 }
 const MAX_STEALTH_REDIRECT_HOPS = 10;
-const REDIRECT_BODY_HEADERS = new Set([
-	"content-encoding",
-	"content-language",
-	"content-location",
-	"content-type",
-]);
+const STEALTH_REDIRECT_POLICY_TRANSPORT: RedirectPolicyTransport = {
+	label: "ctx.stealth",
+	maxHops: MAX_STEALTH_REDIRECT_HOPS,
+};
 
 function isRecord(value: unknown): value is Record<PropertyKey, unknown> {
 	return typeof value === "object" && value !== null;
@@ -399,6 +402,8 @@ function normalizedHeaderEntries(
 // installs Accept-Language via DevTools next to User-Agent; real Chrome only
 // receives it from //net (URLRequestHttpJob::AddExtraHeaders), after
 // Accept-Encoding and before Cookie, as al-placement-capture.json B/C show.
+// chrome-form-post-capture.json was taken without that option and needs no
+// adjustment.
 const CHROME_HEADER_ORDERS = {
 	navigation: [
 		"sec-ch-ua",
@@ -435,7 +440,28 @@ const CHROME_HEADER_ORDERS = {
 		"cookie",
 		"priority",
 	],
-} as const;
+	"form-post": [
+		"content-length",
+		"cache-control",
+		"sec-ch-ua",
+		"sec-ch-ua-mobile",
+		"sec-ch-ua-platform",
+		"upgrade-insecure-requests",
+		"content-type",
+		"user-agent",
+		"origin",
+		"accept",
+		"sec-fetch-site",
+		"sec-fetch-mode",
+		"sec-fetch-user",
+		"sec-fetch-dest",
+		"referer",
+		"accept-encoding",
+		"accept-language",
+		"cookie",
+		"priority",
+	],
+} as const satisfies Record<Exclude<ChromeRequestClass, "xhr">, readonly string[]>;
 
 const CHROME_H1_HEADER_ORDERS = {
 	navigation: [
@@ -492,7 +518,29 @@ const CHROME_H1_HEADER_ORDERS = {
 		"accept-language",
 		"cookie",
 	],
-} as const;
+	"form-post": [
+		"host",
+		"connection",
+		"content-length",
+		"cache-control",
+		"sec-ch-ua",
+		"sec-ch-ua-mobile",
+		"sec-ch-ua-platform",
+		"upgrade-insecure-requests",
+		"content-type",
+		"user-agent",
+		"origin",
+		"accept",
+		"sec-fetch-site",
+		"sec-fetch-mode",
+		"sec-fetch-user",
+		"sec-fetch-dest",
+		"referer",
+		"accept-encoding",
+		"accept-language",
+		"cookie",
+	],
+} as const satisfies Record<ChromeRequestClass, readonly string[]>;
 
 const CHROME_H1_HEADER_NAMES: Record<string, string> = {
 	host: "Host",
@@ -516,7 +564,30 @@ const CHROME_H1_HEADER_NAMES: Record<string, string> = {
 	range: "Range",
 };
 
-type ChromeRequestClass = keyof typeof CHROME_HEADER_ORDERS | "xhr";
+type ChromeRequestClass = NonNullable<NonNullable<StealthFetchOptions["stealth"]>["requestClass"]>;
+
+/**
+ * What each request class sends besides its header order, one row per public
+ * `stealth.requestClass`:
+ *
+ * - `fetchMetadata: "navigate"` is a document navigation: `Sec-Fetch-Mode:
+ *   navigate`, `Sec-Fetch-Dest: document`, `Sec-Fetch-User: ?1` unless
+ *   `userActivation` is false, `Upgrade-Insecure-Requests: 1`, and the
+ *   profile's `Accept` and `Priority`. `"cors"` is a script fetch: `cors` /
+ *   `empty`, `Accept: *\/*`, `Priority: u=1, i`.
+ * - `bodyful` sends `Content-Length`.
+ * - `validateCache` sends `Cache-Control: max-age=0`, which Chrome adds to a
+ *   form submission (chrome-form-post-capture.json).
+ */
+const CHROME_REQUEST_CLASSES = {
+	navigation: { fetchMetadata: "navigate", bodyful: false, validateCache: false },
+	"form-post": { fetchMetadata: "navigate", bodyful: true, validateCache: true },
+	xhr: { fetchMetadata: "cors", bodyful: false, validateCache: false },
+	post: { fetchMetadata: "cors", bodyful: true, validateCache: false },
+} as const satisfies Record<
+	ChromeRequestClass,
+	{ fetchMetadata: "navigate" | "cors"; bodyful: boolean; validateCache: boolean }
+>;
 
 function chromeHeaderOrder(
 	requestClass: ChromeRequestClass,
@@ -547,33 +618,11 @@ function chromeHeaderOrder(
 	return order;
 }
 
-function normalizedCallerHeaderEntries(
-	headers: Record<string, string | string[] | undefined>,
-): HeaderTuple[] {
-	const entries = normalizedHeaderEntries(headers);
-	assertCallerHeadersSupported(entries);
-	return entries;
-}
-
 function normalizedCallerHeaderEntriesFromRecord(headers: Record<string, string>): HeaderTuple[] {
 	// The record has been through normalizeHeaders (lowercase, merged names); the
 	// lowercase here only guards later insertions such as the cookie jar's header.
-	const entries: HeaderTuple[] = Object.entries(headers).map(([name, value]) => [
-		name.toLowerCase(),
-		value,
-	]);
-	assertCallerHeadersSupported(entries);
-	return entries;
-}
-
-function assertCallerHeadersSupported(entries: readonly HeaderTuple[]): void {
-	for (const [name] of entries) {
-		if (isStealthOwnedHeaderName(name)) {
-			throw new SDKError(`Stealth transport owns the "${name}" header; remove it from headers.`, {
-				code: "STEALTH_HEADER_OVERRIDE_UNSUPPORTED",
-			});
-		}
-	}
+	assertCallerHeadersSupported(headers);
+	return Object.entries(headers).map(([name, value]) => [name.toLowerCase(), value]);
 }
 
 function chromeRequestClass(
@@ -621,9 +670,10 @@ function buildChromeHeaderTuples(options: {
 		),
 	);
 	const requestClass = chromeRequestClass(options.method, options.requestClass);
+	const shape = CHROME_REQUEST_CLASSES[requestClass];
 	const referer = caller.get("referer");
 	const fetchSite = secFetchSite(options.requestUrl, referer);
-	const isNavigation = requestClass === "navigation";
+	const isNavigation = shape.fetchMetadata === "navigate";
 	const values = new Map<string, string>([
 		["sec-ch-ua", requiredEmulationHeader(emulation, "sec-ch-ua")],
 		["sec-ch-ua-mobile", requiredEmulationHeader(emulation, "sec-ch-ua-mobile")],
@@ -649,12 +699,16 @@ function buildChromeHeaderTuples(options: {
 		values.set("upgrade-insecure-requests", "1");
 		if (options.userActivation !== false) values.set("sec-fetch-user", "?1");
 	}
+	if (shape.validateCache) values.set("cache-control", "max-age=0");
 	for (const [name, value] of callerEntries) values.set(name, value);
 	for (const name of ["content-type", "origin", "referer"] as const) {
 		const value = caller.get(name);
 		if (value !== undefined) values.set(name, value);
 	}
-	if (requestClass === "post") {
+	// A bodyful class keeps its class across a 301/302/303 that rewrites the method
+	// to GET (a form submission's redirect stays a validated navigation), but the
+	// rewritten request has no body and Chrome sends no Content-Length for it.
+	if (shape.bodyful && options.method !== "GET" && options.method !== "HEAD") {
 		values.set(
 			"content-length",
 			caller.get("content-length") ?? String(Buffer.byteLength(options.body ?? "")),
@@ -1194,10 +1248,30 @@ function locationHeader(headers: Record<string, string>): string | undefined {
 	return undefined;
 }
 
-function withoutRedirectBodyHeaders(headers: Record<string, string>): Record<string, string> {
-	return Object.fromEntries(
-		Object.entries(headers).filter(([name]) => !REDIRECT_BODY_HEADERS.has(name.toLowerCase())),
-	);
+/**
+ * Snapshot a caller's `redirectPolicy` before the first send. The policy acts
+ * on redirects the transport follows, so it requires the default
+ * `redirect: "follow"`.
+ */
+function snapshotStealthRedirectPolicy(options: StealthFetchOptions) {
+	let policy: ReturnType<typeof normalizeRedirectPolicy>;
+	try {
+		policy = normalizeRedirectPolicy(options.redirectPolicy, STEALTH_REDIRECT_POLICY_TRANSPORT);
+	} catch (error) {
+		if (error instanceof TransportError) throw error;
+		throw invalidRedirectPolicy(
+			STEALTH_REDIRECT_POLICY_TRANSPORT,
+			"could not be read safely",
+			error instanceof Error ? error : undefined,
+		);
+	}
+	if (policy && options.redirect !== undefined && options.redirect !== "follow") {
+		throw invalidRedirectPolicy(
+			STEALTH_REDIRECT_POLICY_TRANSPORT,
+			`it applies to followed redirects and cannot be combined with redirect: "${options.redirect}"`,
+		);
+	}
+	return policy;
 }
 
 function assertStealthRedirectUrl(url: string): void {
@@ -1318,7 +1392,9 @@ function startStealthExchange(
 				requestClass:
 					requestClass === "navigation" && options.stealth?.userActivation === false
 						? "script_navigation"
-						: requestClass,
+						: requestClass === "form-post"
+							? "form_post"
+							: requestClass,
 				...(status === undefined ? {} : { status }),
 				...(errorCode ? { errorCode } : {}),
 				...(error === undefined
@@ -1359,6 +1435,10 @@ async function fetchStealthRedirectChain(
 	let followedHops = 0;
 	let response: StealthTransportResponse;
 	const deadline = options.timeout ? performance.now() + options.timeout : undefined;
+	// Snapshotted and validated by session.fetch before the first send.
+	const redirectPolicy = options.redirectPolicy;
+	const initialOrigin = new URL(requestUrl).origin;
+	const visitedRequests = new Set([`${method} ${requestUrl}`]);
 
 	let finishExchange: ReturnType<typeof startStealthExchange> | undefined;
 	let status: number | undefined;
@@ -1415,28 +1495,48 @@ async function fetchStealthRedirectChain(
 				});
 			}
 
-			const nextUrl = resolveRedirectUrl(
-				response.headers.get("location") ?? undefined,
-				response.url ?? currentUrl,
-			);
-			if (!nextUrl) break;
-			if (followedHops >= MAX_STEALTH_REDIRECT_HOPS) {
+			let nextUrl: string;
+			let nextMethod: StealthMethod;
+			if (redirectPolicy) {
 				discardStealthRedirectBody(response);
-				throw new TransportError(
-					`Stealth request exceeded the ${MAX_STEALTH_REDIRECT_HOPS}-redirect limit`,
-					{ code: "transport_network_error", status: 0 },
+				({ nextUrl, nextMethod } = nextSameOriginRedirectHop({
+					policy: redirectPolicy,
+					initialOrigin,
+					currentUrl,
+					responseUrl: response.url ?? currentUrl,
+					status: response.status,
+					method: currentMethod,
+					location: response.headers.get("location") ?? undefined,
+					followedHops,
+					visitedRequests,
+				}));
+			} else {
+				const location = resolveRedirectUrl(
+					response.headers.get("location") ?? undefined,
+					response.url ?? currentUrl,
 				);
+				if (!location) break;
+				if (followedHops >= MAX_STEALTH_REDIRECT_HOPS) {
+					discardStealthRedirectBody(response);
+					throw new TransportError(
+						`Stealth request exceeded the ${MAX_STEALTH_REDIRECT_HOPS}-redirect limit`,
+						{ code: "transport_network_error", status: 0 },
+					);
+				}
+				discardStealthRedirectBody(response);
+				nextUrl = location;
+				nextMethod = nextRedirectMethod(response.status, currentMethod);
 			}
 			assertStealthRedirectUrl(nextUrl);
-			discardStealthRedirectBody(response);
-			const nextMethod = nextRedirectMethod(response.status, currentMethod);
-			if (nextMethod !== currentMethod) {
-				currentBody = undefined;
-				currentHeaders = withoutRedirectBodyHeaders(currentHeaders);
-			}
+			currentHeaders = redirectHopHeaders(
+				{ from: currentUrl, to: nextUrl, methodChanged: nextMethod !== currentMethod },
+				currentHeaders,
+			);
+			if (nextMethod !== currentMethod) currentBody = undefined;
 			currentMethod = nextMethod;
 			currentUrl = nextUrl;
 			followedHops += 1;
+			visitedRequests.add(`${currentMethod} ${currentUrl}`);
 			emitStealthTelemetry(telemetry?.sink, (sink) => sink.recordRedirectHop());
 			finishExchange(status);
 			finishExchange = undefined;
@@ -1558,10 +1658,15 @@ function createSessionFetcher(
 		let entry = clients.get(cacheKey);
 		if (!entry) {
 			entry = {
+				// The header shape only partitions sessions; it is never handed to wreq as
+				// session `defaultHeaders`. wreq merges those into every request that lacks
+				// one of their names, so a header the SDK left off on purpose — a credential
+				// dropped on a cross-origin redirect hop, the body headers after POST -> GET,
+				// a cookie for another host — would come back with the first request's value.
+				// Every request already sends its complete ordered header list.
 				session: wreq.createSession({
 					browser,
 					os,
-					...(defaultHeaders ? { defaultHeaders } : {}),
 					...(proxyUrl ? { proxy: proxyUrl } : {}),
 					...(ignoreTlsErrors ? { insecure: true } : {}),
 					timeout: 30_000,
@@ -1761,7 +1866,11 @@ function createSessionFetcher(
 			const { hasExplicitRetryPolicy, method, stealthRetryOptions } = (() => {
 				try {
 					const method = normalizeMethod(options.method ?? "GET");
-					normalizedCallerHeaderEntries(options.headers ?? {});
+					assertCallerHeadersSupported(options.headers ?? {});
+					// Replace the caller's object with the validated copy: the walker, a
+					// retry and an explicit replay all read this snapshot.
+					const redirectPolicy = snapshotStealthRedirectPolicy(options);
+					if (redirectPolicy) options.redirectPolicy = redirectPolicy;
 					const hasExplicitRetryPolicy = options.retry !== undefined;
 					const stealthRetryOptions =
 						normalizeProxyTransportRetryOptions(options.retry, {
@@ -2527,11 +2636,13 @@ function createSessionFetcher(
 					stopWhen,
 					params,
 					sensitiveParams,
+					headers: initialHeaders,
 					...fetchOptions
 				} = options;
 				const hops: StealthRedirectHop[] = [];
 				let method = normalizeMethod(options.method ?? "GET");
 				let body = options.body;
+				let headers = initialHeaders;
 				let response: StealthResponse | undefined;
 				const visitedRequests = new Set<string>();
 				const initialParams = params
@@ -2593,6 +2704,7 @@ function createSessionFetcher(
 					try {
 						response = await session.fetch(currentUrl, {
 							...fetchOptions,
+							...(headers ? { headers } : {}),
 							body,
 							method,
 							...(hopIndex === 0 && initialParams ? { params: initialParams } : {}),
@@ -2689,6 +2801,16 @@ function createSessionFetcher(
 							cookieStore: cookieJar.serialize(),
 						};
 					}
+					if (headers) {
+						headers = redirectHopHeaders(
+							{
+								from: currentUrl,
+								to: decision.nextUrl,
+								methodChanged: decision.nextMethod !== method,
+							},
+							headers,
+						);
+					}
 					if (decision.nextMethod !== method) {
 						body = undefined;
 					}

package/src/runtime/trace.ts

@@ -191,7 +191,11 @@ function normalizeAttributes(
 				? redactedKey === REDACTION_FAILED
 					? REDACTION_FAILED
 					: REDACTED_QUERY_VALUE
-				: normalizeAttributeValue(value, diagnosticAttributeRedactor(key, redact), retainBigints);
+				: normalizeAttributeValue(
+						value,
+						diagnosticAttributeRedactor(key, redact, value),
+						retainBigints,
+					);
 			return [normalizedKey, normalizedValue] as const;
 		})
 		.filter(
@@ -226,7 +230,7 @@ function prepareResourceAttributesForExport(
 					? redactedKey === REDACTION_FAILED
 						? REDACTION_FAILED
 						: REDACTED_QUERY_VALUE
-					: sanitizeSpanNameForOutput(value, diagnosticAttributeRedactor(key, redact)),
+					: sanitizeSpanNameForOutput(value, diagnosticAttributeRedactor(key, redact, value)),
 			];
 		}),
 	);

package/src/server/serve-implementation.ts

@@ -127,6 +127,7 @@ import {
 	tenantOpaqueKeys,
 } from "../runtime/request-telemetry.js";
 import type * as ResolverRuntimeModule from "../runtime/resolver.js";
+import { resolverSpanAttributes } from "../runtime/resolver-span-attributes.js";
 import {
 	bindResolverTelemetry,
 	createUnsupportedResolverClient,
@@ -702,7 +703,7 @@ function createStealthChallengeDetection(
 							const solve = () => resolver.solve(challenge, solveSignal, recorder ?? undefined);
 							return recorder
 								? recorder.runSpan("resolver.solve", solve, {
-										attributes: { challenge_kind: challenge.kind },
+										attributes: resolverSpanAttributes({ challenge_kind: challenge.kind }),
 									})
 								: solve();
 						},

package/src/testing/run.ts

@@ -10,6 +10,7 @@ import { createTestHandleContext } from "../runtime/handle.js";
  */
 const STANDARD_TEST_CONNECTION_ID = "standard-test-connection";
 import { createMemoryProviderRuntimeState } from "../runtime/state.js";
+import { assertCallerHeadersSupported } from "../runtime/stealth-owned-headers.js";
 import { createUnsupportedOcrClient } from "../runtime/ocr.js";
 import { createUnsupportedSttClient } from "../runtime/stt.js";
 import {
@@ -356,8 +357,19 @@ function createUpstreamContext(
 		options?: unknown,
 	): Promise<HttpResponse> =>
 		toHttpResponse(await dispatch({ transport: "http", method, url, body, options }));
-	const stealthCall = async (url: string, options?: { method?: string; body?: unknown }) =>
-		toStealthResponse(
+	// The production transport rejects owned headers (User-Agent, Sec-Fetch-*,
+	// sec-ch-ua*, ...) before any byte is sent; the fake applies the same
+	// function, so a provider test fails exactly where production would.
+	const stealthCall = async (
+		url: string,
+		options?: {
+			method?: string;
+			body?: unknown;
+			headers?: Readonly<Record<string, string | readonly string[] | undefined>>;
+		},
+	) => {
+		assertCallerHeadersSupported(options?.headers ?? {});
+		return toStealthResponse(
 			await dispatch({
 				transport: "stealth",
 				method: options?.method?.toUpperCase() ?? "GET",
@@ -367,6 +379,7 @@ function createUpstreamContext(
 			}),
 			url,
 		);
+	};
 
 	const createBrowserPage = (): BrowserPage => {
 		let currentUrl = "about:blank";

package/src/trace-sanitization.ts

@@ -52,7 +52,7 @@ export function sanitizeTraceAttributes(
 				const valueRedactor =
 					typeof value === "bigint" || isDiagnosticBigintAttribute(attributes, key)
 						? diagnosticStructuredRedactor(redact)
-						: diagnosticAttributeRedactor(key, redact);
+						: diagnosticAttributeRedactor(key, redact, value);
 				const sanitizedValue = keyChanged
 					? redactedKey === REDACTION_FAILED
 						? REDACTION_FAILED

package/src/types.ts

@@ -1414,7 +1414,23 @@ export interface StealthFetchOptions extends Omit<RequestOptions, "redirectPolic
 	headers?: Record<string, string | string[]>;
 	method?: HttpMethod;
 	body?: string | Buffer;
+	/**
+	 * Defaults to `"follow"`. Every followed hop drops the request-body headers
+	 * when the method changes, `Authorization` and `Proxy-Authorization` when it
+	 * leaves the current origin, and an explicit `Cookie` when it leaves the
+	 * current site (scheme plus registrable domain); after that the session
+	 * cookie jar supplies cookies for the new site.
+	 */
 	redirect?: "follow" | "manual" | "error";
+	/**
+	 * Opt-in same-origin redirect enforcement, the ctx.http `redirectPolicy`
+	 * for the stealth transport. A redirect whose target leaves the initial
+	 * origin, exceeds `maxHops` (at most 10, the stealth redirect limit),
+	 * loops, or has no usable `Location` throws `HttpRedirectError` before any
+	 * request is sent to that target. Requires the default
+	 * `redirect: "follow"`.
+	 */
+	redirectPolicy?: HttpRedirectPolicy;
 	/**
 	 * Maximum decoded response-body bytes to buffer. When set, the stealth
 	 * transport aborts the response and throws `response_too_large` if the
@@ -1440,13 +1456,21 @@ export interface StealthFetchOptions extends Omit<RequestOptions, "redirectPolic
 		 * replacement for hand-set `Sec-Fetch-*` headers, which the transport
 		 * rejects; it drives `Sec-Fetch-Mode`/`Dest`/`User`, `Accept`, `Priority`,
 		 * and `Upgrade-Insecure-Requests`.
+		 *
+		 * `form-post` is an HTML `<form method="post">` submission: a navigation
+		 * (`navigate`/`document`, the document `Accept`, `Upgrade-Insecure-Requests`)
+		 * that carries a body, in the header order Chrome uses for it, with
+		 * `Cache-Control: max-age=0` and the caller's `Content-Type`, `Origin` and
+		 * `Referer` in place. It is never inferred, since a form submission and a
+		 * script `fetch()` POST look the same to the transport.
 		 */
-		requestClass?: "navigation" | "xhr" | "post";
+		requestClass?: "navigation" | "form-post" | "xhr" | "post";
 		/**
-		 * Whether a `navigation` request carries a user gesture and therefore
-		 * `Sec-Fetch-User: ?1` (default `true`). Set `false` for script-driven
-		 * navigations such as `location.replace`, which Chrome sends without it.
-		 * Ignored for the `xhr` and `post` request classes.
+		 * Whether a `navigation` or `form-post` request carries a user gesture and
+		 * therefore `Sec-Fetch-User: ?1` (default `true`). Set `false` for
+		 * script-driven navigations such as `location.replace` or
+		 * `form.submit()`, which Chrome sends without it. Ignored for the `xhr` and
+		 * `post` request classes.
 		 */
 		userActivation?: boolean;
 		/**
@@ -1551,7 +1575,7 @@ export interface StealthRedirectHop {
 }
 
 export interface StealthRedirectRunOptions
-	extends Omit<StealthFetchOptions, "redirect"> {
+	extends Omit<StealthFetchOptions, "redirect" | "redirectPolicy"> {
 	url: string;
 	maxHops?: number;
 	stopWhen?: (hop: StealthRedirectHop) => boolean | Promise<boolean>;