@apifuse/provider-sdk 2.2.0-beta.58 → 2.2.0-beta.60

package/src/lint.ts

@@ -550,18 +550,29 @@ function uniqueFields(fields: string[]): string[] {
 	return Array.from(new Set(fields));
 }
 
-function isSensitiveSchema(schema: unknown): boolean {
-	if (!schema || typeof schema !== "object" || !("meta" in schema)) {
-		return false;
+function readSensitivityDeclaration(
+	schema: unknown,
+	seen = new Set<unknown>(),
+): boolean | undefined {
+	// Declarations sit on the leaf; same-path wrappers such as .optional(),
+	// .nullable(), .pipe() or z.lazy() carry no metadata of their own, so look
+	// through them.
+	if (!isSchema(schema) || seen.has(schema)) return undefined;
+	seen.add(schema);
+	const value = Reflect.get(getSchemaMetadata(schema), APIFUSE_SENSITIVE_META_KEY);
+	if (typeof value === "boolean") return value;
+	const def = getSchemaDef(schema);
+	const children =
+		def.type === "pipe"
+			? [def.in, def.out]
+			: def.type === "lazy" && typeof def.getter === "function"
+				? [def.getter()]
+				: [def.innerType];
+	for (const child of children) {
+		const declaration = readSensitivityDeclaration(child, seen);
+		if (declaration !== undefined) return declaration;
 	}
-	const meta = schema.meta;
-	if (typeof meta !== "function") return false;
-	const metadata = meta.call(schema);
-	return (
-		!!metadata &&
-		typeof metadata === "object" &&
-		Reflect.get(metadata, APIFUSE_SENSITIVE_META_KEY) === true
-	);
+	return undefined;
 }
 
 function getSchemaMetadata(schema: SchemaLike): Record<string, unknown> {
@@ -591,9 +602,24 @@ const SENSITIVE_FIELD_NAMES = new Set([
 	"paymenturl",
 ]);
 
+// Only phone-shaped keys can hold reviewed public data (facility phones); the
+// rest are credentials, which a public declaration must not launder.
+const PUBLIC_DECLARABLE_FIELD_NAMES = new Set(["phone", "phonenumber"]);
+
+function normalizeFieldName(name: string): string {
+	return name.toLowerCase().replace(/[-_\s]/g, "");
+}
+
 function isSensitiveFieldName(name: string): boolean {
-	const normalized = name.toLowerCase().replace(/[-_\s]/g, "");
-	return SENSITIVE_FIELD_NAMES.has(normalized);
+	return SENSITIVE_FIELD_NAMES.has(normalizeFieldName(name));
+}
+
+function isDeclaredSensitiveField(key: string, schema: unknown): boolean {
+	const declaration = readSensitivityDeclaration(schema);
+	return (
+		declaration === true ||
+		(declaration === false && PUBLIC_DECLARABLE_FIELD_NAMES.has(normalizeFieldName(key)))
+	);
 }
 
 function collectUnmarkedSensitiveFields(
@@ -608,7 +634,7 @@ function collectUnmarkedSensitiveFields(
 	const out: string[] = [];
 	for (const [key, child] of Object.entries(getObjectShape(schema))) {
 		const childPath = basePath ? `${basePath}.${key}` : key;
-		if (isSensitiveFieldName(key) && !isSensitiveSchema(child)) {
+		if (isSensitiveFieldName(key) && !isDeclaredSensitiveField(key, child)) {
 			out.push(childPath);
 		}
 		out.push(...collectUnmarkedSensitiveFields(child, childPath, seen));
@@ -1032,9 +1058,9 @@ function browserVersionLiteralMessage(finding: BrowserVersionLiteralFinding): st
 		case "profile":
 			return `Hardcoded stealth profile "${finding.literal}" pins a browser version and will rot. Select the browser and OS structurally, for example stealth: { browser: "chrome", os: "macos" }.`;
 		case "user-agent":
-			return `Hardcoded User-Agent browser version "${finding.literal}" can disagree with the stealth TLS fingerprint. Remove the literal and derive it from the structured profile, for example getStealthProfile({ browser: "chrome", os: "macos" }).userAgent.`;
+			return `Hardcoded User-Agent browser version "${finding.literal}" can disagree with the stealth TLS fingerprint. Remove the literal. ctx.stealth owns User-Agent and rejects a caller value with STEALTH_HEADER_OVERRIDE_UNSUPPORTED, so omit the header there and select the profile with stealth: { browser: "chrome", os: "macos" }. For ctx.http only, derive it from getStealthProfile({ browser: "chrome", os: "macos" }).userAgent.`;
 		case "sec-ch-ua":
-			return 'Hardcoded sec-ch-ua versions can disagree with the stealth TLS fingerprint. Remove the literal and let ctx.stealth generate client hints from stealth: { browser: "chrome", os: "macos" }; derive any explicit User-Agent with getStealthProfile({ browser: "chrome", os: "macos" }).userAgent.';
+			return 'Hardcoded sec-ch-ua versions can disagree with the stealth TLS fingerprint. Remove the literal and let ctx.stealth generate client hints from stealth: { browser: "chrome", os: "macos" }. ctx.stealth also owns User-Agent, so omit that header there as well; for ctx.http only, derive it from getStealthProfile({ browser: "chrome", os: "macos" }).userAgent.';
 	}
 }
 
@@ -1401,7 +1427,7 @@ export function lintOperation(op: {
 			rule: "sensitive-field-unmarked",
 			level: "warn",
 			field,
-			message: `Schema field "${field}" looks sensitive; mark it with fields.*(), field(..., { sensitive: true }), or sensitive(...).`,
+			message: `Schema field "${field}" looks sensitive; mark it with fields.*() or sensitive(...); publicField(...) is accepted only for reviewed public phone-shaped fields.`,
 		});
 	}
 
@@ -1410,7 +1436,7 @@ export function lintOperation(op: {
 			rule: "sensitive-field-unmarked",
 			level: "warn",
 			field,
-			message: `Schema field "${field}" looks sensitive; mark it with fields.*(), field(..., { sensitive: true }), or sensitive(...).`,
+			message: `Schema field "${field}" looks sensitive; mark it with fields.*() or sensitive(...); publicField(...) is accepted only for reviewed public phone-shaped fields.`,
 		});
 	}
 

package/src/provider.ts

@@ -133,9 +133,13 @@ export {
 	createProviderEnvironment,
 	createInProcessProviderEngine,
 	ENGINE_OWNED_PROXY_CREDENTIAL_ENV_NAMES,
+	ENGINE_OWNED_RESOLVER_CREDENTIAL_ENV_NAMES,
+	ENGINE_OWNED_RUNTIME_ENV_NAMES,
 	ENGINE_OWNED_TELEMETRY_ENV_NAMES,
 	isEngineOwnedEnvName,
 	isEngineOwnedProxyCredentialName,
+	isEngineOwnedResolverCredentialName,
+	isEngineOwnedRuntimeEnvName,
 	isEngineOwnedTelemetryEnvName,
 	PROVIDER_CAPABILITY_KEYS,
 	PROVIDER_ENGINE_PROTOCOL_VERSION,
@@ -164,6 +168,8 @@ export {
 	PROVIDER_RUNTIME_CHOICE_TOKEN_MASTER_SECRET_ENV,
 } from "./runtime/choice.js";
 export {
+	APIFUSE_CONTENT_PROVENANCE_META_KEY,
+	APIFUSE_CONTENT_TRUST_META_KEY,
 	APIFUSE_DESCRIPTION_KEY_META_KEY,
 	APIFUSE_REDACTION_MARKER,
 	APIFUSE_SENSITIVE_KIND_META_KEY,
@@ -173,11 +179,13 @@ export {
 	field,
 	fields,
 	isSensitiveSchema,
+	publicField,
 	redactPayload,
 	type SensitiveFieldKind,
 	type SensitiveFieldOptions,
 	type SensitivePath,
 	sensitive,
+	untrustedContent,
 	z,
 } from "./schema.js";
 export type {
@@ -261,6 +269,7 @@ export type {
 	ProviderResolvedFile,
 	ProviderRuntimeTarget,
 	ProviderRuntimeState,
+	ProviderStealthConfig,
 	ProviderStateDurationString,
 	ProviderStateNamespace,
 	ProviderSupportLevel,
@@ -273,6 +282,7 @@ export type {
 	StateNamespaceScope,
 	StateValue,
 	StateWriteOptions,
+	StealthChallengeClassification,
 } from "./types.js";
 export {
 	NativeEgressGrantExpiredError,

package/src/runtime/bounded-expiring-map.ts

@@ -0,0 +1,67 @@
+/**
+ * Map whose entries carry an expiry and whose size is capped.
+ *
+ * Expired entries are dropped when read and swept on every insert, so a
+ * process that keeps minting new keys does not retain dead values. When the
+ * cap is still exceeded after the sweep, the least recently used entry is
+ * evicted and counted in `capacityEvictions` so callers can tell the two
+ * kinds of removal apart. Callers pass `now` explicitly so a single timestamp
+ * is used across a lookup-then-store sequence.
+ */
+export class BoundedExpiringMap<K, V> {
+	readonly #entries = new Map<K, V>();
+	readonly #maxEntries: number;
+	readonly #expiresAt: (value: V) => number;
+	#capacityEvictions = 0;
+
+	constructor(maxEntries: number, expiresAt: (value: V) => number) {
+		if (!Number.isInteger(maxEntries) || maxEntries <= 0) {
+			throw new RangeError("maxEntries must be a positive integer");
+		}
+		this.#maxEntries = maxEntries;
+		this.#expiresAt = expiresAt;
+	}
+
+	get size(): number {
+		return this.#entries.size;
+	}
+
+	/** Entries removed to satisfy `maxEntries`; expiry removals are not counted. */
+	get capacityEvictions(): number {
+		return this.#capacityEvictions;
+	}
+
+	get(key: K, now: number): V | undefined {
+		if (!this.#entries.has(key)) return undefined;
+		const value = this.#entries.get(key) as V;
+		this.#entries.delete(key);
+		if (this.#expiresAt(value) <= now) return undefined;
+		// Re-insert so Map iteration order doubles as the LRU order.
+		this.#entries.set(key, value);
+		return value;
+	}
+
+	set(key: K, value: V, now: number): void {
+		for (const [existingKey, existing] of this.#entries) {
+			if (this.#expiresAt(existing) <= now) this.#entries.delete(existingKey);
+		}
+		this.#entries.delete(key);
+		if (this.#expiresAt(value) <= now) return;
+		this.#entries.set(key, value);
+		if (this.#entries.size > this.#maxEntries) {
+			const oldest = this.#entries.keys().next();
+			if (!oldest.done) {
+				this.#entries.delete(oldest.value);
+				this.#capacityEvictions += 1;
+			}
+		}
+	}
+
+	delete(key: K): boolean {
+		return this.#entries.delete(key);
+	}
+
+	clear(): void {
+		this.#entries.clear();
+	}
+}

package/src/runtime/browser.ts

@@ -1,3 +1,4 @@
+import { readDiagnosticEnv } from "./diagnostic-env.js";
 import { createRequire } from "node:module";
 import type { CDPSession, Frame, LaunchOptions, Locator, Page, Request, Route } from "playwright";
 
@@ -495,7 +496,7 @@ type SupportedBrowserClient = {
 };
 
 function getDefaultCdpPoolUrl(env = process.env): string | undefined {
-	return env.APIFUSE__CDP_POOL__URL;
+	return readDiagnosticEnv("APIFUSE__CDP_POOL__URL", env);
 }
 
 async function importOptionalModule<T extends object>(moduleName: string): Promise<T> {

package/src/runtime/cache.ts

@@ -1,3 +1,4 @@
+import { readDiagnosticEnv } from "./diagnostic-env.js";
 import { createHash, createHmac } from "node:crypto";
 
 import { providerCacheRedisUrlFromEnv } from "../config/loader.js";
@@ -362,7 +363,7 @@ async function withRedisFallback<T>(operation: () => Promise<T>): Promise<T | un
 
 export function createProviderCache(options: ProviderCacheOptions): ProviderCache {
 	const redisUrl = options.redisUrl ?? providerCacheRedisUrlFromEnv();
-	const configuredPepper = process.env[APIFUSE__CACHE__KEY_PEPPER_ENV];
+	const configuredPepper = readDiagnosticEnv(APIFUSE__CACHE__KEY_PEPPER_ENV);
 	const pepper = configuredPepper && configuredPepper.length > 0 ? configuredPepper : undefined;
 	const backend = getSharedBackend(redisUrl);
 	const memoryMaxEntries = Math.max(1, options.memoryMaxEntries ?? DEFAULT_MEMORY_MAX_ENTRIES);
@@ -589,15 +590,22 @@ export function createProviderCache(options: ProviderCacheOptions): ProviderCach
 	};
 }
 
+const bypassCaches = new WeakSet<ProviderCache>();
+
+/** Internal cache capability, separate from hit/miss metadata. */
+export function isProviderCacheBypassed(cache: ProviderCache): boolean {
+	return bypassCaches.has(cache);
+}
+
 export function createBypassProviderCache(
 	options: Pick<ProviderCacheOptions, "providerId">,
 ): ProviderCache {
-	const configuredPepper = process.env[APIFUSE__CACHE__KEY_PEPPER_ENV];
+	const configuredPepper = readDiagnosticEnv(APIFUSE__CACHE__KEY_PEPPER_ENV);
 	const pepper = configuredPepper && configuredPepper.length > 0 ? configuredPepper : undefined;
 	const events: ProviderCacheLookupMeta[] = [];
 	const secretScopedKeys = new Set<string>();
 
-	return {
+	const cache: ProviderCache = {
 		key(namespace, parts, keyOptions?: ProviderCacheKeyOptions) {
 			const extra = new Set((keyOptions?.redactFields ?? []).map((field) => field.toLowerCase()));
 			const normalized = normalizeKeyPart(parts, extra, pepper);
@@ -643,6 +651,8 @@ export function createBypassProviderCache(
 			};
 		},
 	};
+	bypassCaches.add(cache);
+	return cache;
 }
 
 export function resetProviderCacheForTests(): void {

package/src/runtime/diagnostic-env-values.ts

@@ -0,0 +1,56 @@
+export function resolvedStringVariants(value: string | undefined): string[] {
+	if (value === undefined) return [];
+	return [...new Set([value, value.trim()])].filter((candidate) => candidate.length > 0);
+}
+
+function decodedComponent(value: string): string {
+	try {
+		return decodeURIComponent(value);
+	} catch {
+		return value;
+	}
+}
+
+export function urlCredentialComponents(
+	value: string,
+	options: { includePathAndQuery: boolean },
+): string[] {
+	let url: URL;
+	try {
+		url = new URL(value.trim());
+	} catch {
+		return [];
+	}
+	const username = decodedComponent(url.username);
+	const password = decodedComponent(url.password);
+	const components = [username, password, username && password ? `${username}:${password}` : ""];
+	if (options.includePathAndQuery) {
+		for (const segment of url.pathname.split("/")) {
+			if (segment) components.push(decodedComponent(segment));
+		}
+		for (const [key, entryValue] of url.searchParams) {
+			if (key) components.push(key);
+			if (entryValue) {
+				components.push(entryValue, `${key}=${entryValue}`);
+			}
+		}
+	}
+	return [...new Set(components.flatMap((component) => resolvedStringVariants(component)))];
+}
+
+export function otlpHeaderCredentialValues(value: string): string[] {
+	const variants = resolvedStringVariants(value);
+	for (const member of value.split(",")) {
+		const separator = member.indexOf("=");
+		if (separator <= 0) continue;
+		const headerValue = decodedComponent(member.slice(separator + 1).trim());
+		variants.push(...resolvedStringVariants(headerValue));
+		const headerName = decodedComponent(member.slice(0, separator).trim()).toLowerCase();
+		if (headerName === "authorization" || headerName === "proxy-authorization") {
+			// A transport may echo just the credential without its HTTP auth scheme.
+			const credential = /^(?:Bearer|Basic)\s+(.+)$/i.exec(headerValue)?.[1];
+			if (credential) variants.push(...resolvedStringVariants(credential));
+		}
+	}
+	return [...new Set(variants)];
+}

package/src/runtime/diagnostic-env.ts

@@ -0,0 +1,55 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+
+import {
+	resolvedStringVariants,
+	urlCredentialComponents,
+	otlpHeaderCredentialValues,
+} from "./diagnostic-env-values.js";
+import {
+	type CompiledDiagnosticSensitiveValues,
+	registerProcessDiagnosticValues,
+	suppressProcessDiagnosticValues,
+	withAppDiagnosticFallback,
+} from "./diagnostic-redactor.js";
+
+type EnvObserver = (name: string, value: string) => void;
+export interface DiagnosticEnvScope {
+	observe: EnvObserver;
+	finished: boolean;
+	staticValues?: CompiledDiagnosticSensitiveValues;
+}
+const observers = new AsyncLocalStorage<DiagnosticEnvScope>();
+
+/** Resolve and register atomically, before any consumer can use a live env credential. */
+export function readDiagnosticEnv(
+	name: string,
+	env: Readonly<Record<string, string | undefined>> = process.env,
+): string | undefined {
+	const value = env[name];
+	if (value !== undefined) {
+		const scope = observers.getStore();
+		if (scope && !scope.finished) scope.observe(name, value);
+		else {
+			// ALS survives the response. Finished requests register into the app's
+			// inherited fallback lifetime, never a dead request's private inventory.
+			if (Buffer.byteLength(value) > 64 * 1024) suppressProcessDiagnosticValues();
+			else
+				registerProcessDiagnosticValues([
+					...resolvedStringVariants(value),
+					...urlCredentialComponents(value, {
+						includePathAndQuery: name === "APIFUSE__CDP_POOL__URL",
+					}),
+					...(name === "OTEL_EXPORTER_OTLP_HEADERS" || name === "OTEL_EXPORTER_OTLP_TRACES_HEADERS"
+						? otlpHeaderCredentialValues(value)
+						: []),
+				]);
+		}
+	}
+	return value;
+}
+
+export function withDiagnosticEnv<T>(observer: EnvObserver | DiagnosticEnvScope, fn: () => T): T {
+	const scope = typeof observer === "function" ? { observe: observer, finished: false } : observer;
+	const run = () => observers.run(scope, fn);
+	return scope.staticValues ? withAppDiagnosticFallback(scope.staticValues, run) : run();
+}