@apifuse/provider-sdk 2.2.0-beta.59 → 2.2.0-beta.60

package/src/schema.ts

@@ -105,6 +105,8 @@ export type SensitiveFieldKind =
 export interface SensitiveFieldOptions {
 	/**
 	 * Mark this schema as sensitive. Defaults to true for the helper presets.
+	 * An explicit `false` emits `x-apifuse-sensitive: false` (reviewed public
+	 * data, never redacted) and drops any sensitivity kind.
 	 */
 	sensitive?: boolean;
 	/**
@@ -208,10 +210,27 @@ export function field<TSchema extends ZodType>(
 			? schema.describe(options.description)
 			: schema;
 	const metadata = described.meta() ?? {};
+	const isSensitive = options.sensitive ?? true;
+	// A public declaration only rewrites the meta of the node it receives;
+	// z.toJSONSchema folds that over the inner leaf, so `false` on a wrapper
+	// around a sensitive() leaf would publish `false` next to the leaf's kind
+	// while collectSensitivePaths() still redacts. Refuse the contradiction.
+	if (!isSensitive && wrapsSensitiveDeclaration(described)) {
+		throw new TypeError(
+			"publicField()/field({ sensitive: false }) wraps a sensitive() declaration; apply it to the leaf, before .optional(), .nullable(), .default(), .pipe(), z.lazy() or similar wrappers.",
+		);
+	}
+	// Zod merges meta with the parent chain, so an inherited kind can only be
+	// cleared by writing undefined over it (dropped from JSON Schema output).
+	const kind = isSensitive
+		? (options.kind ?? Reflect.get(metadata, APIFUSE_SENSITIVE_KIND_META_KEY))
+		: undefined;
 	return described.meta({
 		...metadata,
-		...((options.sensitive ?? true) ? { [APIFUSE_SENSITIVE_META_KEY]: true } : {}),
-		...(options.kind ? { [APIFUSE_SENSITIVE_KIND_META_KEY]: options.kind } : {}),
+		[APIFUSE_SENSITIVE_META_KEY]: isSensitive,
+		...(kind !== undefined || APIFUSE_SENSITIVE_KIND_META_KEY in metadata
+			? { [APIFUSE_SENSITIVE_KIND_META_KEY]: kind }
+			: {}),
 	});
 }
 
@@ -222,6 +241,18 @@ export function sensitive<TSchema extends ZodType>(
 	return field(schema, { sensitive: true, kind });
 }
 
+/**
+ * Declare a field as reviewed public data (business, facility, product, or
+ * place names and contacts) so a key-based privacy classifier hit counts as
+ * declared instead of undeclared. Emits `x-apifuse-sensitive: false` on the
+ * JSON Schema leaf; nothing is redacted. Use `sensitive()` for anything that
+ * identifies a person. Apply it to the leaf, before `.optional()` or
+ * `.nullable()`; wrapping a `sensitive()` declaration throws at definition time.
+ */
+export function publicField<TSchema extends ZodType>(schema: TSchema): TSchema {
+	return field(schema, { sensitive: false });
+}
+
 function sensitiveString(
 	kind: SensitiveFieldKind,
 	description: string,
@@ -382,6 +413,36 @@ function readZodMetadata(schema: unknown): object | undefined {
 	return metadata && typeof metadata === "object" ? metadata : undefined;
 }
 
+// Same-path wrappers (optional, nullable, default, catch, readonly, pipe,
+// lazy, ...) share the JSON Schema leaf with the schema they wrap.
+function wrapsSensitiveDeclaration(schema: unknown, seen = new Set<unknown>()): boolean {
+	const def = readZodDef(schema);
+	if (!def || seen.has(schema)) return false;
+	seen.add(schema);
+	return sameLeafChildren(def).some(
+		(child) => isSensitiveSchema(child) || wrapsSensitiveDeclaration(child, seen),
+	);
+}
+
+function sameLeafChildren(def: object): unknown[] {
+	switch (Reflect.get(def, "type")) {
+		case "pipe":
+			return [Reflect.get(def, "in"), Reflect.get(def, "out")];
+		case "lazy": {
+			// A forward-reference getter can throw before its target is
+			// initialised; treat that as no declaration rather than failing.
+			const getter = Reflect.get(def, "getter");
+			try {
+				return typeof getter === "function" ? [getter()] : [];
+			} catch {
+				return [];
+			}
+		}
+		default:
+			return [Reflect.get(def, "innerType")];
+	}
+}
+
 function readZodDef(schema: unknown): object | undefined {
 	if (!schema || typeof schema !== "object") return undefined;
 	const def = Reflect.get(schema, "def") ?? Reflect.get(schema, "_def");

package/src/server/self-test-redaction.ts

@@ -1,8 +1,8 @@
 import type { ProviderDefinition } from "../types.js";
 import {
-	PROVIDER_RUNTIME_SELF_TEST_MASTER_SECRET_ENV,
-	PROVIDER_RUNTIME_SELF_TEST_MASTER_SECRET_PREVIOUS_ENV,
-} from "./self-test-token.js";
+	collectDiagnosticSensitiveValues,
+	SELF_TEST_SENSITIVE_SOURCE_KINDS,
+} from "./sensitive-values.js";
 
 export const SELF_TEST_REDACTED_PLACEHOLDER = "[REDACTED]";
 
@@ -31,39 +31,12 @@ export function collectSelfTestSensitiveValues(
 		credentialInputs?: Readonly<Record<string, string>>;
 	} = {},
 ): string[] {
-	const env = options.env ?? process.env;
-	const envNames = new Set<string>([
-		PROVIDER_RUNTIME_SELF_TEST_MASTER_SECRET_ENV,
-		PROVIDER_RUNTIME_SELF_TEST_MASTER_SECRET_PREVIOUS_ENV,
-	]);
-	for (const secret of provider.secrets ?? []) {
-		envNames.add(secret.name);
-	}
-	const healthProbe = provider.healthProbe ?? provider.healthMonitor;
-	for (const name of healthProbe?.requiredSecrets ?? []) {
-		envNames.add(name);
-	}
-	for (const name of Object.values(healthProbe?.credentialInputs ?? {})) {
-		envNames.add(name);
-	}
-
-	const values = new Set<string>();
-	for (const name of envNames) {
-		const value = env[name];
-		if (typeof value === "string" && value.length >= MIN_SENSITIVE_VALUE_LENGTH) {
-			values.add(value);
-		}
-	}
-	for (const value of Object.values(options.credentialInputs ?? {})) {
-		if (value.length >= MIN_SENSITIVE_VALUE_LENGTH) {
-			values.add(value);
-		}
-	}
-	return [...values];
-}
-
-function escapeRegExp(value: string): string {
-	return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	return collectDiagnosticSensitiveValues(provider, {
+		...options,
+		sourceKinds: SELF_TEST_SENSITIVE_SOURCE_KINDS,
+		minimumLength: MIN_SENSITIVE_VALUE_LENGTH,
+		includeResolvedEnvVariants: false,
+	});
 }
 
 /**
@@ -79,10 +52,7 @@ export function redactSelfTestText(
 	let redacted = text;
 	for (const value of sensitiveValues) {
 		if (value.length < MIN_SENSITIVE_VALUE_LENGTH) continue;
-		redacted = redacted.replace(
-			new RegExp(escapeRegExp(value), "g"),
-			SELF_TEST_REDACTED_PLACEHOLDER,
-		);
+		redacted = redacted.replaceAll(value, SELF_TEST_REDACTED_PLACEHOLDER);
 	}
 	for (const pattern of HEADER_SHAPED_SECRETS) {
 		redacted = redacted.replace(

package/src/server/sensitive-values.ts

@@ -0,0 +1,200 @@
+import {
+	resolvedStringVariants,
+	urlCredentialComponents,
+	otlpHeaderCredentialValues,
+} from "../runtime/diagnostic-env-values.js";
+import {
+	PROVIDER_CACHE_REDIS_URL_ENV,
+	PROVIDER_STATE_REDIS_URL_ENV,
+	REDIS_URL_ENV,
+} from "../config/loader.js";
+import {
+	ENGINE_OWNED_PROXY_CREDENTIAL_ENV_NAMES,
+	ENGINE_OWNED_RESOLVER_CREDENTIAL_ENV_NAMES,
+} from "../engine.js";
+import { APIFUSE__CACHE__KEY_PEPPER_ENV } from "../runtime/cache.js";
+import { PROVIDER_RUNTIME_CHOICE_TOKEN_MASTER_SECRET_ENV } from "../runtime/choice.js";
+import { APIFUSE__ENGINE__CEREMONY_LEASE_KEY } from "../runtime/egress-lease.js";
+import {
+	APIFUSE__OCR__API_KEY_ENV,
+	APIFUSE__OCR__CLOUDFLARE_API_TOKEN_ENV,
+} from "../runtime/ocr.js";
+import {
+	OTEL_EXPORTER_OTLP_HEADERS,
+	OTEL_EXPORTER_OTLP_TRACES_HEADERS,
+	OTEL_EXPORTER_OTLP_ENDPOINT,
+	OTEL_EXPORTER_OTLP_TRACES_ENDPOINT,
+} from "../runtime/otlp.js";
+import { APIFUSE__CDP_POOL__URL } from "../runtime/resolver-config.js";
+import { APIFUSE__STT__CLOUDFLARE_API_TOKEN_ENV } from "../runtime/stt.js";
+import type { ProviderDefinition } from "../types.js";
+import {
+	PROVIDER_RUNTIME_SELF_TEST_MASTER_SECRET_ENV,
+	PROVIDER_RUNTIME_SELF_TEST_MASTER_SECRET_PREVIOUS_ENV,
+} from "./self-test-token.js";
+
+export const SELF_TEST_SENSITIVE_SOURCE_KINDS = [
+	"selfTestMasterSecrets",
+	"providerSecrets",
+	"healthProbeSecrets",
+	"requestCredentials",
+] as const;
+
+/** Explicit source allowlist: no environment-name or payload-shape secret sweep. */
+export const DIAGNOSTIC_SENSITIVE_SOURCE_KINDS = [
+	...SELF_TEST_SENSITIVE_SOURCE_KINDS,
+	"engineProxyCredentials",
+	"engineSolverKeys",
+	"engineCeremonyLeaseCredentials",
+	"choiceMasterSecret",
+	"ocrCredentials",
+	"sttCredentials",
+	"cdpCredentials",
+	"statefulForwardingSecret",
+	"redisCredentials",
+	"otlpHeaders",
+	"cacheKeyPepper",
+	"sensitiveParams",
+] as const;
+export type DiagnosticSensitiveSourceKind = (typeof DIAGNOSTIC_SENSITIVE_SOURCE_KINDS)[number];
+
+/**
+ * Startup inventory; request-scoped readers additionally register live values on access.
+ */
+export function collectStaticDiagnosticSensitiveValues(
+	provider: ProviderDefinition,
+	options: {
+		env?: Readonly<Record<string, string | undefined>>;
+		statefulForwardingSecret?: string;
+	} = {},
+): string[] {
+	return collectDiagnosticSensitiveValues(provider, {
+		...options,
+		sourceKinds: DIAGNOSTIC_SENSITIVE_SOURCE_KINDS.filter(
+			(kind) => kind !== "requestCredentials" && kind !== "sensitiveParams",
+		),
+	});
+}
+
+/** Generalised self-test collector, also used to seed and enrich ordinary request scopes. */
+export function collectDiagnosticSensitiveValues(
+	provider: ProviderDefinition,
+	options: {
+		env?: Readonly<Record<string, string | undefined>>;
+		credentialInputs?: Readonly<Record<string, unknown>>;
+		connectionSecrets?: Readonly<Record<string, string | undefined>>;
+		sensitiveParams?: readonly string[];
+		statefulForwardingSecret?: string;
+		sourceKinds?: readonly DiagnosticSensitiveSourceKind[];
+		minimumLength?: number;
+		/** Preserve the legacy self-test response collector when false. */
+		includeResolvedEnvVariants?: boolean;
+	} = {},
+): string[] {
+	const env = options.env ?? process.env;
+	const envValues = (names: readonly string[]) => names.map((name) => env[name]);
+	const resolvedEnvValues = (names: readonly string[]) =>
+		envValues(names).flatMap((value) =>
+			value === undefined
+				? []
+				: options.includeResolvedEnvVariants === false
+					? [value]
+					: [value, value.trim()],
+		);
+	const healthProbe = provider.healthProbe ?? provider.healthMonitor;
+	const sources: Record<DiagnosticSensitiveSourceKind, () => readonly unknown[]> = {
+		selfTestMasterSecrets: () =>
+			resolvedEnvValues([
+				PROVIDER_RUNTIME_SELF_TEST_MASTER_SECRET_ENV,
+				PROVIDER_RUNTIME_SELF_TEST_MASTER_SECRET_PREVIOUS_ENV,
+			]),
+		providerSecrets: () => resolvedEnvValues((provider.secrets ?? []).map((secret) => secret.name)),
+		healthProbeSecrets: () =>
+			resolvedEnvValues([
+				...(healthProbe?.requiredSecrets ?? []),
+				...Object.values(healthProbe?.credentialInputs ?? {}),
+			]),
+		requestCredentials: () => [
+			...Object.values(options.credentialInputs ?? {}),
+			...Object.values(options.connectionSecrets ?? {}),
+		],
+		engineProxyCredentials: () => resolvedEnvValues(ENGINE_OWNED_PROXY_CREDENTIAL_ENV_NAMES),
+		// These are the resolver-config API keys after #251's engine credential projection.
+		engineSolverKeys: () => resolvedEnvValues(ENGINE_OWNED_RESOLVER_CREDENTIAL_ENV_NAMES),
+		engineCeremonyLeaseCredentials: () => resolvedEnvValues([APIFUSE__ENGINE__CEREMONY_LEASE_KEY]),
+		choiceMasterSecret: () => resolvedEnvValues([PROVIDER_RUNTIME_CHOICE_TOKEN_MASTER_SECRET_ENV]),
+		ocrCredentials: () =>
+			resolvedEnvValues([APIFUSE__OCR__CLOUDFLARE_API_TOKEN_ENV, APIFUSE__OCR__API_KEY_ENV]),
+		sttCredentials: () => resolvedEnvValues([APIFUSE__STT__CLOUDFLARE_API_TOKEN_ENV]),
+		cdpCredentials: () =>
+			resolvedStringVariants(env[APIFUSE__CDP_POOL__URL]).flatMap((url) => [
+				url,
+				...urlCredentialComponents(url, { includePathAndQuery: true }),
+			]),
+		statefulForwardingSecret: () => resolvedStringVariants(options.statefulForwardingSecret),
+		redisCredentials: () =>
+			[PROVIDER_CACHE_REDIS_URL_ENV, PROVIDER_STATE_REDIS_URL_ENV, REDIS_URL_ENV]
+				.flatMap((name) => resolvedStringVariants(env[name]))
+				.flatMap((url) => urlCredentialComponents(url, { includePathAndQuery: false })),
+		otlpHeaders: () => [
+			...[OTEL_EXPORTER_OTLP_TRACES_HEADERS, OTEL_EXPORTER_OTLP_HEADERS].flatMap((name) =>
+				env[name] === undefined ? [] : otlpHeaderCredentialValues(env[name]),
+			),
+			...[OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT].flatMap((name) =>
+				resolvedStringVariants(env[name]).flatMap((url) =>
+					urlCredentialComponents(url, { includePathAndQuery: false }),
+				),
+			),
+		],
+		cacheKeyPepper: () => resolvedEnvValues([APIFUSE__CACHE__KEY_PEPPER_ENV]),
+		sensitiveParams: () => options.sensitiveParams ?? [],
+	};
+	const values = new Set<string>();
+	for (const kind of options.sourceKinds ?? DIAGNOSTIC_SENSITIVE_SOURCE_KINDS) {
+		for (const value of sources[kind]()) {
+			if (typeof value === "string" && value.length >= (options.minimumLength ?? 1))
+				values.add(value);
+		}
+	}
+	return [...values];
+}
+
+const diagnosticEnvNames = new WeakMap<ProviderDefinition, Set<string>>();
+
+/** Derive the live-read allowlist from the same collector as the startup inventory. */
+export function createDiagnosticEnvObserver(
+	provider: ProviderDefinition,
+	register: (values: readonly string[]) => void,
+	has?: (value: string) => boolean,
+): (name: string, value: string) => void {
+	let names = diagnosticEnvNames.get(provider);
+	if (!names) {
+		names = new Set<string>();
+		const captured = names;
+		collectStaticDiagnosticSensitiveValues(provider, {
+			env: new Proxy(
+				{},
+				{
+					get: (_target, key) => {
+						if (typeof key === "string") captured.add(key);
+						return undefined;
+					},
+				},
+			),
+		});
+		diagnosticEnvNames.set(provider, names);
+	}
+	const seen = new Map<string, string>();
+	return (name, value) => {
+		if (!names.has(name) || seen.get(name) === value) return;
+		if (has?.(value) && has(value.trim())) return;
+		// Includes URL userinfo/CDP components and OTLP auth values, using the same rules.
+		register([
+			...resolvedStringVariants(value),
+			...collectStaticDiagnosticSensitiveValues(provider, {
+				env: { [name]: value },
+			}),
+		]);
+		seen.set(name, value);
+	};
+}