@apifuse/provider-sdk 2.2.0-beta.7 → 2.2.0-beta.70

package/src/runtime/cache-telemetry.ts

@@ -0,0 +1,247 @@
+import { Buffer } from "node:buffer";
+import type { ProviderCacheLookupMeta } from "../types.js";
+import {
+	closedEnum,
+	type ClosedEnum,
+	type SpanIndex,
+	type TelemetryContributor,
+} from "./request-telemetry.js";
+import { createTelemetryObserver } from "./http-telemetry-guard.js";
+
+export type CacheTelemetrySource = "memory" | "redis" | "loader";
+
+export type CacheTelemetrySample = {
+	key: string;
+	hit: boolean;
+	stale: boolean;
+	ageMs?: number;
+	source: CacheTelemetrySource;
+};
+
+export type CacheTelemetryLogPayload = {
+	telemetryFailed?: true;
+	lookups: number;
+	hits: number;
+	stale: number;
+	misses: number;
+	writes: number;
+	loaderErrors: number;
+	writeErrors: number;
+	source: { memory: number; redis: number; loader: number };
+	redisFallbacks: number;
+	redisMode: ClosedEnum<"not_configured" | "configured" | "degraded">;
+	/** Successful Redis commands, including reads, writes, and deletes; memory-only work counts zero. */
+	redisRoundTrips: number;
+	ageMs?: { p50: number; max: number };
+	samples: CacheTelemetrySample[];
+	dropped: number;
+};
+
+export type CacheTelemetryHeaderPayload = Omit<
+	CacheTelemetryLogPayload,
+	"samples" | "ageMs" | "telemetryFailed"
+> & {
+	ageP50?: number;
+	ageMax?: number;
+};
+
+export interface CacheTelemetrySink {
+	markTelemetryFailed?(): void;
+	setRedisConfigured?(configured: boolean): void;
+	recordLookup(meta: ProviderCacheLookupMeta): void;
+	recordWrite(): void;
+	recordLoaderError?(): void;
+	recordWriteError?(): void;
+	recordRedisFallback(): void;
+	recordRedisRoundTrip?(): void;
+}
+
+const cacheTelemetryBindings = new WeakMap<object, CacheTelemetrySink>();
+
+/** Internal construction binding; keeps sink plumbing off ProviderCacheOptions. */
+export function bindCacheTelemetry<T extends object>(
+	options: T,
+	sink: CacheTelemetrySink | undefined,
+): T {
+	if (sink) cacheTelemetryBindings.set(options, sink);
+	return options;
+}
+
+/** Internal lookup used only by the cache factory. */
+export function cacheTelemetryFor(options: object): CacheTelemetrySink | undefined {
+	return cacheTelemetryBindings.get(options);
+}
+
+export function guardCacheTelemetry(sink: CacheTelemetrySink | undefined): CacheTelemetrySink {
+	const observe = createTelemetryObserver(sink);
+	return {
+		setRedisConfigured: (configured) => observe(() => sink?.setRedisConfigured?.(configured)),
+		recordLookup: (meta) => observe(() => sink?.recordLookup(meta)),
+		recordWrite: () => observe(() => sink?.recordWrite()),
+		recordLoaderError: () => observe(() => sink?.recordLoaderError?.()),
+		recordWriteError: () => observe(() => sink?.recordWriteError?.()),
+		recordRedisFallback: () => observe(() => sink?.recordRedisFallback()),
+		recordRedisRoundTrip: () => observe(() => sink?.recordRedisRoundTrip?.()),
+	};
+}
+
+const MAX_SAMPLES = 24;
+const MAX_TEXT = 300;
+
+function integer(value: number): number {
+	return Number.isFinite(value)
+		? Math.min(Number.MAX_SAFE_INTEGER, Math.max(0, Math.floor(value)))
+		: Number.MAX_SAFE_INTEGER;
+}
+
+function add(a: number, b: number): number {
+	return Math.min(Number.MAX_SAFE_INTEGER, integer(a) + integer(b));
+}
+
+function boundedText(value: string, redact?: (text: string) => string): string {
+	let result: string;
+	try {
+		result = redact?.(value) ?? value;
+	} catch {
+		result = "[REDACTION_FAILED]";
+	}
+	const length = Math.min(result.length, MAX_TEXT);
+	const units = Buffer.alloc(length * 2);
+	for (let index = 0; index < length; index += 1)
+		units.writeUInt16LE(result.charCodeAt(index), index * 2);
+	return units.toString("utf16le");
+}
+
+export class CacheTelemetryCollector
+	implements
+		TelemetryContributor<CacheTelemetryLogPayload, CacheTelemetryHeaderPayload>,
+		CacheTelemetrySink
+{
+	readonly key = "cache" as const;
+	readonly #redact?: (text: string) => string;
+	#telemetryFailed = false;
+	#lookups = 0;
+	#hits = 0;
+	#stale = 0;
+	#misses = 0;
+	#writes = 0;
+	#loaderErrors = 0;
+	#writeErrors = 0;
+	#source = { memory: 0, redis: 0, loader: 0 };
+	#redisFallbacks = 0;
+	#configured: "not_configured" | "configured" | "degraded";
+	#redisRoundTrips = 0;
+	#ages: number[] = [];
+	#maxAge = 0;
+	#samples: CacheTelemetrySample[] = [];
+	#dropped = 0;
+
+	constructor(options: { redact?: (text: string) => string } = {}) {
+		this.#redact = options.redact;
+		this.#configured = "not_configured";
+	}
+	setRedisConfigured(configured: boolean): void {
+		if (configured && this.#configured === "not_configured") this.#configured = "configured";
+	}
+	markTelemetryFailed(): void {
+		this.#telemetryFailed = true;
+	}
+
+	recordLookup(meta: ProviderCacheLookupMeta): void {
+		if (!["memory", "redis", "loader"].includes(meta.source)) {
+			this.#telemetryFailed = true;
+			return;
+		}
+		this.#lookups = add(this.#lookups, 1);
+		if (meta.hit) this.#hits = add(this.#hits, 1);
+		else this.#misses = add(this.#misses, 1);
+		if (meta.stale) this.#stale = add(this.#stale, 1);
+		this.#source[meta.source] = add(this.#source[meta.source], 1);
+		if (meta.source === "redis" && this.#configured === "not_configured")
+			this.#configured = "configured";
+		if (meta.ageMs !== undefined) {
+			const age = integer(meta.ageMs);
+			if (this.#ages.length < MAX_SAMPLES) this.#ages.push(age);
+			this.#maxAge = Math.max(this.#maxAge, age);
+		}
+		if (this.#samples.length < MAX_SAMPLES) {
+			this.#samples.push({
+				key: boundedText(meta.key, this.#redact),
+				hit: meta.hit,
+				stale: meta.stale,
+				...(meta.ageMs === undefined ? {} : { ageMs: integer(meta.ageMs) }),
+				source: meta.source,
+			});
+		} else this.#dropped = add(this.#dropped, 1);
+	}
+
+	recordWrite(): void {
+		this.#writes = add(this.#writes, 1);
+	}
+	recordLoaderError(): void {
+		this.#loaderErrors = add(this.#loaderErrors, 1);
+	}
+	recordWriteError(): void {
+		this.#writeErrors = add(this.#writeErrors, 1);
+	}
+
+	recordRedisFallback(): void {
+		this.#redisFallbacks = add(this.#redisFallbacks, 1);
+		this.#configured = "degraded";
+	}
+
+	recordRedisRoundTrip(): void {
+		this.#redisRoundTrips = add(this.#redisRoundTrips, 1);
+	}
+
+	toLogPayload(_spans: SpanIndex): CacheTelemetryLogPayload | undefined {
+		if (
+			this.#lookups === 0 &&
+			this.#writes === 0 &&
+			this.#redisFallbacks === 0 &&
+			!this.#telemetryFailed
+		)
+			return undefined;
+		const sorted = [...this.#ages].sort((a, b) => a - b);
+		const p50 = sorted.length ? sorted[Math.floor((sorted.length - 1) / 2)]! : undefined;
+		return {
+			...(this.#telemetryFailed ? { telemetryFailed: true as const } : {}),
+			lookups: this.#lookups,
+			hits: this.#hits,
+			stale: this.#stale,
+			misses: this.#misses,
+			writes: this.#writes,
+			loaderErrors: this.#loaderErrors,
+			writeErrors: this.#writeErrors,
+			source: { ...this.#source },
+			redisFallbacks: this.#redisFallbacks,
+			redisMode: closedEnum(this.#configured),
+			redisRoundTrips: this.#redisRoundTrips,
+			...(p50 === undefined ? {} : { ageMs: { p50, max: this.#maxAge } }),
+			samples: this.#samples.map((sample) => ({ ...sample })),
+			dropped: this.#dropped,
+		};
+	}
+
+	toHeaderPayload(log: CacheTelemetryLogPayload): CacheTelemetryHeaderPayload {
+		return {
+			lookups: log.lookups,
+			hits: log.hits,
+			stale: log.stale,
+			misses: log.misses,
+			writes: log.writes,
+			loaderErrors: log.loaderErrors,
+			writeErrors: log.writeErrors,
+			source: {
+				memory: log.source.memory,
+				redis: log.source.redis,
+				loader: log.source.loader,
+			},
+			redisFallbacks: log.redisFallbacks,
+			redisMode: log.redisMode,
+			redisRoundTrips: log.redisRoundTrips,
+			...(log.ageMs ? { ageP50: log.ageMs.p50, ageMax: log.ageMs.max } : {}),
+			dropped: log.dropped,
+		};
+	}
+}

package/src/runtime/cache.ts

@@ -1,6 +1,8 @@
-import { createHash } from "node:crypto";
+import { readDiagnosticEnv } from "./diagnostic-env.js";
+import { createHash, createHmac } from "node:crypto";
 
 import { providerCacheRedisUrlFromEnv } from "../config/loader.js";
+import { ProviderError } from "../errors.js";
 import type {
 	ProviderCache,
 	ProviderCacheGetOrSetOptions,
@@ -9,6 +11,7 @@ import type {
 	ProviderCacheResponseMeta,
 	ProviderCacheResult,
 } from "../types.js";
+import { cacheTelemetryFor, guardCacheTelemetry } from "./cache-telemetry.js";
 import {
 	createProviderRedisClient,
 	ensureRedisReady,
@@ -43,9 +46,12 @@ export type ProviderCacheOptions = {
 	now?: () => number;
 };
 
+export const APIFUSE__CACHE__KEY_PEPPER_ENV = "APIFUSE__CACHE__KEY_PEPPER";
+
 const DEFAULT_PREFIX = "apifuse:provider-cache:v1";
 const DEFAULT_MEMORY_MAX_ENTRIES = 1_000;
 const DEFAULT_REDIS_TIMEOUT_MS = 150;
+const SECRET_SCOPED_KEY_MARKER = "[secret-scoped";
 const SECRET_FIELD_NAMES = new Set([
 	"authorization",
 	"cookie",
@@ -61,6 +67,7 @@ const SECRET_FIELD_NAMES = new Set([
 ]);
 
 const sharedBackends = new Map<string, SharedCacheBackend>();
+let warnedAboutUnpepperedSecretKeys = false;
 
 function backendKey(redisUrl: string | undefined): string {
 	return redisUrl ?? "memory";
@@ -107,19 +114,172 @@ function shouldRedactField(name: string, extra: Set<string>): boolean {
 	);
 }
 
-function normalizeKeyPart(value: unknown, extra: Set<string>): unknown {
+type NormalizedKeyPart = {
+	value: unknown;
+	secretScoped: boolean;
+};
+
+function unsupportedSecretValue(path: string, reason: string): never {
+	throw new ProviderError(`Secret cache-key values must be JSON-safe; ${reason} at ${path}.`, {
+		code: "CACHE_KEY_SECRET_VALUE_UNSUPPORTED",
+		fix: "Convert the secret cache-key selector to JSON-safe primitives, arrays, or plain objects.",
+	});
+}
+
+function assertJsonSafeSecretValue(
+	value: unknown,
+	reportedPath: string,
+	ancestors = new Set<object>(),
+): void {
+	if (value === undefined) return;
+	if (value === null || typeof value === "string" || typeof value === "boolean") return;
+	if (typeof value === "number") {
+		if (!Number.isFinite(value))
+			unsupportedSecretValue(reportedPath, "non-finite numbers are unsupported");
+		if (Object.is(value, -0)) unsupportedSecretValue(reportedPath, "negative zero is unsupported");
+		return;
+	}
+	if (typeof value !== "object") {
+		unsupportedSecretValue(reportedPath, `${typeof value} values are unsupported`);
+	}
+	if (ancestors.has(value)) unsupportedSecretValue(reportedPath, "cyclic values are unsupported");
+
+	ancestors.add(value);
+	try {
+		if (Array.isArray(value)) {
+			if (Object.getOwnPropertySymbols(value).length > 0) {
+				unsupportedSecretValue(reportedPath, "symbol-keyed array properties are unsupported");
+			}
+			const expectedNames = new Set(["length"]);
+			for (let index = 0; index < value.length; index += 1) {
+				const key = String(index);
+				expectedNames.add(key);
+				const descriptor = Object.getOwnPropertyDescriptor(value, key);
+				if (!descriptor) unsupportedSecretValue(reportedPath, "sparse arrays are unsupported");
+				if (!descriptor.enumerable || !("value" in descriptor)) {
+					unsupportedSecretValue(reportedPath, "array accessors are unsupported");
+				}
+				assertJsonSafeSecretValue(descriptor.value, reportedPath, ancestors);
+			}
+			if (Object.getOwnPropertyNames(value).some((name) => !expectedNames.has(name))) {
+				unsupportedSecretValue(reportedPath, "custom array properties are unsupported");
+			}
+			return;
+		}
+
+		const prototype = Object.getPrototypeOf(value);
+		if (prototype !== Object.prototype && prototype !== null) {
+			unsupportedSecretValue(reportedPath, "non-plain objects are unsupported");
+		}
+		if (Object.getOwnPropertySymbols(value).length > 0) {
+			unsupportedSecretValue(reportedPath, "symbol-keyed properties are unsupported");
+		}
+		for (const key of Object.getOwnPropertyNames(value)) {
+			const descriptor = Object.getOwnPropertyDescriptor(value, key);
+			if (!descriptor?.enumerable || !("value" in descriptor)) {
+				unsupportedSecretValue(
+					reportedPath,
+					"non-enumerable properties and accessors are unsupported",
+				);
+			}
+			assertJsonSafeSecretValue(descriptor.value, reportedPath, ancestors);
+		}
+	} finally {
+		ancestors.delete(value);
+	}
+}
+
+function containsUndefined(value: unknown): boolean {
+	if (value === undefined) return true;
+	if (Array.isArray(value)) return value.some(containsUndefined);
+	if (isRecord(value)) return Object.values(value).some(containsUndefined);
+	return false;
+}
+
+function tagSecretValue(value: unknown): unknown {
+	if (value === undefined) return ["undefined"];
+	if (value === null) return ["null"];
+	if (typeof value === "string") return ["string", value];
+	if (typeof value === "number") return ["number", value];
+	if (typeof value === "boolean") return ["boolean", value];
+	if (Array.isArray(value)) return ["array", value.map(tagSecretValue)];
+	return [
+		"object",
+		Object.entries(value as Record<string, unknown>).map(([key, entry]) => [
+			key,
+			tagSecretValue(entry),
+		]),
+	];
+}
+
+function serializeSecretValue(value: unknown): string {
+	if (!containsUndefined(value)) return JSON.stringify([value]);
+	return `undefined-v1:${JSON.stringify(tagSecretValue(value))}`;
+}
+
+function warnAboutUnpepperedSecretKey(): void {
+	if (warnedAboutUnpepperedSecretKeys) return;
+	warnedAboutUnpepperedSecretKeys = true;
+	console.warn(
+		JSON.stringify({
+			level: "warn",
+			event: "provider_cache_secret_key_unpeppered",
+			message: `Secret-bearing cache keys are using unkeyed SHA-256 because ${APIFUSE__CACHE__KEY_PEPPER_ENV} is not configured.`,
+		}),
+	);
+}
+
+function normalizeKeyPart(
+	value: unknown,
+	extra: Set<string>,
+	pepper: string | undefined,
+): NormalizedKeyPart {
 	if (Array.isArray(value)) {
-		return value.map((entry) => normalizeKeyPart(entry, extra));
+		const entries = value.map((entry) => normalizeKeyPart(entry, extra, pepper));
+		return {
+			value: entries.map((entry) => entry.value),
+			secretScoped: entries.some((entry) => entry.secretScoped),
+		};
 	}
 	if (isRecord(value)) {
-		const normalized: Record<string, unknown> = {};
+		const normalized: Record<string, unknown> = Object.create(null);
+		let secretScoped = false;
 		for (const key of Object.keys(value).sort()) {
-			if (shouldRedactField(key, extra)) continue;
-			normalized[key] = normalizeKeyPart(value[key], extra);
+			const part = shouldRedactField(key, extra)
+				? hashSecretValue(value[key], key, extra, pepper)
+				: normalizeKeyPart(value[key], extra, pepper);
+			normalized[key] = part.value;
+			secretScoped ||= part.secretScoped;
 		}
-		return normalized;
+		return { value: normalized, secretScoped };
 	}
-	return value;
+	return { value, secretScoped: false };
+}
+
+function hashSecretValue(
+	value: unknown,
+	fieldName: string,
+	extra: Set<string>,
+	pepper: string | undefined,
+): NormalizedKeyPart {
+	assertJsonSafeSecretValue(value, `${fieldName} (inside secret value)`);
+	const canonical = serializeSecretValue(normalizeKeyPart(value, extra, pepper).value);
+	if (pepper === undefined) {
+		warnAboutUnpepperedSecretKey();
+		const digest = createHash("sha256").update(canonical).digest("hex");
+		return { value: `sha256:${digest}`, secretScoped: true };
+	}
+	const digest = createHmac("sha256", pepper).update(canonical).digest("hex");
+	return { value: `hmac-sha256:${digest}`, secretScoped: true };
+}
+
+function metadataKeys(events: ProviderCacheLookupMeta[], secretScopedKeys: Set<string>): string[] {
+	let secretScopedIndex = 0;
+	return Array.from(new Set(events.map((event) => event.key))).map((key) => {
+		if (!secretScopedKeys.has(key)) return key;
+		secretScopedIndex += 1;
+		return `${SECRET_SCOPED_KEY_MARKER}#${secretScopedIndex}]`;
+	});
 }
 
 function stableHash(value: unknown): string {
@@ -198,13 +358,20 @@ async function withRedisFallback<T>(operation: () => Promise<T>): Promise<T | un
 
 export function createProviderCache(options: ProviderCacheOptions): ProviderCache {
 	const redisUrl = options.redisUrl ?? providerCacheRedisUrlFromEnv();
+	const configuredPepper = readDiagnosticEnv(APIFUSE__CACHE__KEY_PEPPER_ENV);
+	const pepper = configuredPepper && configuredPepper.length > 0 ? configuredPepper : undefined;
 	const backend = getSharedBackend(redisUrl);
 	const memoryMaxEntries = Math.max(1, options.memoryMaxEntries ?? DEFAULT_MEMORY_MAX_ENTRIES);
 	const now = options.now ?? Date.now;
 	const events: ProviderCacheLookupMeta[] = [];
+	const telemetry = guardCacheTelemetry(cacheTelemetryFor(options));
+	telemetry.setRedisConfigured?.(!!redisUrl);
+	let redisUnavailable = false;
+	const secretScopedKeys = new Set<string>();
 
 	function record(meta: ProviderCacheLookupMeta): void {
 		events.push(meta);
+		telemetry?.recordLookup(meta);
 	}
 
 	function sweepMemory(currentTime: number): void {
@@ -248,13 +415,22 @@ export function createProviderCache(options: ProviderCacheOptions): ProviderCach
 	} | null> {
 		const redis = backend.redis;
 		if (!redis || !(await ensureRedisReady(redis, DEFAULT_REDIS_TIMEOUT_MS))) {
+			if (redis) {
+				redisUnavailable = true;
+				telemetry?.recordRedisFallback();
+			}
 			return null;
 		}
 
 		const raw = await withRedisFallback(async () => {
 			return await redis.get(key);
 		});
-		if (typeof raw !== "string" && raw !== null) return null;
+		if (typeof raw !== "string" && raw !== null) {
+			redisUnavailable = true;
+			telemetry.recordRedisFallback();
+			return null;
+		}
+		telemetry?.recordRedisRoundTrip?.();
 
 		const envelope = safeParseEnvelope(raw);
 		if (!envelope) return null;
@@ -305,6 +481,7 @@ export function createProviderCache(options: ProviderCacheOptions): ProviderCach
 		value: T,
 		cacheOptions: ProviderCacheGetOrSetOptions,
 	): Promise<void> {
+		telemetry?.recordWrite();
 		const currentTime = now();
 		const freshTtlMs = jitteredTtlMs(cacheOptions.ttlMs, cacheOptions.jitterPct);
 		const staleIfErrorMs = cacheOptions.staleIfErrorMs ?? 0;
@@ -319,9 +496,21 @@ export function createProviderCache(options: ProviderCacheOptions): ProviderCach
 
 		const redis = backend.redis;
 		if (!redis || !(await ensureRedisReady(redis, DEFAULT_REDIS_TIMEOUT_MS))) {
+			if (redis) {
+				redisUnavailable = true;
+				telemetry?.recordRedisFallback();
+				telemetry?.recordWriteError?.();
+			}
 			return;
 		}
-		await withRedisFallback(() => redis.set(key, JSON.stringify(envelope), "PX", staleTtlMs));
+		const written = await withRedisFallback(() =>
+			redis.set(key, JSON.stringify(envelope), "PX", staleTtlMs),
+		);
+		if (written === undefined) {
+			redisUnavailable = true;
+			telemetry?.recordRedisFallback();
+			telemetry?.recordWriteError?.();
+		} else telemetry?.recordRedisRoundTrip?.();
 	}
 
 	async function loadAndStore<T>(
@@ -346,6 +535,13 @@ export function createProviderCache(options: ProviderCacheOptions): ProviderCach
 			if (staleCandidate?.meta.stale) {
 				return staleCandidate;
 			}
+			telemetry?.recordLookup({
+				key,
+				hit: false,
+				stale: false,
+				source: "loader",
+			});
+			telemetry?.recordLoaderError?.();
 			throw error;
 		}
 	}
@@ -353,25 +549,38 @@ export function createProviderCache(options: ProviderCacheOptions): ProviderCach
 	return {
 		key(namespace, parts, keyOptions?: ProviderCacheKeyOptions) {
 			const extra = new Set((keyOptions?.redactFields ?? []).map((field) => field.toLowerCase()));
-			const normalized = normalizeKeyPart(parts, extra);
-			return `${DEFAULT_PREFIX}:${options.providerId}:${namespace}:${stableHash(normalized)}`;
+			const normalized = normalizeKeyPart(parts, extra, pepper);
+			const key = `${DEFAULT_PREFIX}:${options.providerId}:${namespace}:${stableHash(normalized.value)}`;
+			if (normalized.secretScoped) secretScopedKeys.add(key);
+			return key;
 		},
 
 		async get<T = unknown>(key: string): Promise<ProviderCacheResult<T> | null> {
 			const result = await read<T>(key);
 			if (result) record(result.meta);
+			else
+				telemetry.recordLookup({
+					key,
+					hit: false,
+					stale: false,
+					source: backend.redis && !redisUnavailable ? "redis" : "memory",
+				});
 			return result;
 		},
 
 		set: write,
 
 		async delete(key: string): Promise<void> {
+			telemetry?.recordWrite();
 			backend.memory.delete(key);
 			const redis = backend.redis;
 			if (!redis || !(await ensureRedisReady(redis, DEFAULT_REDIS_TIMEOUT_MS))) {
+				if (redis) telemetry?.recordRedisFallback();
 				return;
 			}
-			await withRedisFallback(() => redis.del(key));
+			const deleted = await withRedisFallback(() => redis.del(key));
+			if (deleted === undefined) telemetry?.recordRedisFallback();
+			else telemetry?.recordRedisRoundTrip?.();
 		},
 
 		async getOrSet<T = unknown>(
@@ -387,10 +596,22 @@ export function createProviderCache(options: ProviderCacheOptions): ProviderCach
 
 			const existingInflight = backend.inflight.get(key);
 			if (existingInflight) {
-				const inflightResult = await existingInflight;
-				const result = resultWithValue<T>(inflightResult.value, inflightResult.meta);
-				record(result.meta);
-				return result;
+				try {
+					const inflightResult = await existingInflight;
+					const result = resultWithValue<T>(inflightResult.value, inflightResult.meta);
+					record(result.meta);
+					return result;
+				} catch (error) {
+					const meta: ProviderCacheLookupMeta = {
+						key,
+						hit: false,
+						stale: false,
+						source: "loader",
+					};
+					telemetry.recordLookup(meta);
+					telemetry?.recordLoaderError?.();
+					throw error;
+				}
 			}
 
 			const promise: Promise<ProviderCacheResult<unknown>> = loadAndStore(
@@ -413,23 +634,35 @@ export function createProviderCache(options: ProviderCacheOptions): ProviderCach
 			return {
 				hit: events.some((event) => event.hit),
 				stale: events.some((event) => event.stale),
-				keys: Array.from(new Set(events.map((event) => event.key))),
+				keys: metadataKeys(events, secretScopedKeys),
 				source: sourceSummary(events),
 			};
 		},
 	};
 }
 
+const bypassCaches = new WeakSet<ProviderCache>();
+
+/** Internal cache capability, separate from hit/miss metadata. */
+export function isProviderCacheBypassed(cache: ProviderCache): boolean {
+	return bypassCaches.has(cache);
+}
+
 export function createBypassProviderCache(
 	options: Pick<ProviderCacheOptions, "providerId">,
 ): ProviderCache {
+	const configuredPepper = readDiagnosticEnv(APIFUSE__CACHE__KEY_PEPPER_ENV);
+	const pepper = configuredPepper && configuredPepper.length > 0 ? configuredPepper : undefined;
 	const events: ProviderCacheLookupMeta[] = [];
+	const secretScopedKeys = new Set<string>();
 
-	return {
+	const cache: ProviderCache = {
 		key(namespace, parts, keyOptions?: ProviderCacheKeyOptions) {
 			const extra = new Set((keyOptions?.redactFields ?? []).map((field) => field.toLowerCase()));
-			const normalized = normalizeKeyPart(parts, extra);
-			return `${DEFAULT_PREFIX}:${options.providerId}:${namespace}:${stableHash(normalized)}`;
+			const normalized = normalizeKeyPart(parts, extra, pepper);
+			const key = `${DEFAULT_PREFIX}:${options.providerId}:${namespace}:${stableHash(normalized.value)}`;
+			if (normalized.secretScoped) secretScopedKeys.add(key);
+			return key;
 		},
 
 		async get<T = unknown>(_key: string): Promise<ProviderCacheResult<T> | null> {
@@ -464,11 +697,13 @@ export function createBypassProviderCache(
 			return {
 				hit: false,
 				stale: false,
-				keys: Array.from(new Set(events.map((event) => event.key))),
+				keys: metadataKeys(events, secretScopedKeys),
 				source: sourceSummary(events),
 			};
 		},
 	};
+	bypassCaches.add(cache);
+	return cache;
 }
 
 export function resetProviderCacheForTests(): void {
@@ -478,4 +713,5 @@ export function resetProviderCacheForTests(): void {
 		backend.redis?.disconnect();
 	}
 	sharedBackends.clear();
+	warnedAboutUnpepperedSecretKeys = false;
 }