@apifuse/provider-sdk 2.2.0-beta.66 → 2.2.0-beta.68

package/AUTHORING.md

@@ -680,6 +680,31 @@ The registered mappings are:
 | Unregistered input `ValidationError` code | 400 |
 | Other unregistered `ProviderError` code | 500 |
 
+A registered code that names an upstream outcome also carries a category. When
+a `ProviderError` (or any subclass, `ValidationError` included) with one of
+these codes sets no `category`, the server reports the code's category in
+`X-ApiFuse-Error-Observability`, and the public `source` follows from it:
+
+| Error code | Category when unset | `source` |
+| --- | --- | --- |
+| `NOT_FOUND`, `not_found`, `NO_DATA` | `upstream_http` | `upstream_failure` |
+| `RATE_LIMITED`, `UPSTREAM_RATE_LIMIT`, `LIMITED_NUMBER_OF_SERVICE_REQUESTS_EXCEEDS_ERROR` | `upstream_rate_limited` | `upstream_rule` |
+| `UPSTREAM_REJECTED` | `upstream_rejected` | `upstream_rule` |
+| `UPSTREAM_ERROR` | `upstream_http` | `upstream_failure` |
+| `BLOCKED` | `anti_bot_blocked` | `upstream_failure` |
+| `UPSTREAM_AUTH_ERROR` | `upstream_auth` | `upstream_failure` |
+| `UPSTREAM_SCHEMA_ERROR` | `upstream_schema_drift` | `upstream_failure` |
+
+So `throw new ProviderError("…", { code: "UPSTREAM_SCHEMA_ERROR" })` is enough
+for the health monitor to grade a drifted upstream `down`; without the
+category it would be served as `source: "apifuse"` and graded unknown. An
+explicit `category` always wins, an operation-declared rejection status
+(409/410/422) still classifies as `upstream_rejected`, and retryability does
+not follow the category: it stays the instance option, then the declaration,
+then `false`. Every other code, including `AUTH_REQUIRED`, `INVALID_REQUEST`,
+and your own codes, is classified as before (`provider_error` unless you set a
+category).
+
 An unregistered non-validation `ProviderError` code returns HTTP 500 and emits
 the greppable `unregistered_provider_error_code` signal with the code in the
 structured failure log. A matching operation declaration, including one that
@@ -701,7 +726,9 @@ SDK-owned and keep their 502/504 mapping.
   refused a *platform-managed* service key. The caller holds no credential, so
   401 ("re-authenticate") asks for something the caller cannot do, and 502
   ("the upstream is sick") promises a recovery that will never arrive. It is a
-  deployment/config defect, exactly like `MISSING_SECRET`.
+  deployment/config defect, exactly like `MISSING_SECRET`. The status is about
+  what the caller can do; the attribution is still the upstream's, since the
+  upstream is what refused the key, so the category is `upstream_auth`.
 - `UPSTREAM_SCHEMA_ERROR` is non-retryable: a retry returns the same payload
   the provider already could not normalize.
 - `INVALID_REQUEST` is the spelling for caller-side bad input. `INVALID_INPUT`,

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @apifuse/provider-sdk Changelog
 
+## 2.2.0-beta.68
+
+- Release candidate for main commit d630a344c18b2b192edfd09ef92bc30a482b6b8d.
+
+## 2.2.0-beta.67
+
+- Release candidate for main commit 9ef92b86b293253597801e29764d76beb0a3f3b7.
+
 ## 2.2.0-beta.66
 
 - Release candidate for main commit e7f93a6c71e3bca3a680861c0968e22dbe5d1d40.

package/README.md

@@ -309,10 +309,14 @@ apifuse bounty-check [path]
 apifuse perf providers/korea-air-quality --operation realtime --params '{"stationName":"종로구"}'
 ```
 
-`apifuse record` is for real upstream-backed operations that declare
-`upstream.baseUrl` and call the upstream through `ctx.http` or `ctx.stealth`. The
-generated local-only `ping` operation intentionally has no upstream and should
-be replaced before recording fixtures.
+`apifuse record` is for real upstream-backed operations that call the upstream
+through `ctx.http` or `ctx.stealth`. Absolute request URLs need no declaration; a
+relative URL resolves the way `serve` resolves it — against the operation's
+`upstream.baseUrl`, else the provider's first declared one (`ctx.stealth` then
+falls back to `https://<allowedHosts[0]>`). A relative `ctx.http` URL with no base
+is refused with the `upstream.baseUrl` to declare. The generated local-only `ping`
+operation intentionally has no upstream and should be replaced before recording
+fixtures.
 
 ## Bounty submission readiness
 

package/bin/apifuse-record.ts

@@ -8,6 +8,7 @@ import { pathToFileURL } from "node:url";
 
 import {
 	createBypassProviderCache,
+	createCredentialContext,
 	createHttpClient,
 	createInProcessProviderEngine,
 	createOcrClientFromEnv,
@@ -27,7 +28,16 @@ import {
 	TransportError,
 	ValidationError,
 } from "../src/index.js";
+import {
+	type LocalConnection,
+	LOCAL_CONNECTION_ID,
+	LOCAL_CONNECTION_SECRETS_ENV,
+	providerRequiresConnection,
+	resolveLocalConnection,
+} from "../src/cli/local-connection.js";
 import { createCliResolverRuntime } from "../src/cli/resolver-runtime.js";
+import { isAbsoluteUrl } from "../src/runtime/http.js";
+import { getProviderBaseUrl, getProviderStealthBaseUrl } from "../src/runtime/provider.js";
 import { assertProcessEngineModeSupported } from "../src/runtime/engine-mode.js";
 import type { JsonValue } from "../src/contract-json.js";
 import {
@@ -71,6 +81,10 @@ type CliArgs = {
 	operation?: string;
 	params: string;
 	sanitize: boolean;
+	/** Repeated `--credential key=value`, in order. */
+	credentials: string[];
+	/** `--no-credential`: run without a connection even when auth declares one. */
+	noCredential: boolean;
 };
 
 type ProviderRuntime = ProviderDefinition;
@@ -81,6 +95,11 @@ const HELP_TEXT = `Usage: apifuse record [path] --operation <operation> --params
 
 Calls a real upstream-backed operation through ctx.http or ctx.stealth and writes __fixtures__/raw.json.
 
+Absolute request URLs need no declaration. A relative URL resolves the way serve resolves it: against
+the operation's upstream.baseUrl, else the provider's first declared one (ctx.stealth then falls back
+to https://<allowedHosts[0]>). A relative ctx.http URL with no base anywhere is refused and names
+the upstream.baseUrl to declare.
+
 Streaming responses are recorded as evidence (status, selected headers, full-body SHA-256 and byte
 count, plus a ${STREAM_PREVIEW_BYTES}-byte base64 preview). Test replay is evidence-only: ctx.http.stream exposes the
 preview as its body and the original body_sha256/body_bytes as response metadata.
@@ -94,8 +113,18 @@ Options:
   --append                 preserve the existing fixture and append this capture
   --sanitize               redact common token/header fields (default)
   --no-sanitize            disable common-field redaction (sensitiveParams are always redacted)
+  --credential <k>=<v>     credential the gateway would inject (repeatable)
+  --no-credential          record without a credential even when auth declares one
   --help, -h               show this help
 
+Credentials:
+  A provider whose auth.mode is not "none" reads a credential the gateway
+  resolves and injects at invocation time -- for "platform-managed" that is the
+  platform-owned key, which is not a provider secret and is not in the
+  checkout. Supply it with --credential, or with
+  ${LOCAL_CONNECTION_SECRETS_ENV}='{"<key>":"<value>"}'. The keys are the ones
+  the provider passes to ctx.credential.get(...).
+
 Example:
   apifuse record providers/korea-air-quality --operation realtime --params '{"stationName":"jongno"}'`;
 
@@ -111,10 +140,13 @@ export async function main() {
 		const operation = provider.operations[operationName];
 		const parsedParams = await parseParams(operation, args.params);
 
+		const connection = resolveRecordConnection(provider, args);
+
 		capture = createCaptureContext(
 			provider,
-			resolveOperationBaseUrl(provider, operationName),
+			resolveRecordBaseUrls(provider, operationName),
 			args.sanitize,
+			connection,
 		);
 
 		console.log(`[apifuse record] Calling ${operationName} on ${provider.id}...`);
@@ -123,6 +155,7 @@ export async function main() {
 		try {
 			result = await executeOperation(provider, operationName, capture.ctx, parsedParams);
 		} catch (operationError) {
+			capture.throwIfRefused();
 			let partial: unknown;
 			try {
 				partial = await capture.getCapturedRaw();
@@ -147,6 +180,9 @@ export async function main() {
 			);
 			if (telemetryLine) console.log(telemetryLine);
 		}
+		// Checked on success too: an operation that caught the refusal and carried
+		// on would otherwise save a fixture missing the request it never made.
+		capture.throwIfRefused();
 		const captured = await capture.getCapturedRaw();
 
 		if (captured === undefined) {
@@ -181,6 +217,41 @@ export async function main() {
 	}
 }
 
+/**
+ * The connection this run attaches, or a hard failure when the provider needs
+ * one and none is configured.
+ *
+ * Failing here rather than at the handler is the whole point. Before this, a
+ * platform-managed provider recorded with `credential: { mode: "none" }`, so
+ * `ctx.credential.get(...)` returned `undefined` and the run died on the
+ * provider's own `MISSING_SECRET` — indistinguishable, to the author, from an
+ * upstream outage or a bad key. It reported as a capability the recorder
+ * lacked only if you already knew to look.
+ */
+function resolveRecordConnection(
+	provider: ProviderRuntime,
+	args: CliArgs,
+): LocalConnection | undefined {
+	if (args.noCredential) {
+		if (providerRequiresConnection(provider)) {
+			console.warn(
+				`[apifuse record] --no-credential: recording ${provider.id} without the credential its auth.mode "${provider.auth?.mode}" declares.`,
+			);
+		}
+		return undefined;
+	}
+
+	const resolution = resolveLocalConnection(provider, {
+		overrides: args.credentials,
+	});
+	if (!resolution.ok)
+		throw new ProviderError(resolution.reason, {
+			code: "CREDENTIAL_UNAVAILABLE",
+			fix: "Supply the credential with --credential or the environment variable named above.",
+		});
+	return resolution.connection;
+}
+
 function normalizeArgs(argv: string[]): string[] {
 	return argv[0] === "record" ? argv.slice(1) : argv;
 }
@@ -191,6 +262,8 @@ function parseArgs(argv: string[]): CliArgs {
 	let params = "{}";
 	let sanitize = true;
 	let append = false;
+	const credentials: string[] = [];
+	let noCredential = false;
 
 	for (let index = 0; index < argv.length; index += 1) {
 		const arg = argv[index];
@@ -247,6 +320,26 @@ function parseArgs(argv: string[]): CliArgs {
 			continue;
 		}
 
+		if (arg === "--credential") {
+			const value = argv[index + 1];
+			if (!value) {
+				throw new Error("Missing value for --credential.");
+			}
+			credentials.push(value);
+			index += 1;
+			continue;
+		}
+
+		if (arg.startsWith("--credential=")) {
+			credentials.push(arg.slice("--credential=".length));
+			continue;
+		}
+
+		if (arg === "--no-credential") {
+			noCredential = true;
+			continue;
+		}
+
 		if (arg.startsWith("-")) {
 			throw new Error(`Unknown option: ${arg}`);
 		}
@@ -259,7 +352,15 @@ function parseArgs(argv: string[]): CliArgs {
 		throw new Error(`Unexpected argument: ${arg}`);
 	}
 
-	return { append, providerPath, operation, params, sanitize };
+	return {
+		append,
+		providerPath,
+		operation,
+		params,
+		sanitize,
+		credentials,
+		noCredential,
+	};
 }
 
 /**
@@ -444,22 +545,58 @@ async function parseParams(
 	return operation.input ? parseSchema(operation.input, parsed, "record.params") : parsed;
 }
 
-function resolveOperationBaseUrl(provider: ProviderRuntime, operationName: string): string {
-	const baseUrl = provider.operations[operationName]?.upstream?.baseUrl;
-	if (!baseUrl) {
-		throw new Error(
-			`Operation "${operationName}" for provider "${provider.id}" must define upstream.baseUrl.`,
-		);
-	}
+/**
+ * The bases the recorder's `ctx.http` and `ctx.stealth` resolve relative
+ * request URLs against. An absent base means only absolute URLs resolve.
+ */
+export type RecordBaseUrls = {
+	readonly http?: string;
+	readonly stealth?: string;
+};
 
-	return baseUrl;
+/**
+ * `upstream.baseUrl` only resolves relative request URLs (and seeds the stealth
+ * cookie jar's default origin); the recorder uses it for nothing else, so an
+ * operation that requests absolute URLs records without one. The operation's
+ * own base wins; otherwise the recorder binds the bases `serve` binds, so a
+ * relative URL resolves in a recording exactly as it does in production.
+ */
+export function resolveRecordBaseUrls(
+	provider: ProviderRuntime,
+	operationName: string,
+): RecordBaseUrls {
+	const declared = provider.operations[operationName]?.upstream?.baseUrl;
+	const own = typeof declared === "string" && declared.length > 0 ? declared : undefined;
+	const http = own ?? getProviderBaseUrl(provider);
+	const stealth = own ?? getProviderStealthBaseUrl(provider);
+	return { ...(http ? { http } : {}), ...(stealth ? { stealth } : {}) };
+}
+
+/** A request the recorder cannot resolve without a declaration; fails the run even if the operation catches it. */
+class RecordRefusalError extends Error {
+	constructor(message: string) {
+		super(message);
+		this.name = "RecordRefusalError";
+	}
 }
 
 export function createCaptureContext(
 	provider: ProviderRuntime,
-	baseUrl: string,
+	baseUrls: RecordBaseUrls,
 	sanitize: boolean,
+	connection?: LocalConnection,
 ) {
+	let refusal: RecordRefusalError | undefined;
+	const refuse = (message: string): never => {
+		refusal ??= new RecordRefusalError(message);
+		throw refusal;
+	};
+	const assertResolvableHttpUrl = (url: string) => {
+		if (baseUrls.http || isAbsoluteUrl(url)) return;
+		refuse(
+			`ctx.http cannot resolve the relative URL "${requestPathForFixture(url)}": no operation of provider "${provider.id}" declares upstream.baseUrl. Declare upstream.baseUrl on the recorded operation, or request an absolute URL.`,
+		);
+	};
 	let nextCaptureOrder = 0;
 	let nextStreamOrdinal = 0;
 	let capturedRaw: JsonValue | undefined;
@@ -477,6 +614,25 @@ export function createCaptureContext(
 		provider,
 		cache,
 	);
+	// The injected credential is the most sensitive value in the run, and the
+	// recorder prints provider errors and writes fixtures. Seed its VALUES into
+	// the redaction set before anything can execute, so an upstream error that
+	// echoes the key, and any fixture that captured it, are redacted by the same
+	// machinery that already covers declared query secrets.
+	//
+	// Values only, never the key names: `sensitiveParamNames` means "declared
+	// query-key position", and `redactFixture` deliberately skips common-field
+	// sanitization for a key that appears in it. Seeding `access_token` there
+	// would therefore write a *newly issued* upstream `access_token` — which
+	// does not equal the injected value — to raw.json in plaintext. A credential
+	// key that really is a query parameter is added to the name set by
+	// `captureSensitiveRequestValues` when the provider sends it, which is the
+	// only place that claim is true.
+	for (const value of Object.values(connection?.secrets ?? {})) {
+		if (value.length > 0) sensitiveParamValues.add(value);
+	}
+	registerSensitiveValues([...sensitiveParamValues]);
+
 	const captureSensitiveParams = (url: string, options?: SensitiveRequestOptions) => {
 		try {
 			captureSensitiveRequestValues(url, options, sensitiveParamNames, sensitiveParamValues);
@@ -500,7 +656,8 @@ export function createCaptureContext(
 		rawCaptures.push({ order, value: json });
 	};
 
-	const http = captureHttpClient(createHttpClient(baseUrl), {
+	const http = captureHttpClient(createHttpClient(baseUrls.http), {
+		assertResolvableUrl: assertResolvableHttpUrl,
 		reserveOrder: reserveCaptureOrder,
 		reserveStreamOrdinal: () => {
 			nextStreamOrdinal += 1;
@@ -509,7 +666,7 @@ export function createCaptureContext(
 		onSensitiveParams: captureSensitiveParams,
 		onResponse: (order, response) => retainRawCapture(order, response.data),
 		onStreamResponse: (order, ordinal, requestUrl, method, response) => {
-			const resolvedRequestUrl = new URL(requestUrl, baseUrl).toString();
+			const resolvedRequestUrl = new URL(requestUrl, baseUrls.http).toString();
 			const request = {
 				ordinal,
 				method,
@@ -532,36 +689,46 @@ export function createCaptureContext(
 			capturedSse = {
 				order,
 				method,
-				path: requestPathForFixture(new URL(requestUrl, baseUrl).toString()),
+				path: requestPathForFixture(new URL(requestUrl, baseUrls.http).toString()),
 			};
 		},
 	});
-	const stealth = proxyStealthClient(
-		createStealthClient(baseUrl, {
-			...(provider.stealth ? { stealth: provider.stealth } : {}),
-		}),
-		captureSensitiveParams,
-		(order, response) => retainRawCapture(order, normalizeCapturedStealthResponse(response)),
-		reserveCaptureOrder,
-	);
+	// Without a base `serve` binds no stealth transport either, so every call is refused.
+	const refuseStealth = (): never =>
+		refuse(
+			`ctx.stealth has no base URL: provider "${provider.id}" declares neither upstream.baseUrl nor allowedHosts. Declare allowedHosts on the provider (serve binds ctx.stealth to https://<allowedHosts[0]>) or upstream.baseUrl on the recorded operation.`,
+		);
+	const stealth: StealthClient = baseUrls.stealth
+		? proxyStealthClient(
+				createStealthClient(baseUrls.stealth, {
+					...(provider.stealth ? { stealth: provider.stealth } : {}),
+				}),
+				captureSensitiveParams,
+				(order, response) => retainRawCapture(order, normalizeCapturedStealthResponse(response)),
+				reserveCaptureOrder,
+			)
+		: { fetch: async () => refuseStealth(), createSession: refuseStealth };
 
 	const providerEnvironment = createProviderEnvironment(
 		process.env,
 		provider.secrets?.map((secret) => secret.name) ?? [],
 	);
 	const env = { get: (key: string) => readDiagnosticEnv(key, providerEnvironment) };
-	const credential = {
-		mode: "none" as const,
-		get: () => undefined,
-		getAll: () => ({}),
-		getAccessToken: () => undefined,
-		getScopes: () => [],
-	};
+	// Built exactly the way `serve` builds it from a gateway-supplied
+	// `request.connection`, so the provider takes its production branch rather
+	// than the always-undefined one a stub credential forces.
+	const credential = createCredentialContext({
+		...(provider.credential?.keys ? { allowedKeys: provider.credential.keys } : {}),
+		...(connection ? { mode: connection.mode, values: connection.secrets } : {}),
+	});
 	const state = createMemoryProviderRuntimeState();
 	const candidates: ProviderEngineBindingCandidates = {
 		env,
 		credential,
-		request: { headers: {}, connectionId: "local-record" },
+		request: {
+			headers: {},
+			connectionId: connection ? LOCAL_CONNECTION_ID : "local-record",
+		},
 		http,
 		cache,
 		state,
@@ -595,7 +762,10 @@ export function createCaptureContext(
 		resolver,
 		handle: createHandleContext({
 			providerId: provider.id,
-			request: { headers: {}, connectionId: "local-record" },
+			request: {
+				headers: {},
+				connectionId: connection ? LOCAL_CONNECTION_ID : "local-record",
+			},
 			state,
 		}),
 	};
@@ -607,6 +777,9 @@ export function createCaptureContext(
 	return {
 		ctx,
 		resolverTelemetry,
+		throwIfRefused: () => {
+			if (refusal) throw refusal;
+		},
 		getCapturedRaw: async () => {
 			if (streamCaptures.length === 0) {
 				if (capturedSse) throw unsupportedSseCaptureError(capturedSse);
@@ -727,6 +900,8 @@ function snapshotRequestOptions<T extends SensitiveRequestOptions>(options: T):
 }
 
 type HttpCaptureCallbacks = {
+	/** Throws before the request when the recorder cannot resolve `url`. */
+	assertResolvableUrl(url: string): void;
 	reserveOrder(): number;
 	reserveStreamOrdinal(): number;
 	onSensitiveParams(url: string, options?: RequestOptions): void;
@@ -751,6 +926,7 @@ function captureHttpClient(client: HttpClient, callbacks: HttpCaptureCallbacks):
 		return response;
 	};
 	const captureRequestOptions = (method: PropertyKey, args: unknown[]) => {
+		callbacks.assertResolvableUrl(String(args[0]));
 		const invocation = parseHttpRequestInvocation(method, args);
 		const options = invocation ? requestOptionsFromHttpInvocation(invocation) : undefined;
 		if (!invocation || !options) return;