@apifuse/provider-sdk 2.2.0-beta.66 → 2.2.0-beta.68

package/src/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl

@@ -1,37 +1,142 @@
 ---
 name: health-checks-and-fail-closed
-description: Writing health checks that can actually fail, and fail-closed guards at envelope and row level. Load before writing healthCheck blocks or error handling.
+description: Writing health checks the platform monitor actually executes, and fail-closed guards at envelope and row level. Load before writing healthCheck blocks or error handling.
 ---
 
 # Health checks and fail-closed guards
 
-## Health checks that can actually fail
-`Array.isArray(data.items)` alone can never fail. Every list operation's
-health check must be able to detect the zero-rows regression.
+## Only `scenario` is monitored in production
 
-Assertion contract (per SDK `HealthCheckCase`): THROW to fail the case
-(recorded as `down`); return `{ status: "degraded", label }` to flag without
-failing; return nothing for `ok`. There is no `"down"` return value.
+A `healthCheck` case takes one of two mutually exclusive shapes, and they do
+not run in the same place:
+
+| field | executed by | when |
+|---|---|---|
+| `scenario` | the platform health monitor | every suite interval, in production |
+| `assertions` | the provider runtime's own self-test | only when that endpoint is called |
+
+The monitor runs a case's SERIALIZED scenario and nothing else: a closure
+cannot cross the registry's serialization boundary. A case that carries only
+`assertions` is still published as a probe, but its outcome is permanently
+`unknown` / `monitoring_unavailable` — the operation reads as monitored on the
+status page while nothing is ever checked. `apifuse check` warns about this
+(`health-check-assertions-not-monitored`).
+
+Anything that must be caught in production belongs in `scenario`. Keep
+`assertions` for a local self-test, or drop it.
+
+## A scenario that can actually fail
+
+`Array.isArray(data.items)` alone can never fail. Every list operation's probe
+must be able to detect the zero-rows regression AND a mapper regression that
+empties fields on a present row.
+
+Step kinds: `operation` invokes, `assert` FAILS the probe (recorded `down`),
+`guard` DEGRADES without failing it.
+
+```ts
+// health-scenarios.ts
+import { type AssertionExpression, defineHealthScenario } from "@apifuse/provider-sdk/provider";
+
+export const PROBE_INPUT = { district: "<a guaranteed-dense, stable id>" } as const;
+
+const response = (path: readonly (string | number)[]) => ({
+  ref: { namespace: "steps" as const, binding: "response", path: [...path] },
+});
+
+const evidence: AssertionExpression = {
+  kind: "all",
+  clauses: [
+    { kind: "predicate", operator: "status_2xx", actual: response(["status_code"]) },
+    // Dense query MUST return rows; zero rows = upstream contract drift.
+    { kind: "predicate", operator: "array_length_gte", actual: response(["data", "items"]), expected: 1 },
+    // A semantic field on a real row, so an emptying mapper regression fails too.
+    { kind: "predicate", operator: "non_empty", actual: response(["data", "items", 0, "name"]) },
+  ],
+};
+
+export const listItemsScenario = defineHealthScenario({
+  scenarioVersion: 2,
+  id: "list-items.dense-district",
+  display: {
+    titleKey: "health.operations.list-items.dense-district.title",
+    descriptionKey: "health.operations.list-items.dense-district.description",
+  },
+  schedule: { kind: "interval", intervalMs: 60 * 60_000, jitterMs: 0 },
+  timeoutMs: 15_000,
+  coversOperations: ["list-items"],
+  credentialRefs: [],
+  steps: [
+    { id: "read", result: "response", kind: "operation", operationId: "list-items", inputTemplate: { ...PROBE_INPUT } },
+    { id: "verify", result: "verified", kind: "assert", coversOperations: ["list-items"], expression: evidence },
+  ],
+});
+```
 
 ```ts
-assertions: ({ status, data }) => {
-  if (status !== 200) {
-    throw new Error(`<op> request failed with status ${status}`);
-  }
-  if (!Array.isArray(data.items)) {
-    throw new Error("<op> missing items array");
-  }
-  // Dense query MUST return rows; zero rows = upstream contract drift
-  if (data.items.length === 0) {
-    return { status: "degraded", label: "<op> dense query returned 0 rows" };
-  }
+// operations/list-items.ts
+healthCheck: {
+  interval: "1h",
+  timeoutMs: 15_000,
+  cases: [{ name: "dense-district", input: { ...PROBE_INPUT }, scenario: listItemsScenario }],
 }
 ```
 
-- Choose health-check inputs that are guaranteed-dense (major city district,
-  a stable well-known entity id). Verify the id still exists when picking it.
-- Also assert one semantic field on the first row (e.g. `items[0].name` is a
-  non-empty string) so a mapper regression that empties fields degrades too.
+- Choose probe inputs that are guaranteed-dense (major city district, a stable
+  well-known entity id). Verify the id still exists when picking it.
+- `scenario.coversOperations` may only name the operation that owns the case.
+
+## Stale-if-error serves must be guarded, or an outage reads green
+
+If the operation reads its upstream through `ctx.cache.getOrSet(..., {
+staleIfErrorMs })`, an upstream outage is answered with HTTP 200 and a
+well-formed, fully schema-valid body for the whole stale window. Every clause
+over `status_code` and `data` still passes, so the probe rolls up `ok` while
+the upstream is down, and only flips to `down` after the window expires — the
+first minutes of every outage are reported as healthy.
+
+`served_stale_cache` on the operation step result is the only operand that
+separates a stale serve from a live read. It is always present, so the
+reference resolves to `false` rather than raising `reference_unresolvable`.
+Add ONE guard step per stale-if-error probe:
+
+```ts
+{
+  id: "guard-list-items-freshness",
+  result: "freshness-guarded",
+  kind: "guard",
+  condition: {
+    kind: "predicate",
+    operator: "not_equals",
+    actual: { ref: { namespace: "steps" as const, binding: "response", path: ["served_stale_cache"] } },
+    expected: true,
+  },
+  onFail: {
+    attribute: [
+      {
+        operationId: "list-items",
+        status: "degraded",
+        reasonCode: "expected_absence",
+        reasonKey: "health.operations.list-items.dense-district.servedStaleCache",
+      },
+    ],
+    stop: "scenario",
+  },
+}
+```
+
+- `degraded`, not `down`: the cache policy is behaving as promised and callers
+  are still served; it is the upstream that is unavailable, and that has to be
+  visible instead of green.
+- `expected_absence` is the only `reasonCode` the guard attribution schema
+  admits today, and it reads correctly here — the FRESH read is what was
+  absent. The tenant-facing wording lives in the localized `reasonKey`.
+- Place the freshness guard BEFORE any row or emptiness guard. A stale serve
+  that happens to be empty would otherwise be attributed to the emptiness
+  reason, which is a confident answer to the wrong question.
+- Do not change the handler for this. The envelope's `meta.stale` is written
+  by the serve layer, so `return result.value` discarding `result.meta` does
+  not suppress the signal.
 
 ## Fail-closed: envelope level
 - Upstream error headers/codes → structured `ProviderError` with a stable
@@ -51,6 +156,14 @@ array is non-empty, but every row normalizes to nothing.
 - Regression-test both layers separately: a bad envelope AND a good envelope
   with unmappable rows.
 
+## Self-test assertions (optional, not monitored)
+A case without a `scenario` may carry an `assertions` lambda, executed only by
+the provider runtime's own self-test. THROW to fail the case (recorded
+`down`); return `{ status: "degraded", label }` to flag without failing;
+return nothing for `ok`. There is no `"down"` return value. It MUST NOT touch
+the scheduler, recorder, or any runtime type. Never use it as the only check
+on an operation you need monitored.
+
 ## Error message hygiene
 `ProviderError.message` reaches the tenant verbatim. Never interpolate
 upstream free text that may contain personal data (names, phone numbers,
@@ -58,8 +171,10 @@ addresses); allowlist known code tokens and keep raw bodies in server-side
 details/logs only.
 
 ## Checklist
-- [ ] Every list op health check flags 0 rows on a dense query
+- [ ] Every monitored operation's case carries `scenario` (not only `assertions`)
+- [ ] Every list op probe fails on 0 rows for a dense query
 - [ ] One semantic field asserted on a real row
+- [ ] Every `staleIfErrorMs` read has a `served_stale_cache` guard, placed first
 - [ ] Swallowed-error test exists (non-ok mock → handler rejects)
 - [ ] Non-empty upstream → zero normalized rows throws
 - [ ] No upstream free text in customer-facing error messages

package/src/cli/templates/provider/README.md.tpl

@@ -137,8 +137,8 @@ Structured errors return an `error` object with `code`, `message`,
 ## Next steps
 
 1. Replace the sample `ping` operation with real upstream logic.
-2. Once the real operation declares `upstream.baseUrl` and uses `ctx.http` or
-   `ctx.stealth`, record a fixture with:
+2. Once the real operation calls the upstream through `ctx.http` or `ctx.stealth`
+   (absolute URLs, or relative ones under its `upstream.baseUrl`), record a fixture with:
    `bun run record -- --operation <operation> --params '<json-input>'`.
 3. Replace the starter `healthCheckUnsupported` with a real `healthCheck` for read-only upstream operations when safe.
 4. Extend tests and operation metadata until the provider is bounty-ready.
@@ -158,8 +158,14 @@ Every operation must declare exactly one of:
 The generated `ping` operation uses `healthCheckUnsupported` only because it is
 a local scaffold check, not a real upstream API probe.
 
-`healthCheck.cases[].assertions` receives `{ data, status, durationMs, meta }`.
-`data` is the parsed operation output. Use this shape in real operations:
+A case carries either `scenario` or `assertions`, and only `scenario` is
+executed in production: the platform health monitor runs a case's serialized
+scenario and nothing else, because a closure cannot cross the registry's
+serialization boundary. A case with only `assertions` still publishes a probe,
+but its outcome is permanently `unknown` / `monitoring_unavailable` — the
+operation reads as monitored while nothing is checked, and `apifuse check`
+warns about it (`health-check-assertions-not-monitored`). Use this shape in
+real operations:
 
 ```ts
 healthCheck: {
@@ -167,11 +173,13 @@ healthCheck: {
   cases: [{
     name: "lookup baseline",
     input: { q: "btc" },
-    assertions: ({ data, status, durationMs }) => {
-      if (status !== 200 || data.results.length === 0 || durationMs > 3000) {
-        return { status: "degraded", label: "lookup baseline changed" };
-      }
-    },
+    scenario: lookupBaselineScenario, // defineHealthScenario({ ... })
   }],
 }
 ```
+
+If the operation reads through `ctx.cache.getOrSet(..., { staleIfErrorMs })`,
+add a guard step on the operation step result's `served_stale_cache` as well —
+otherwise an upstream outage is answered with a schema-valid 200 and the probe
+reports `ok` for the whole stale window. See
+`.agents/skills/health-checks-and-fail-closed/SKILL.md` for both shapes.

package/src/config/loader.ts

@@ -255,25 +255,62 @@ const SMARTPROXY_ALLOCATOR_RETRY_BASE_MS = 25;
 const SMARTPROXY_EXTRACTION_CACHE_TTL_MS = 15_000;
 const SMARTPROXY_EXTRACTION_SOFT_REFRESH_MS = 10_000;
 
+/** Env names that can supply a provider store's Redis URL. */
+export type ProviderRedisUrlEnvName =
+	| typeof PROVIDER_STATE_REDIS_URL_ENV
+	| typeof PROVIDER_CACHE_REDIS_URL_ENV
+	| typeof REDIS_URL_ENV;
+
+/** Which env name supplied a store's Redis URL, alongside the URL itself. */
+export type ProviderRedisUrlResolution = {
+	readonly url: string;
+	/** The env name that actually supplied `url`. */
+	readonly envName: ProviderRedisUrlEnvName;
+	/**
+	 * True when `url` came from a shared fallback env rather than the store's
+	 * own. A state store on `fallback: true` is sharing an instance that was
+	 * provisioned for something else (apifuse#2144).
+	 */
+	readonly fallback: boolean;
+};
+
+// Precedence is declared once here so the URL accessors, the boot report and
+// any future consumer can never disagree about which env wins.
+const CACHE_REDIS_URL_PRECEDENCE = [PROVIDER_CACHE_REDIS_URL_ENV, REDIS_URL_ENV] as const;
+const STATE_REDIS_URL_PRECEDENCE = [
+	PROVIDER_STATE_REDIS_URL_ENV,
+	PROVIDER_CACHE_REDIS_URL_ENV,
+	REDIS_URL_ENV,
+] as const;
+
+function resolveRedisUrlFromEnv(
+	precedence: readonly ProviderRedisUrlEnvName[],
+): ProviderRedisUrlResolution | undefined {
+	for (const [index, envName] of precedence.entries()) {
+		const url = readDiagnosticEnv(envName)?.trim();
+		if (url) return { url, envName, fallback: index > 0 };
+	}
+	return undefined;
+}
+
 function redisUrlFromEnv(): string | undefined {
-	return (
-		readDiagnosticEnv(PROVIDER_CACHE_REDIS_URL_ENV)?.trim() ||
-		readDiagnosticEnv(REDIS_URL_ENV)?.trim() ||
-		undefined
-	);
+	return providerCacheRedisResolutionFromEnv()?.url;
+}
+
+export function providerCacheRedisResolutionFromEnv(): ProviderRedisUrlResolution | undefined {
+	return resolveRedisUrlFromEnv(CACHE_REDIS_URL_PRECEDENCE);
+}
+
+export function providerStateRedisResolutionFromEnv(): ProviderRedisUrlResolution | undefined {
+	return resolveRedisUrlFromEnv(STATE_REDIS_URL_PRECEDENCE);
 }
 
 export function providerCacheRedisUrlFromEnv(): string | undefined {
-	return redisUrlFromEnv();
+	return providerCacheRedisResolutionFromEnv()?.url;
 }
 
 export function providerStateRedisUrlFromEnv(): string | undefined {
-	return (
-		readDiagnosticEnv(PROVIDER_STATE_REDIS_URL_ENV)?.trim() ||
-		readDiagnosticEnv(PROVIDER_CACHE_REDIS_URL_ENV)?.trim() ||
-		readDiagnosticEnv(REDIS_URL_ENV)?.trim() ||
-		undefined
-	);
+	return providerStateRedisResolutionFromEnv()?.url;
 }
 
 /** @internal Test-only hook for exercising shared proxy-cache behavior. */

package/src/error-resolution.ts

@@ -1,3 +1,4 @@
+import type { ProviderErrorCategory } from "./observability.js";
 import type { ProviderErrorStatus } from "./types.js";
 
 // This set suppresses the unregistered-provider-error-code signal for codes
@@ -120,11 +121,89 @@ export const SDK_RUNTIME_OWNED_ERROR_CODES = new Set([
 	"not_found",
 ]);
 
+/** One registered upstream code: the status the SDK serves and who failed. */
+export type SdkUpstreamErrorCodeRegistration = {
+	readonly status: ProviderErrorStatus;
+	readonly category: ProviderErrorCategory;
+};
+
+// Registered codes that name an upstream outcome. A row states the status the
+// SDK serves and the observability category together, so an upstream code
+// cannot be registered without saying who failed. Serve reads the category
+// for a `ProviderError` that carries one of these codes and no explicit
+// `category`, and the public `source` follows from it (`sourceForCategory`).
+// Without a row the category defaults to `provider_error` → `source:
+// "apifuse"`, and the health monitor cannot tie a real upstream failure to the
+// upstream: it publishes `monitoring_unavailable` instead of `down`.
+//
+// The category is `categoryForStatus(status)` unless the code names a
+// narrower class than its status can: a schema change, a bot wall, an
+// upstream refusal of the platform-managed key (served 400, see below).
+// Every row is a code the fleet throws for something the upstream did or
+// said; caller-side and SDK-internal codes stay in the table below.
+export const SDK_UPSTREAM_ERROR_CODE_REGISTRY: ReadonlyMap<
+	string,
+	SdkUpstreamErrorCodeRegistration
+> = new Map<string, SdkUpstreamErrorCodeRegistration>([
+	// The SDK's own unknown-operation and unknown-route throws reuse the
+	// not-found spellings; they set an explicit category and stay APIFuse's.
+	["NOT_FOUND", { status: 404, category: "upstream_http" }],
+	["not_found", { status: 404, category: "upstream_http" }],
+	["NO_DATA", { status: 404, category: "upstream_http" }],
+	["RATE_LIMITED", { status: 429, category: "upstream_rate_limited" }],
+	["UPSTREAM_RATE_LIMIT", { status: 429, category: "upstream_rate_limited" }],
+	[
+		"LIMITED_NUMBER_OF_SERVICE_REQUESTS_EXCEEDS_ERROR",
+		{ status: 429, category: "upstream_rate_limited" },
+	],
+	// Deterministic upstream business refusal (honest-provider-error-
+	// contract): the upstream evaluated the request and said no under its
+	// own rules — a conflict with upstream state, never a 5xx.
+	["UPSTREAM_REJECTED", { status: 409, category: "upstream_rejected" }],
+	["UPSTREAM_ERROR", { status: 502, category: "upstream_http" }],
+	// The upstream's bot wall or request protection refused the request. The
+	// status says only "upstream"; the code says which kind.
+	["BLOCKED", { status: 502, category: "anti_bot_blocked" }],
+	// Fleet-consensus provider codes. These are thrown by provider code, not
+	// by the SDK, so they are registered here (status mapping) and not in
+	// SDK_RUNTIME_OWNED_ERROR_CODES: an operation-declared status still wins,
+	// exactly as it does for UPSTREAM_ERROR and BLOCKED above. Registering
+	// them stops the fleet from re-declaring the same rows on every
+	// operation, and stops an undeclared throw from being served as 500.
+	//
+	// A platform-managed upstream service key the upstream refuses. With
+	// `auth.mode: "platform-managed"` the caller holds no credential, so 401
+	// ("re-authenticate") tells the caller to do something it cannot do, and
+	// 502 ("upstream is sick") promises a recovery that will never come. It is
+	// a deployment/config defect — the same class as MISSING_SECRET below,
+	// which is already an explicit 400 for that reason. The upstream is still
+	// the one refusing, so the category is `upstream_auth`; the gateway
+	// records it as `provider_error` for every non-`required` connection mode
+	// so the refusal stays provider-alertable.
+	["UPSTREAM_AUTH_ERROR", { status: 400, category: "upstream_auth" }],
+	// The upstream changed its response shape and the provider cannot
+	// normalize it. 502 because the fault is upstream of us, non-retryable
+	// because a retry returns the same broken payload.
+	["UPSTREAM_SCHEMA_ERROR", { status: 502, category: "upstream_schema_drift" }],
+]);
+
+/**
+ * The category a registered upstream code carries when its `ProviderError`
+ * sets none. `undefined` for every other code, which keeps its existing
+ * default.
+ */
+export function registeredUpstreamErrorCategory(
+	code: string | undefined,
+): ProviderErrorCategory | undefined {
+	return code === undefined ? undefined : SDK_UPSTREAM_ERROR_CODE_REGISTRY.get(code)?.category;
+}
+
 // Canonical SDK status mapping for recognized provider-thrown error codes.
 // serve.ts toStatusCode consults this map (after operation-declared overrides
 // for non-SDK-owned codes), and the authoring lint treats these codes as
 // SDK-registered. Add new codes here instead of duplicating literals in
-// either consumer.
+// either consumer — an upstream-attributed code goes in
+// SDK_UPSTREAM_ERROR_CODE_REGISTRY above, which feeds this map.
 export const SDK_STATUS_MAPPED_PROVIDER_ERROR_CODES: ReadonlyMap<string, ProviderErrorStatus> =
 	new Map<string, ProviderErrorStatus>([
 		["AUTH_REQUIRED", 401],
@@ -132,16 +211,6 @@ export const SDK_STATUS_MAPPED_PROVIDER_ERROR_CODES: ReadonlyMap<string, Provide
 		// Unprovisioned declared secret: a deployment/config defect, never an
 		// upstream failure — explicit 400.
 		["MISSING_SECRET", 400],
-		["NOT_FOUND", 404],
-		["not_found", 404],
-		["NO_DATA", 404],
-		["RATE_LIMITED", 429],
-		["UPSTREAM_RATE_LIMIT", 429],
-		["LIMITED_NUMBER_OF_SERVICE_REQUESTS_EXCEEDS_ERROR", 429],
-		// Deterministic upstream business refusal (honest-provider-error-
-		// contract): the upstream evaluated the request and said no under its
-		// own rules — a conflict with upstream state, never a 5xx.
-		["UPSTREAM_REJECTED", 409],
 		// Only an unverifiable or foreign handle is the caller's: key and binding
 		// faults stay unmapped (500) because the engine host, not the caller, owns them.
 		["EGRESS_LEASE_INVALID", 409],
@@ -157,30 +226,11 @@ export const SDK_STATUS_MAPPED_PROVIDER_ERROR_CODES: ReadonlyMap<string, Provide
 		["HANDLE_EXPIRED", 410],
 		["HANDLE_COMMITTED", 409],
 		["HANDLE_BUSY", 409],
-		["UPSTREAM_ERROR", 502],
-		["BLOCKED", 502],
-		// Fleet-consensus provider codes. These are thrown by provider code, not
-		// by the SDK, so they are registered here (status mapping) and not in
-		// SDK_RUNTIME_OWNED_ERROR_CODES: an operation-declared status still wins,
-		// exactly as it does for UPSTREAM_ERROR and BLOCKED above. Registering
-		// them stops the fleet from re-declaring the same three rows on every
-		// operation, and stops an undeclared throw from being served as 500.
-		//
-		// A platform-managed upstream service key the upstream refuses. With
-		// `auth.mode: "platform-managed"` the caller holds no credential, so 401
-		// ("re-authenticate") tells the caller to do something it cannot do, and
-		// 502 ("upstream is sick") promises a recovery that will never come. It is
-		// a deployment/config defect — the same class as MISSING_SECRET above,
-		// which is already an explicit 400 for that reason.
-		["UPSTREAM_AUTH_ERROR", 400],
-		// The upstream changed its response shape and the provider cannot
-		// normalize it. 502 because the fault is upstream of us, non-retryable
-		// because a retry returns the same broken payload.
-		["UPSTREAM_SCHEMA_ERROR", 502],
-		// Caller-side bad input rejected by the provider. The minority spellings
-		// (INVALID_INPUT / VALIDATION_ERROR / BAD_REQUEST) are deliberately not
-		// registered: they migrate to this spelling on the contract track, and
-		// registering them here would freeze the divergence.
+		// Caller-side bad input rejected by the provider (fleet consensus, like
+		// UPSTREAM_AUTH_ERROR and UPSTREAM_SCHEMA_ERROR above). The minority
+		// spellings (INVALID_INPUT / VALIDATION_ERROR / BAD_REQUEST) are
+		// deliberately not registered: they migrate to this spelling on the
+		// contract track, and registering them here would freeze the divergence.
 		["INVALID_REQUEST", 400],
 		["OCR_UNAVAILABLE", 503],
 		["UNSUPPORTED_OCR_BACKEND", 503],
@@ -194,6 +244,10 @@ export const SDK_STATUS_MAPPED_PROVIDER_ERROR_CODES: ReadonlyMap<string, Provide
 		// retry can clear, and a retryable 5xx there would turn one bad rollout
 		// into gateway-driven load.
 		["PROVIDER_ENGINE_UNAVAILABLE", 503],
+		...Array.from(
+			SDK_UPSTREAM_ERROR_CODE_REGISTRY,
+			([code, registration]) => [code, registration.status] as const,
+		),
 	]);
 
 // Canonical retryability for the fleet-consensus codes registered above.