@apifuse/provider-sdk 2.2.0-beta.67 → 2.2.0-beta.68

package/dist/server/serve-implementation.js

@@ -15,7 +15,7 @@ import { createEngineForMode, isUnavailableProviderEngine, resolveProviderEngine
 import { SDK_VERSION } from "./sdk-version.js";
 import { safeProviderErrorObservability } from "../error-observability.js";
 import { providerErrorStackFrames } from "./error-stack-frames.js";
-import { SDK_OWNED_PROVIDER_ERROR_CODES, SDK_RUNTIME_OWNED_ERROR_CODES, SDK_STATUS_MAPPED_PROVIDER_ERROR_CODES, } from "../error-resolution.js";
+import { registeredUpstreamErrorCategory, SDK_OWNED_PROVIDER_ERROR_CODES, SDK_RUNTIME_OWNED_ERROR_CODES, SDK_STATUS_MAPPED_PROVIDER_ERROR_CODES, } from "../error-resolution.js";
 import { AuthError, isProviderError, isSessionExpiredError, isTransportError, isValidationError, ProviderError, SDKError, } from "../errors.js";
 import { sanitizeDiagnosticText } from "../fixture-sanitization.js";
 import { loadProviderLocaleCatalogs, localizeAuthTurn, } from "../i18n/catalog.js";
@@ -37,7 +37,7 @@ import { HttpTelemetryCollector } from "../runtime/http-telemetry.js";
 import { wrapWithInstrumentation } from "../runtime/instrumentation.js";
 import { createOcrClientFromEnv } from "../runtime/ocr.js";
 import { bindOcrTelemetry, OcrTelemetryCollector } from "../runtime/ocr-telemetry.js";
-import { getProviderBaseUrl } from "../runtime/provider.js";
+import { getProviderBaseUrl, getProviderStealthBaseUrl } from "../runtime/provider.js";
 import { PROXY_AUTH_IP_DENIED_CODE, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_EXHAUSTED_CODE, } from "../runtime/proxy-errors.js";
 import { PROVIDER_TELEMETRY_HEADER, ProxyTelemetryCollector, } from "../runtime/proxy-telemetry.js";
 import { closedEnum, RequestTelemetry, tenantOpaqueKeys, } from "../runtime/request-telemetry.js";
@@ -62,6 +62,7 @@ import { getStealthProfile } from "../stealth/profiles.js";
 import { APIFUSE_STREAM_DONE_EVENT, APIFUSE_STREAM_ERROR_EVENT, encodeSseEvent, error as streamError, } from "../stream.js";
 import { sanitizeSpanNameForOutput, sanitizeTraceAttributes } from "../trace-sanitization.js";
 import { VALID_OPERATION_ERROR_STATUSES } from "../types.js";
+import { providerStateBackendLogEvent, } from "./state-backend-report.js";
 import { resolveSelfTestMasterSecrets } from "./self-test-token.js";
 import { collectStaticDiagnosticSensitiveValues, createDiagnosticEnvObserver, } from "./sensitive-values.js";
 import { resolveServerTraceContextOptions } from "./trace-output.js";
@@ -340,14 +341,6 @@ function bindResolverSignalWithoutRuntime(resolver, defaultSignal) {
     inheritResolverTelemetryBinding(resolver, boundResolver);
     return boundResolver;
 }
-function getProviderStealthBaseUrl(provider) {
-    const baseUrl = getProviderBaseUrl(provider);
-    if (baseUrl) {
-        return baseUrl;
-    }
-    const firstHost = provider.allowedHosts?.[0];
-    return firstHost ? `https://${firstHost}` : undefined;
-}
 function getProviderStealthProfile(provider) {
     return provider.stealth ? getStealthProfile(provider.stealth) : undefined;
 }
@@ -849,8 +842,9 @@ function zodDetails(error, redact) {
 // (credential_unavailable) usually means a caller credential problem, an
 // internal stateful-routing deadline is APIFuse-owned despite its timeout
 // category, and the built-in upstream failure families keep their upstream
-// attribution even when the author left the category at the provider_error
-// default.
+// attribution even under an explicit non-upstream category. An unset category
+// needs no override: a registered upstream code already carries its category
+// (SDK_UPSTREAM_ERROR_CODE_REGISTRY), and the source follows from it.
 function publicErrorSource(error, category) {
     if (error instanceof StatefulRoutingDeadlineError)
         return "apifuse";
@@ -1120,6 +1114,18 @@ function providerObservabilityDetails(error, declaredErrorCode) {
         ...(error.upstreamStatus ? { upstreamStatus: error.upstreamStatus } : {}),
     };
 }
+// The category a `ProviderError` (validation errors included) gets when its
+// author set none, ahead of the branch's own default: an operation-declared
+// rejection status (409/410/422) → upstream_rejected, then the registered
+// upstream code's category (UPSTREAM_SCHEMA_ERROR → upstream_schema_drift,
+// NOT_FOUND → upstream_http, ...). `undefined` leaves the branch default.
+function derivedErrorCategory(error, declaredStatus) {
+    if (isEmittableErrorStatus(declaredStatus) &&
+        categoryForStatus(declaredStatus) === "upstream_rejected") {
+        return "upstream_rejected";
+    }
+    return registeredUpstreamErrorCategory(providerErrorCode(error));
+}
 function classifiedErrorObservabilityDetails(error, declaredErrorCode) {
     const effectiveDeclaration = sdkOwnsErrorResolution(error) ? undefined : declaredErrorCode;
     const providerDetails = providerObservabilityDetails(error, effectiveDeclaration);
@@ -1131,12 +1137,10 @@ function classifiedErrorObservabilityDetails(error, declaredErrorCode) {
         return {
             category: providerCategory
                 ? providerCategory
-                : isEmittableErrorStatus(declaredStatus) &&
-                    categoryForStatus(declaredStatus) === "upstream_rejected"
-                    ? "upstream_rejected"
-                    : isEmittableErrorStatus(declaredStatus) && declaredStatus >= 500
+                : (derivedErrorCategory(error, declaredStatus) ??
+                    (isEmittableErrorStatus(declaredStatus) && declaredStatus >= 500
                         ? "provider_error"
-                        : "input_validation",
+                        : "input_validation")),
             taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
             retryable: isProviderError(error)
                 ? (providerErrorOption(error, "retryable") ?? effectiveDeclaration?.retryable ?? false)
@@ -1151,18 +1155,12 @@ function classifiedErrorObservabilityDetails(error, declaredErrorCode) {
         };
     }
     if (isProviderError(error)) {
-        // Deterministic upstream refusals default to the rejection category:
-        // the UPSTREAM_REJECTED family and any operation-declared rejection
-        // status (409/410/422) classify as upstream_rejected unless the
-        // author set an explicit category.
-        const declaredStatus = effectiveDeclaration?.status;
-        const rejectionDefault = providerErrorCode(error) === "UPSTREAM_REJECTED" ||
-            (isEmittableErrorStatus(declaredStatus) &&
-                categoryForStatus(declaredStatus) === "upstream_rejected")
-            ? "upstream_rejected"
-            : "provider_error";
+        // Retryability does not follow the derived category: it stays
+        // instance ?? declared ?? false below.
         return {
-            category: providerErrorOption(error, "category") ?? rejectionDefault,
+            category: providerErrorOption(error, "category") ??
+                derivedErrorCategory(error, effectiveDeclaration?.status) ??
+                "provider_error",
             taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
             retryable: providerErrorOption(error, "retryable") ?? effectiveDeclaration?.retryable ?? false,
         };
@@ -2560,7 +2558,13 @@ function createServerAppWithCapabilityModules(provider, serverOptions, capabilit
         });
     }
     app.notFound((c) => {
-        const error = new ProviderError("Not found", { code: "not_found", retryable: false });
+        // An unknown route is not an upstream answer: the explicit category keeps
+        // the registered not_found code from attributing it to an upstream.
+        const error = new ProviderError("Not found", {
+            code: "not_found",
+            category: "provider_error",
+            retryable: false,
+        });
         const observabilityDetails = errorObservabilityDetails(error);
         return responseWithErrorObservability(c.json(toErrorResponse(error, undefined, observabilityDetails, undefined, errorEnvelopeLocalization({
             catalogs: localeCatalogs,
@@ -2968,6 +2972,16 @@ export async function serve(provider, options = {}) {
     // One structured line per process boot, before the listener binds: the fleet
     // audit (ADR-0011 migration) counts these.
     logger(engineModeLogEvent(provider, engineSelection.resolution, !isUnavailableProviderEngine(engineSelection.engine)));
+    // Same contract for the stores: one line per process boot naming the state
+    // and cache backends this pod resolved, so "which Redis is this provider's
+    // state actually on" is a log grep instead of an inference from the
+    // Deployment spec (apifuse#2302).
+    logger(providerStateBackendLogEvent({
+        providerId: provider.id,
+        sdkVersion: SDK_VERSION,
+        injectedState: options.state !== undefined,
+        allowMemoryFallback: options.allowMemoryStateFallback === true,
+    }));
     const servers = [];
     try {
         servers.push(bunRuntime.serve({

package/dist/server/state-backend-report.d.ts

@@ -0,0 +1,70 @@
+/**
+ * What a provider store resolved to at boot.
+ *
+ * `endpoint` is deliberately host and port only: the URL can carry a password
+ * (`redis://:secret@host:6379`), and the diagnostic inventory already treats
+ * those credential components — and only those — as sensitive
+ * (`collectDiagnosticSensitiveValues`, `urlCredentialComponents(url, {
+ * includePathAndQuery: false })`). A URL that does not parse contributes no
+ * endpoint at all rather than a redacted-looking fragment of itself.
+ */
+export type ProviderStoreBackendReport = {
+    /**
+     * `redis` when a URL resolved, `memory` for the in-process fallback,
+     * `unsupported` when state has no backend and every call fails closed,
+     * `injected` when the host handed the server a store object.
+     */
+    readonly backend: "redis" | "memory" | "unsupported" | "injected";
+    /** Env name that supplied the Redis URL. Present only for `redis`. */
+    readonly envName?: string;
+    /**
+     * `host:port` of the resolved Redis. Never includes credentials, path or
+     * query. Absent when the URL does not parse, or when its query carries an
+     * ioredis option that moves the endpoint (`path`, `port`, `host`, `family`)
+     * so the authority is not where the client connects — see
+     * `state_redis_url_ambiguous_endpoint`.
+     */
+    readonly endpoint?: string;
+    /** URL scheme, e.g. `redis` or `rediss`. Present only for `redis`. */
+    readonly scheme?: string;
+    /** True when the URL came from a shared fallback env rather than the store's own. */
+    readonly fallback?: boolean;
+};
+/**
+ * One line per process boot naming the stores the provider resolved. Without
+ * it, a provider silently riding the cache Redis for its state is
+ * indistinguishable from one on a durable instance, from outside the pod and
+ * from the pod's own logs (apifuse#2302, the reason apifuse#2144 could only be
+ * verified by inference from the Deployment spec).
+ */
+export type ProviderStateBackendLogEvent = {
+    /** `warn` when a warning was raised, otherwise `info`. */
+    readonly level: "info" | "warn";
+    readonly event: "provider_state_backend";
+    readonly providerId: string;
+    readonly state: ProviderStoreBackendReport;
+    readonly cache: ProviderStoreBackendReport;
+    readonly sdkVersion: string;
+    /**
+     * Non-fatal findings, e.g. `state_redis_url_fallback` (the state store is
+     * sharing an instance provisioned for something else),
+     * `state_redis_url_unparsed`, or `state_redis_url_ambiguous_endpoint` (the
+     * URL's query carries an ioredis option that moves the endpoint, so the
+     * authority is not reported as the endpoint). Absent when the resolution
+     * was unambiguous.
+     */
+    readonly warnings?: readonly string[];
+};
+/**
+ * Build the boot event from the same env precedence the stores themselves use
+ * (`providerStateRedisResolutionFromEnv` / `providerCacheRedisResolutionFromEnv`),
+ * so the line can never describe a backend other than the one in use.
+ */
+export declare function providerStateBackendLogEvent(input: {
+    readonly providerId: string;
+    readonly sdkVersion: string;
+    /** The host passed an explicit state store, so no env resolution happened. */
+    readonly injectedState: boolean;
+    /** Mirrors `createProviderRuntimeStateFromEnv({ allowMemoryFallback })`. */
+    readonly allowMemoryFallback: boolean;
+}): ProviderStateBackendLogEvent;

package/dist/server/state-backend-report.js

@@ -0,0 +1,113 @@
+import { providerCacheRedisResolutionFromEnv, providerStateRedisResolutionFromEnv, } from "../config/loader.js";
+/**
+ * Whether ioredis' own URL parsing would move the endpoint away from the
+ * authority this function can see. `createProviderRedisClient` hands the raw
+ * URL to `new Redis(url, …)`, and `parseURL` ends with
+ * `defaults(result, queryOptions)` — lodash `defaults`, so a query key only
+ * applies where the authority supplied nothing:
+ *
+ * - `path` is never derived from the authority of a `redis://` URL (that
+ *   pathname is the db index), so `?path=` always applies — and
+ *   `StandaloneConnector` connects to `options.path` INSTEAD of host/port.
+ *   The client is then on a Unix socket the authority says nothing about.
+ * - `port` applies only when the URL carries no explicit port.
+ * - `host` never applies here: we return early unless `parsed.hostname` is
+ *   set, so `result.host` is always already populated.
+ * - `family` selects an address family; it does not move `host:port`.
+ *
+ * Verified against the pinned ioredis 5.11.1. In those cases the endpoint is
+ * WITHHELD and the ambiguity named, rather than reported as the authority — a
+ * diagnostic whose whole job is to answer "which Redis is this provider on"
+ * must not answer it wrongly. The query is never echoed: it can carry a
+ * password, which is why `endpoint` is host-and-port only to begin with.
+ */
+function queryMovesEndpoint(parsed) {
+    if (parsed.searchParams.has("path"))
+        return true;
+    return parsed.searchParams.has("port") && parsed.port === "";
+}
+function redisEndpoint(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return {};
+    }
+    // No authority means this is not the `redis://host:port` shape at all (e.g.
+    // a bare `host:port`, which parses as a scheme). Report nothing rather than
+    // a fragment of a string whose parts we have not identified.
+    if (!parsed.hostname)
+        return {};
+    const scheme = parsed.protocol.replace(/:$/, "");
+    if (queryMovesEndpoint(parsed))
+        return { scheme, ambiguous: true };
+    return {
+        scheme,
+        endpoint: parsed.port ? `${parsed.hostname}:${parsed.port}` : parsed.hostname,
+    };
+}
+function redisReport(resolution) {
+    const { endpoint, scheme, ambiguous } = redisEndpoint(resolution.url);
+    return {
+        parsed: endpoint !== undefined || ambiguous === true,
+        ambiguous: ambiguous === true,
+        report: {
+            backend: "redis",
+            envName: resolution.envName,
+            ...(endpoint === undefined ? {} : { endpoint }),
+            ...(scheme === undefined ? {} : { scheme }),
+            ...(resolution.fallback ? { fallback: true } : {}),
+        },
+    };
+}
+/**
+ * Build the boot event from the same env precedence the stores themselves use
+ * (`providerStateRedisResolutionFromEnv` / `providerCacheRedisResolutionFromEnv`),
+ * so the line can never describe a backend other than the one in use.
+ */
+export function providerStateBackendLogEvent(input) {
+    const warnings = [];
+    let state;
+    if (input.injectedState) {
+        state = { backend: "injected" };
+    }
+    else {
+        const resolution = providerStateRedisResolutionFromEnv();
+        if (resolution) {
+            const { report, parsed, ambiguous } = redisReport(resolution);
+            state = report;
+            if (!parsed)
+                warnings.push("state_redis_url_unparsed");
+            if (ambiguous)
+                warnings.push("state_redis_url_ambiguous_endpoint");
+            if (resolution.fallback)
+                warnings.push("state_redis_url_fallback");
+        }
+        else {
+            state = { backend: input.allowMemoryFallback ? "memory" : "unsupported" };
+        }
+    }
+    const cacheResolution = providerCacheRedisResolutionFromEnv();
+    let cache;
+    if (cacheResolution) {
+        const { report, parsed, ambiguous } = redisReport(cacheResolution);
+        cache = report;
+        if (!parsed)
+            warnings.push("cache_redis_url_unparsed");
+        if (ambiguous)
+            warnings.push("cache_redis_url_ambiguous_endpoint");
+    }
+    else {
+        cache = { backend: "memory" };
+    }
+    return {
+        level: warnings.length > 0 ? "warn" : "info",
+        event: "provider_state_backend",
+        providerId: input.providerId,
+        state,
+        cache,
+        sdkVersion: input.sdkVersion,
+        ...(warnings.length === 0 ? {} : { warnings }),
+    };
+}

package/package.json

@@ -1,5 +1,5 @@
 {
-	"version": "2.2.0-beta.67",
+	"version": "2.2.0-beta.68",
 	"name": "@apifuse/provider-sdk",
 	"private": false,
 	"type": "module",

package/src/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl

@@ -1,37 +1,142 @@
 ---
 name: health-checks-and-fail-closed
-description: Writing health checks that can actually fail, and fail-closed guards at envelope and row level. Load before writing healthCheck blocks or error handling.
+description: Writing health checks the platform monitor actually executes, and fail-closed guards at envelope and row level. Load before writing healthCheck blocks or error handling.
 ---
 
 # Health checks and fail-closed guards
 
-## Health checks that can actually fail
-`Array.isArray(data.items)` alone can never fail. Every list operation's
-health check must be able to detect the zero-rows regression.
+## Only `scenario` is monitored in production
 
-Assertion contract (per SDK `HealthCheckCase`): THROW to fail the case
-(recorded as `down`); return `{ status: "degraded", label }` to flag without
-failing; return nothing for `ok`. There is no `"down"` return value.
+A `healthCheck` case takes one of two mutually exclusive shapes, and they do
+not run in the same place:
+
+| field | executed by | when |
+|---|---|---|
+| `scenario` | the platform health monitor | every suite interval, in production |
+| `assertions` | the provider runtime's own self-test | only when that endpoint is called |
+
+The monitor runs a case's SERIALIZED scenario and nothing else: a closure
+cannot cross the registry's serialization boundary. A case that carries only
+`assertions` is still published as a probe, but its outcome is permanently
+`unknown` / `monitoring_unavailable` — the operation reads as monitored on the
+status page while nothing is ever checked. `apifuse check` warns about this
+(`health-check-assertions-not-monitored`).
+
+Anything that must be caught in production belongs in `scenario`. Keep
+`assertions` for a local self-test, or drop it.
+
+## A scenario that can actually fail
+
+`Array.isArray(data.items)` alone can never fail. Every list operation's probe
+must be able to detect the zero-rows regression AND a mapper regression that
+empties fields on a present row.
+
+Step kinds: `operation` invokes, `assert` FAILS the probe (recorded `down`),
+`guard` DEGRADES without failing it.
+
+```ts
+// health-scenarios.ts
+import { type AssertionExpression, defineHealthScenario } from "@apifuse/provider-sdk/provider";
+
+export const PROBE_INPUT = { district: "<a guaranteed-dense, stable id>" } as const;
+
+const response = (path: readonly (string | number)[]) => ({
+  ref: { namespace: "steps" as const, binding: "response", path: [...path] },
+});
+
+const evidence: AssertionExpression = {
+  kind: "all",
+  clauses: [
+    { kind: "predicate", operator: "status_2xx", actual: response(["status_code"]) },
+    // Dense query MUST return rows; zero rows = upstream contract drift.
+    { kind: "predicate", operator: "array_length_gte", actual: response(["data", "items"]), expected: 1 },
+    // A semantic field on a real row, so an emptying mapper regression fails too.
+    { kind: "predicate", operator: "non_empty", actual: response(["data", "items", 0, "name"]) },
+  ],
+};
+
+export const listItemsScenario = defineHealthScenario({
+  scenarioVersion: 2,
+  id: "list-items.dense-district",
+  display: {
+    titleKey: "health.operations.list-items.dense-district.title",
+    descriptionKey: "health.operations.list-items.dense-district.description",
+  },
+  schedule: { kind: "interval", intervalMs: 60 * 60_000, jitterMs: 0 },
+  timeoutMs: 15_000,
+  coversOperations: ["list-items"],
+  credentialRefs: [],
+  steps: [
+    { id: "read", result: "response", kind: "operation", operationId: "list-items", inputTemplate: { ...PROBE_INPUT } },
+    { id: "verify", result: "verified", kind: "assert", coversOperations: ["list-items"], expression: evidence },
+  ],
+});
+```
 
 ```ts
-assertions: ({ status, data }) => {
-  if (status !== 200) {
-    throw new Error(`<op> request failed with status ${status}`);
-  }
-  if (!Array.isArray(data.items)) {
-    throw new Error("<op> missing items array");
-  }
-  // Dense query MUST return rows; zero rows = upstream contract drift
-  if (data.items.length === 0) {
-    return { status: "degraded", label: "<op> dense query returned 0 rows" };
-  }
+// operations/list-items.ts
+healthCheck: {
+  interval: "1h",
+  timeoutMs: 15_000,
+  cases: [{ name: "dense-district", input: { ...PROBE_INPUT }, scenario: listItemsScenario }],
 }
 ```
 
-- Choose health-check inputs that are guaranteed-dense (major city district,
-  a stable well-known entity id). Verify the id still exists when picking it.
-- Also assert one semantic field on the first row (e.g. `items[0].name` is a
-  non-empty string) so a mapper regression that empties fields degrades too.
+- Choose probe inputs that are guaranteed-dense (major city district, a stable
+  well-known entity id). Verify the id still exists when picking it.
+- `scenario.coversOperations` may only name the operation that owns the case.
+
+## Stale-if-error serves must be guarded, or an outage reads green
+
+If the operation reads its upstream through `ctx.cache.getOrSet(..., {
+staleIfErrorMs })`, an upstream outage is answered with HTTP 200 and a
+well-formed, fully schema-valid body for the whole stale window. Every clause
+over `status_code` and `data` still passes, so the probe rolls up `ok` while
+the upstream is down, and only flips to `down` after the window expires — the
+first minutes of every outage are reported as healthy.
+
+`served_stale_cache` on the operation step result is the only operand that
+separates a stale serve from a live read. It is always present, so the
+reference resolves to `false` rather than raising `reference_unresolvable`.
+Add ONE guard step per stale-if-error probe:
+
+```ts
+{
+  id: "guard-list-items-freshness",
+  result: "freshness-guarded",
+  kind: "guard",
+  condition: {
+    kind: "predicate",
+    operator: "not_equals",
+    actual: { ref: { namespace: "steps" as const, binding: "response", path: ["served_stale_cache"] } },
+    expected: true,
+  },
+  onFail: {
+    attribute: [
+      {
+        operationId: "list-items",
+        status: "degraded",
+        reasonCode: "expected_absence",
+        reasonKey: "health.operations.list-items.dense-district.servedStaleCache",
+      },
+    ],
+    stop: "scenario",
+  },
+}
+```
+
+- `degraded`, not `down`: the cache policy is behaving as promised and callers
+  are still served; it is the upstream that is unavailable, and that has to be
+  visible instead of green.
+- `expected_absence` is the only `reasonCode` the guard attribution schema
+  admits today, and it reads correctly here — the FRESH read is what was
+  absent. The tenant-facing wording lives in the localized `reasonKey`.
+- Place the freshness guard BEFORE any row or emptiness guard. A stale serve
+  that happens to be empty would otherwise be attributed to the emptiness
+  reason, which is a confident answer to the wrong question.
+- Do not change the handler for this. The envelope's `meta.stale` is written
+  by the serve layer, so `return result.value` discarding `result.meta` does
+  not suppress the signal.
 
 ## Fail-closed: envelope level
 - Upstream error headers/codes → structured `ProviderError` with a stable
@@ -51,6 +156,14 @@ array is non-empty, but every row normalizes to nothing.
 - Regression-test both layers separately: a bad envelope AND a good envelope
   with unmappable rows.
 
+## Self-test assertions (optional, not monitored)
+A case without a `scenario` may carry an `assertions` lambda, executed only by
+the provider runtime's own self-test. THROW to fail the case (recorded
+`down`); return `{ status: "degraded", label }` to flag without failing;
+return nothing for `ok`. There is no `"down"` return value. It MUST NOT touch
+the scheduler, recorder, or any runtime type. Never use it as the only check
+on an operation you need monitored.
+
 ## Error message hygiene
 `ProviderError.message` reaches the tenant verbatim. Never interpolate
 upstream free text that may contain personal data (names, phone numbers,
@@ -58,8 +171,10 @@ addresses); allowlist known code tokens and keep raw bodies in server-side
 details/logs only.
 
 ## Checklist
-- [ ] Every list op health check flags 0 rows on a dense query
+- [ ] Every monitored operation's case carries `scenario` (not only `assertions`)
+- [ ] Every list op probe fails on 0 rows for a dense query
 - [ ] One semantic field asserted on a real row
+- [ ] Every `staleIfErrorMs` read has a `served_stale_cache` guard, placed first
 - [ ] Swallowed-error test exists (non-ok mock → handler rejects)
 - [ ] Non-empty upstream → zero normalized rows throws
 - [ ] No upstream free text in customer-facing error messages

package/src/cli/templates/provider/README.md.tpl

@@ -137,8 +137,8 @@ Structured errors return an `error` object with `code`, `message`,
 ## Next steps
 
 1. Replace the sample `ping` operation with real upstream logic.
-2. Once the real operation declares `upstream.baseUrl` and uses `ctx.http` or
-   `ctx.stealth`, record a fixture with:
+2. Once the real operation calls the upstream through `ctx.http` or `ctx.stealth`
+   (absolute URLs, or relative ones under its `upstream.baseUrl`), record a fixture with:
    `bun run record -- --operation <operation> --params '<json-input>'`.
 3. Replace the starter `healthCheckUnsupported` with a real `healthCheck` for read-only upstream operations when safe.
 4. Extend tests and operation metadata until the provider is bounty-ready.
@@ -158,8 +158,14 @@ Every operation must declare exactly one of:
 The generated `ping` operation uses `healthCheckUnsupported` only because it is
 a local scaffold check, not a real upstream API probe.
 
-`healthCheck.cases[].assertions` receives `{ data, status, durationMs, meta }`.
-`data` is the parsed operation output. Use this shape in real operations:
+A case carries either `scenario` or `assertions`, and only `scenario` is
+executed in production: the platform health monitor runs a case's serialized
+scenario and nothing else, because a closure cannot cross the registry's
+serialization boundary. A case with only `assertions` still publishes a probe,
+but its outcome is permanently `unknown` / `monitoring_unavailable` — the
+operation reads as monitored while nothing is checked, and `apifuse check`
+warns about it (`health-check-assertions-not-monitored`). Use this shape in
+real operations:
 
 ```ts
 healthCheck: {
@@ -167,11 +173,13 @@ healthCheck: {
   cases: [{
     name: "lookup baseline",
     input: { q: "btc" },
-    assertions: ({ data, status, durationMs }) => {
-      if (status !== 200 || data.results.length === 0 || durationMs > 3000) {
-        return { status: "degraded", label: "lookup baseline changed" };
-      }
-    },
+    scenario: lookupBaselineScenario, // defineHealthScenario({ ... })
   }],
 }
 ```
+
+If the operation reads through `ctx.cache.getOrSet(..., { staleIfErrorMs })`,
+add a guard step on the operation step result's `served_stale_cache` as well —
+otherwise an upstream outage is answered with a schema-valid 200 and the probe
+reports `ok` for the whole stale window. See
+`.agents/skills/health-checks-and-fail-closed/SKILL.md` for both shapes.

package/src/config/loader.ts

@@ -255,25 +255,62 @@ const SMARTPROXY_ALLOCATOR_RETRY_BASE_MS = 25;
 const SMARTPROXY_EXTRACTION_CACHE_TTL_MS = 15_000;
 const SMARTPROXY_EXTRACTION_SOFT_REFRESH_MS = 10_000;
 
+/** Env names that can supply a provider store's Redis URL. */
+export type ProviderRedisUrlEnvName =
+	| typeof PROVIDER_STATE_REDIS_URL_ENV
+	| typeof PROVIDER_CACHE_REDIS_URL_ENV
+	| typeof REDIS_URL_ENV;
+
+/** Which env name supplied a store's Redis URL, alongside the URL itself. */
+export type ProviderRedisUrlResolution = {
+	readonly url: string;
+	/** The env name that actually supplied `url`. */
+	readonly envName: ProviderRedisUrlEnvName;
+	/**
+	 * True when `url` came from a shared fallback env rather than the store's
+	 * own. A state store on `fallback: true` is sharing an instance that was
+	 * provisioned for something else (apifuse#2144).
+	 */
+	readonly fallback: boolean;
+};
+
+// Precedence is declared once here so the URL accessors, the boot report and
+// any future consumer can never disagree about which env wins.
+const CACHE_REDIS_URL_PRECEDENCE = [PROVIDER_CACHE_REDIS_URL_ENV, REDIS_URL_ENV] as const;
+const STATE_REDIS_URL_PRECEDENCE = [
+	PROVIDER_STATE_REDIS_URL_ENV,
+	PROVIDER_CACHE_REDIS_URL_ENV,
+	REDIS_URL_ENV,
+] as const;
+
+function resolveRedisUrlFromEnv(
+	precedence: readonly ProviderRedisUrlEnvName[],
+): ProviderRedisUrlResolution | undefined {
+	for (const [index, envName] of precedence.entries()) {
+		const url = readDiagnosticEnv(envName)?.trim();
+		if (url) return { url, envName, fallback: index > 0 };
+	}
+	return undefined;
+}
+
 function redisUrlFromEnv(): string | undefined {
-	return (
-		readDiagnosticEnv(PROVIDER_CACHE_REDIS_URL_ENV)?.trim() ||
-		readDiagnosticEnv(REDIS_URL_ENV)?.trim() ||
-		undefined
-	);
+	return providerCacheRedisResolutionFromEnv()?.url;
+}
+
+export function providerCacheRedisResolutionFromEnv(): ProviderRedisUrlResolution | undefined {
+	return resolveRedisUrlFromEnv(CACHE_REDIS_URL_PRECEDENCE);
+}
+
+export function providerStateRedisResolutionFromEnv(): ProviderRedisUrlResolution | undefined {
+	return resolveRedisUrlFromEnv(STATE_REDIS_URL_PRECEDENCE);
 }
 
 export function providerCacheRedisUrlFromEnv(): string | undefined {
-	return redisUrlFromEnv();
+	return providerCacheRedisResolutionFromEnv()?.url;
 }
 
 export function providerStateRedisUrlFromEnv(): string | undefined {
-	return (
-		readDiagnosticEnv(PROVIDER_STATE_REDIS_URL_ENV)?.trim() ||
-		readDiagnosticEnv(PROVIDER_CACHE_REDIS_URL_ENV)?.trim() ||
-		readDiagnosticEnv(REDIS_URL_ENV)?.trim() ||
-		undefined
-	);
+	return providerStateRedisResolutionFromEnv()?.url;
 }
 
 /** @internal Test-only hook for exercising shared proxy-cache behavior. */