@apifuse/provider-sdk 2.2.0-beta.67 → 2.2.0-beta.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +34 -2
- package/CHANGELOG.md +8 -0
- package/README.md +18 -7
- package/bin/apifuse-record.ts +81 -21
- package/bin/apifuse-submit-check.ts +170 -28
- package/dist/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl +138 -23
- package/dist/cli/templates/provider/README.md.tpl +17 -9
- package/dist/config/loader.d.ts +16 -0
- package/dist/config/loader.js +25 -8
- package/dist/error-resolution.d.ts +13 -0
- package/dist/error-resolution.js +72 -35
- package/dist/fixture-sanitization.js +159 -15
- package/dist/lint.d.ts +2 -0
- package/dist/lint.js +549 -11
- package/dist/runtime/diagnostic-redactor.d.ts +6 -1
- package/dist/runtime/diagnostic-redactor.js +16 -14
- package/dist/runtime/executor.js +3 -0
- package/dist/runtime/http.d.ts +2 -0
- package/dist/runtime/http.js +17 -103
- package/dist/runtime/instrumentation.js +4 -1
- package/dist/runtime/provider.d.ts +7 -0
- package/dist/runtime/provider.js +14 -0
- package/dist/runtime/redirect-hop-headers.d.ts +19 -0
- package/dist/runtime/redirect-hop-headers.js +58 -0
- package/dist/runtime/redirects.d.ts +33 -1
- package/dist/runtime/redirects.js +113 -0
- package/dist/runtime/resolver-span-attributes.d.ts +34 -0
- package/dist/runtime/resolver-span-attributes.js +93 -0
- package/dist/runtime/resolver-usage.d.ts +8 -0
- package/dist/runtime/resolver-usage.js +5 -11
- package/dist/runtime/resolver-vendors/browser.js +5 -4
- package/dist/runtime/resolver-vendors/capsolver.js +11 -10
- package/dist/runtime/resolver-vendors/twocaptcha.js +9 -8
- package/dist/runtime/resolver.js +7 -6
- package/dist/runtime/stealth-owned-headers.d.ts +21 -8
- package/dist/runtime/stealth-owned-headers.js +28 -8
- package/dist/runtime/stealth-telemetry.d.ts +1 -1
- package/dist/runtime/stealth.js +158 -46
- package/dist/runtime/trace.js +2 -2
- package/dist/server/index.d.ts +1 -0
- package/dist/server/serve-implementation.d.ts +2 -1
- package/dist/server/serve-implementation.js +45 -30
- package/dist/server/state-backend-report.d.ts +70 -0
- package/dist/server/state-backend-report.js +113 -0
- package/dist/testing/run.js +14 -7
- package/dist/trace-sanitization.js +1 -1
- package/dist/types.d.ts +30 -6
- package/package.json +2 -2
- package/src/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl +138 -23
- package/src/cli/templates/provider/README.md.tpl +17 -9
- package/src/config/loader.ts +49 -12
- package/src/error-resolution.ts +89 -35
- package/src/fixture-sanitization.ts +175 -17
- package/src/lint.ts +595 -10
- package/src/runtime/diagnostic-redactor.ts +19 -13
- package/src/runtime/executor.ts +3 -0
- package/src/runtime/http.ts +24 -112
- package/src/runtime/instrumentation.ts +4 -1
- package/src/runtime/provider.ts +15 -0
- package/src/runtime/redirect-hop-headers.ts +71 -0
- package/src/runtime/redirects.ts +150 -1
- package/src/runtime/resolver-span-attributes.ts +139 -0
- package/src/runtime/resolver-usage.ts +8 -5
- package/src/runtime/resolver-vendors/browser.ts +5 -4
- package/src/runtime/resolver-vendors/capsolver.ts +11 -10
- package/src/runtime/resolver-vendors/twocaptcha.ts +9 -8
- package/src/runtime/resolver.ts +7 -6
- package/src/runtime/stealth-owned-headers.ts +34 -8
- package/src/runtime/stealth-telemetry.ts +6 -1
- package/src/runtime/stealth.ts +180 -58
- package/src/runtime/trace.ts +6 -2
- package/src/server/index.ts +4 -0
- package/src/server/serve-implementation.ts +58 -31
- package/src/server/state-backend-report.ts +185 -0
- package/src/testing/run.ts +15 -2
- package/src/trace-sanitization.ts +1 -1
- package/src/types.ts +30 -6
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { providerCacheRedisResolutionFromEnv, providerStateRedisResolutionFromEnv, } from "../config/loader.js";
|
|
2
|
+
/**
|
|
3
|
+
* Whether ioredis' own URL parsing would move the endpoint away from the
|
|
4
|
+
* authority this function can see. `createProviderRedisClient` hands the raw
|
|
5
|
+
* URL to `new Redis(url, …)`, and `parseURL` ends with
|
|
6
|
+
* `defaults(result, queryOptions)` — lodash `defaults`, so a query key only
|
|
7
|
+
* applies where the authority supplied nothing:
|
|
8
|
+
*
|
|
9
|
+
* - `path` is never derived from the authority of a `redis://` URL (that
|
|
10
|
+
* pathname is the db index), so `?path=` always applies — and
|
|
11
|
+
* `StandaloneConnector` connects to `options.path` INSTEAD of host/port.
|
|
12
|
+
* The client is then on a Unix socket the authority says nothing about.
|
|
13
|
+
* - `port` applies only when the URL carries no explicit port.
|
|
14
|
+
* - `host` never applies here: we return early unless `parsed.hostname` is
|
|
15
|
+
* set, so `result.host` is always already populated.
|
|
16
|
+
* - `family` selects an address family; it does not move `host:port`.
|
|
17
|
+
*
|
|
18
|
+
* Verified against the pinned ioredis 5.11.1. In those cases the endpoint is
|
|
19
|
+
* WITHHELD and the ambiguity named, rather than reported as the authority — a
|
|
20
|
+
* diagnostic whose whole job is to answer "which Redis is this provider on"
|
|
21
|
+
* must not answer it wrongly. The query is never echoed: it can carry a
|
|
22
|
+
* password, which is why `endpoint` is host-and-port only to begin with.
|
|
23
|
+
*/
|
|
24
|
+
function queryMovesEndpoint(parsed) {
|
|
25
|
+
if (parsed.searchParams.has("path"))
|
|
26
|
+
return true;
|
|
27
|
+
return parsed.searchParams.has("port") && parsed.port === "";
|
|
28
|
+
}
|
|
29
|
+
function redisEndpoint(url) {
|
|
30
|
+
let parsed;
|
|
31
|
+
try {
|
|
32
|
+
parsed = new URL(url);
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
return {};
|
|
36
|
+
}
|
|
37
|
+
// No authority means this is not the `redis://host:port` shape at all (e.g.
|
|
38
|
+
// a bare `host:port`, which parses as a scheme). Report nothing rather than
|
|
39
|
+
// a fragment of a string whose parts we have not identified.
|
|
40
|
+
if (!parsed.hostname)
|
|
41
|
+
return {};
|
|
42
|
+
const scheme = parsed.protocol.replace(/:$/, "");
|
|
43
|
+
if (queryMovesEndpoint(parsed))
|
|
44
|
+
return { scheme, ambiguous: true };
|
|
45
|
+
return {
|
|
46
|
+
scheme,
|
|
47
|
+
endpoint: parsed.port ? `${parsed.hostname}:${parsed.port}` : parsed.hostname,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
function redisReport(resolution) {
|
|
51
|
+
const { endpoint, scheme, ambiguous } = redisEndpoint(resolution.url);
|
|
52
|
+
return {
|
|
53
|
+
parsed: endpoint !== undefined || ambiguous === true,
|
|
54
|
+
ambiguous: ambiguous === true,
|
|
55
|
+
report: {
|
|
56
|
+
backend: "redis",
|
|
57
|
+
envName: resolution.envName,
|
|
58
|
+
...(endpoint === undefined ? {} : { endpoint }),
|
|
59
|
+
...(scheme === undefined ? {} : { scheme }),
|
|
60
|
+
...(resolution.fallback ? { fallback: true } : {}),
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Build the boot event from the same env precedence the stores themselves use
|
|
66
|
+
* (`providerStateRedisResolutionFromEnv` / `providerCacheRedisResolutionFromEnv`),
|
|
67
|
+
* so the line can never describe a backend other than the one in use.
|
|
68
|
+
*/
|
|
69
|
+
export function providerStateBackendLogEvent(input) {
|
|
70
|
+
const warnings = [];
|
|
71
|
+
let state;
|
|
72
|
+
if (input.injectedState) {
|
|
73
|
+
state = { backend: "injected" };
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
const resolution = providerStateRedisResolutionFromEnv();
|
|
77
|
+
if (resolution) {
|
|
78
|
+
const { report, parsed, ambiguous } = redisReport(resolution);
|
|
79
|
+
state = report;
|
|
80
|
+
if (!parsed)
|
|
81
|
+
warnings.push("state_redis_url_unparsed");
|
|
82
|
+
if (ambiguous)
|
|
83
|
+
warnings.push("state_redis_url_ambiguous_endpoint");
|
|
84
|
+
if (resolution.fallback)
|
|
85
|
+
warnings.push("state_redis_url_fallback");
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
state = { backend: input.allowMemoryFallback ? "memory" : "unsupported" };
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
const cacheResolution = providerCacheRedisResolutionFromEnv();
|
|
92
|
+
let cache;
|
|
93
|
+
if (cacheResolution) {
|
|
94
|
+
const { report, parsed, ambiguous } = redisReport(cacheResolution);
|
|
95
|
+
cache = report;
|
|
96
|
+
if (!parsed)
|
|
97
|
+
warnings.push("cache_redis_url_unparsed");
|
|
98
|
+
if (ambiguous)
|
|
99
|
+
warnings.push("cache_redis_url_ambiguous_endpoint");
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
cache = { backend: "memory" };
|
|
103
|
+
}
|
|
104
|
+
return {
|
|
105
|
+
level: warnings.length > 0 ? "warn" : "info",
|
|
106
|
+
event: "provider_state_backend",
|
|
107
|
+
providerId: input.providerId,
|
|
108
|
+
state,
|
|
109
|
+
cache,
|
|
110
|
+
sdkVersion: input.sdkVersion,
|
|
111
|
+
...(warnings.length === 0 ? {} : { warnings }),
|
|
112
|
+
};
|
|
113
|
+
}
|
package/dist/testing/run.js
CHANGED
|
@@ -8,6 +8,7 @@ import { createTestHandleContext } from "../runtime/handle.js";
|
|
|
8
8
|
*/
|
|
9
9
|
const STANDARD_TEST_CONNECTION_ID = "standard-test-connection";
|
|
10
10
|
import { createMemoryProviderRuntimeState } from "../runtime/state.js";
|
|
11
|
+
import { assertCallerHeadersSupported } from "../runtime/stealth-owned-headers.js";
|
|
11
12
|
import { createUnsupportedOcrClient } from "../runtime/ocr.js";
|
|
12
13
|
import { createUnsupportedSttClient } from "../runtime/stt.js";
|
|
13
14
|
import { createNativeEgressAuthorization, NativeNetworkError, snapshotNativeConnectInput, snapshotNativeGrantInput, } from "../runtime/native-network.js";
|
|
@@ -213,13 +214,19 @@ function createUpstreamContext(provider, operationName, upstreamStub) {
|
|
|
213
214
|
return normalizeUpstreamResponse(canned);
|
|
214
215
|
};
|
|
215
216
|
const httpCall = async (method, url, body, options) => toHttpResponse(await dispatch({ transport: "http", method, url, body, options }));
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
217
|
+
// The production transport rejects owned headers (User-Agent, Sec-Fetch-*,
|
|
218
|
+
// sec-ch-ua*, ...) before any byte is sent; the fake applies the same
|
|
219
|
+
// function, so a provider test fails exactly where production would.
|
|
220
|
+
const stealthCall = async (url, options) => {
|
|
221
|
+
assertCallerHeadersSupported(options?.headers ?? {});
|
|
222
|
+
return toStealthResponse(await dispatch({
|
|
223
|
+
transport: "stealth",
|
|
224
|
+
method: options?.method?.toUpperCase() ?? "GET",
|
|
225
|
+
url,
|
|
226
|
+
body: options?.body,
|
|
227
|
+
options,
|
|
228
|
+
}), url);
|
|
229
|
+
};
|
|
223
230
|
const createBrowserPage = () => {
|
|
224
231
|
let currentUrl = "about:blank";
|
|
225
232
|
let currentResponse;
|
|
@@ -22,7 +22,7 @@ export function sanitizeTraceAttributes(attributes, redact, reservedKeys = []) {
|
|
|
22
22
|
const sanitizedKey = keyChanged ? nextKey() : sanitizeTraceText(key);
|
|
23
23
|
const valueRedactor = typeof value === "bigint" || isDiagnosticBigintAttribute(attributes, key)
|
|
24
24
|
? diagnosticStructuredRedactor(redact)
|
|
25
|
-
: diagnosticAttributeRedactor(key, redact);
|
|
25
|
+
: diagnosticAttributeRedactor(key, redact, value);
|
|
26
26
|
const sanitizedValue = keyChanged
|
|
27
27
|
? redactedKey === REDACTION_FAILED
|
|
28
28
|
? REDACTION_FAILED
|
package/dist/types.d.ts
CHANGED
|
@@ -1193,7 +1193,23 @@ export interface StealthFetchOptions extends Omit<RequestOptions, "redirectPolic
|
|
|
1193
1193
|
headers?: Record<string, string | string[]>;
|
|
1194
1194
|
method?: HttpMethod;
|
|
1195
1195
|
body?: string | Buffer;
|
|
1196
|
+
/**
|
|
1197
|
+
* Defaults to `"follow"`. Every followed hop drops the request-body headers
|
|
1198
|
+
* when the method changes, `Authorization` and `Proxy-Authorization` when it
|
|
1199
|
+
* leaves the current origin, and an explicit `Cookie` when it leaves the
|
|
1200
|
+
* current site (scheme plus registrable domain); after that the session
|
|
1201
|
+
* cookie jar supplies cookies for the new site.
|
|
1202
|
+
*/
|
|
1196
1203
|
redirect?: "follow" | "manual" | "error";
|
|
1204
|
+
/**
|
|
1205
|
+
* Opt-in same-origin redirect enforcement, the ctx.http `redirectPolicy`
|
|
1206
|
+
* for the stealth transport. A redirect whose target leaves the initial
|
|
1207
|
+
* origin, exceeds `maxHops` (at most 10, the stealth redirect limit),
|
|
1208
|
+
* loops, or has no usable `Location` throws `HttpRedirectError` before any
|
|
1209
|
+
* request is sent to that target. Requires the default
|
|
1210
|
+
* `redirect: "follow"`.
|
|
1211
|
+
*/
|
|
1212
|
+
redirectPolicy?: HttpRedirectPolicy;
|
|
1197
1213
|
/**
|
|
1198
1214
|
* Maximum decoded response-body bytes to buffer. When set, the stealth
|
|
1199
1215
|
* transport aborts the response and throws `response_too_large` if the
|
|
@@ -1219,13 +1235,21 @@ export interface StealthFetchOptions extends Omit<RequestOptions, "redirectPolic
|
|
|
1219
1235
|
* replacement for hand-set `Sec-Fetch-*` headers, which the transport
|
|
1220
1236
|
* rejects; it drives `Sec-Fetch-Mode`/`Dest`/`User`, `Accept`, `Priority`,
|
|
1221
1237
|
* and `Upgrade-Insecure-Requests`.
|
|
1238
|
+
*
|
|
1239
|
+
* `form-post` is an HTML `<form method="post">` submission: a navigation
|
|
1240
|
+
* (`navigate`/`document`, the document `Accept`, `Upgrade-Insecure-Requests`)
|
|
1241
|
+
* that carries a body, in the header order Chrome uses for it, with
|
|
1242
|
+
* `Cache-Control: max-age=0` and the caller's `Content-Type`, `Origin` and
|
|
1243
|
+
* `Referer` in place. It is never inferred, since a form submission and a
|
|
1244
|
+
* script `fetch()` POST look the same to the transport.
|
|
1222
1245
|
*/
|
|
1223
|
-
requestClass?: "navigation" | "xhr" | "post";
|
|
1246
|
+
requestClass?: "navigation" | "form-post" | "xhr" | "post";
|
|
1224
1247
|
/**
|
|
1225
|
-
* Whether a `navigation` request carries a user gesture and
|
|
1226
|
-
* `Sec-Fetch-User: ?1` (default `true`). Set `false` for
|
|
1227
|
-
* navigations such as `location.replace
|
|
1228
|
-
* Ignored for the `xhr` and
|
|
1248
|
+
* Whether a `navigation` or `form-post` request carries a user gesture and
|
|
1249
|
+
* therefore `Sec-Fetch-User: ?1` (default `true`). Set `false` for
|
|
1250
|
+
* script-driven navigations such as `location.replace` or
|
|
1251
|
+
* `form.submit()`, which Chrome sends without it. Ignored for the `xhr` and
|
|
1252
|
+
* `post` request classes.
|
|
1229
1253
|
*/
|
|
1230
1254
|
userActivation?: boolean;
|
|
1231
1255
|
/**
|
|
@@ -1321,7 +1345,7 @@ export interface StealthRedirectHop {
|
|
|
1321
1345
|
location?: string;
|
|
1322
1346
|
nextUrl?: string;
|
|
1323
1347
|
}
|
|
1324
|
-
export interface StealthRedirectRunOptions extends Omit<StealthFetchOptions, "redirect"> {
|
|
1348
|
+
export interface StealthRedirectRunOptions extends Omit<StealthFetchOptions, "redirect" | "redirectPolicy"> {
|
|
1325
1349
|
url: string;
|
|
1326
1350
|
maxHops?: number;
|
|
1327
1351
|
stopWhen?: (hop: StealthRedirectHop) => boolean | Promise<boolean>;
|
package/package.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"version": "2.2.0-beta.
|
|
2
|
+
"version": "2.2.0-beta.69",
|
|
3
3
|
"name": "@apifuse/provider-sdk",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
@@ -140,7 +140,7 @@
|
|
|
140
140
|
},
|
|
141
141
|
"dependencies": {
|
|
142
142
|
"@clack/prompts": "^1.5.1",
|
|
143
|
-
"@rgrove/parse-xml": "4.2.
|
|
143
|
+
"@rgrove/parse-xml": "4.2.3",
|
|
144
144
|
"@types/ms": "^2.1.0",
|
|
145
145
|
"acorn": "^8.17.0",
|
|
146
146
|
"hono": "^4.12.25",
|
package/src/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl
CHANGED
|
@@ -1,37 +1,142 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: health-checks-and-fail-closed
|
|
3
|
-
description: Writing health checks
|
|
3
|
+
description: Writing health checks the platform monitor actually executes, and fail-closed guards at envelope and row level. Load before writing healthCheck blocks or error handling.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Health checks and fail-closed guards
|
|
7
7
|
|
|
8
|
-
##
|
|
9
|
-
`Array.isArray(data.items)` alone can never fail. Every list operation's
|
|
10
|
-
health check must be able to detect the zero-rows regression.
|
|
8
|
+
## Only `scenario` is monitored in production
|
|
11
9
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
10
|
+
A `healthCheck` case takes one of two mutually exclusive shapes, and they do
|
|
11
|
+
not run in the same place:
|
|
12
|
+
|
|
13
|
+
| field | executed by | when |
|
|
14
|
+
|---|---|---|
|
|
15
|
+
| `scenario` | the platform health monitor | every suite interval, in production |
|
|
16
|
+
| `assertions` | the provider runtime's own self-test | only when that endpoint is called |
|
|
17
|
+
|
|
18
|
+
The monitor runs a case's SERIALIZED scenario and nothing else: a closure
|
|
19
|
+
cannot cross the registry's serialization boundary. A case that carries only
|
|
20
|
+
`assertions` is still published as a probe, but its outcome is permanently
|
|
21
|
+
`unknown` / `monitoring_unavailable` — the operation reads as monitored on the
|
|
22
|
+
status page while nothing is ever checked. `apifuse check` warns about this
|
|
23
|
+
(`health-check-assertions-not-monitored`).
|
|
24
|
+
|
|
25
|
+
Anything that must be caught in production belongs in `scenario`. Keep
|
|
26
|
+
`assertions` for a local self-test, or drop it.
|
|
27
|
+
|
|
28
|
+
## A scenario that can actually fail
|
|
29
|
+
|
|
30
|
+
`Array.isArray(data.items)` alone can never fail. Every list operation's probe
|
|
31
|
+
must be able to detect the zero-rows regression AND a mapper regression that
|
|
32
|
+
empties fields on a present row.
|
|
33
|
+
|
|
34
|
+
Step kinds: `operation` invokes, `assert` FAILS the probe (recorded `down`),
|
|
35
|
+
`guard` DEGRADES without failing it.
|
|
36
|
+
|
|
37
|
+
```ts
|
|
38
|
+
// health-scenarios.ts
|
|
39
|
+
import { type AssertionExpression, defineHealthScenario } from "@apifuse/provider-sdk/provider";
|
|
40
|
+
|
|
41
|
+
export const PROBE_INPUT = { district: "<a guaranteed-dense, stable id>" } as const;
|
|
42
|
+
|
|
43
|
+
const response = (path: readonly (string | number)[]) => ({
|
|
44
|
+
ref: { namespace: "steps" as const, binding: "response", path: [...path] },
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
const evidence: AssertionExpression = {
|
|
48
|
+
kind: "all",
|
|
49
|
+
clauses: [
|
|
50
|
+
{ kind: "predicate", operator: "status_2xx", actual: response(["status_code"]) },
|
|
51
|
+
// Dense query MUST return rows; zero rows = upstream contract drift.
|
|
52
|
+
{ kind: "predicate", operator: "array_length_gte", actual: response(["data", "items"]), expected: 1 },
|
|
53
|
+
// A semantic field on a real row, so an emptying mapper regression fails too.
|
|
54
|
+
{ kind: "predicate", operator: "non_empty", actual: response(["data", "items", 0, "name"]) },
|
|
55
|
+
],
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
export const listItemsScenario = defineHealthScenario({
|
|
59
|
+
scenarioVersion: 2,
|
|
60
|
+
id: "list-items.dense-district",
|
|
61
|
+
display: {
|
|
62
|
+
titleKey: "health.operations.list-items.dense-district.title",
|
|
63
|
+
descriptionKey: "health.operations.list-items.dense-district.description",
|
|
64
|
+
},
|
|
65
|
+
schedule: { kind: "interval", intervalMs: 60 * 60_000, jitterMs: 0 },
|
|
66
|
+
timeoutMs: 15_000,
|
|
67
|
+
coversOperations: ["list-items"],
|
|
68
|
+
credentialRefs: [],
|
|
69
|
+
steps: [
|
|
70
|
+
{ id: "read", result: "response", kind: "operation", operationId: "list-items", inputTemplate: { ...PROBE_INPUT } },
|
|
71
|
+
{ id: "verify", result: "verified", kind: "assert", coversOperations: ["list-items"], expression: evidence },
|
|
72
|
+
],
|
|
73
|
+
});
|
|
74
|
+
```
|
|
15
75
|
|
|
16
76
|
```ts
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
throw new Error("<op> missing items array");
|
|
23
|
-
}
|
|
24
|
-
// Dense query MUST return rows; zero rows = upstream contract drift
|
|
25
|
-
if (data.items.length === 0) {
|
|
26
|
-
return { status: "degraded", label: "<op> dense query returned 0 rows" };
|
|
27
|
-
}
|
|
77
|
+
// operations/list-items.ts
|
|
78
|
+
healthCheck: {
|
|
79
|
+
interval: "1h",
|
|
80
|
+
timeoutMs: 15_000,
|
|
81
|
+
cases: [{ name: "dense-district", input: { ...PROBE_INPUT }, scenario: listItemsScenario }],
|
|
28
82
|
}
|
|
29
83
|
```
|
|
30
84
|
|
|
31
|
-
- Choose
|
|
32
|
-
|
|
33
|
-
-
|
|
34
|
-
|
|
85
|
+
- Choose probe inputs that are guaranteed-dense (major city district, a stable
|
|
86
|
+
well-known entity id). Verify the id still exists when picking it.
|
|
87
|
+
- `scenario.coversOperations` may only name the operation that owns the case.
|
|
88
|
+
|
|
89
|
+
## Stale-if-error serves must be guarded, or an outage reads green
|
|
90
|
+
|
|
91
|
+
If the operation reads its upstream through `ctx.cache.getOrSet(..., {
|
|
92
|
+
staleIfErrorMs })`, an upstream outage is answered with HTTP 200 and a
|
|
93
|
+
well-formed, fully schema-valid body for the whole stale window. Every clause
|
|
94
|
+
over `status_code` and `data` still passes, so the probe rolls up `ok` while
|
|
95
|
+
the upstream is down, and only flips to `down` after the window expires — the
|
|
96
|
+
first minutes of every outage are reported as healthy.
|
|
97
|
+
|
|
98
|
+
`served_stale_cache` on the operation step result is the only operand that
|
|
99
|
+
separates a stale serve from a live read. It is always present, so the
|
|
100
|
+
reference resolves to `false` rather than raising `reference_unresolvable`.
|
|
101
|
+
Add ONE guard step per stale-if-error probe:
|
|
102
|
+
|
|
103
|
+
```ts
|
|
104
|
+
{
|
|
105
|
+
id: "guard-list-items-freshness",
|
|
106
|
+
result: "freshness-guarded",
|
|
107
|
+
kind: "guard",
|
|
108
|
+
condition: {
|
|
109
|
+
kind: "predicate",
|
|
110
|
+
operator: "not_equals",
|
|
111
|
+
actual: { ref: { namespace: "steps" as const, binding: "response", path: ["served_stale_cache"] } },
|
|
112
|
+
expected: true,
|
|
113
|
+
},
|
|
114
|
+
onFail: {
|
|
115
|
+
attribute: [
|
|
116
|
+
{
|
|
117
|
+
operationId: "list-items",
|
|
118
|
+
status: "degraded",
|
|
119
|
+
reasonCode: "expected_absence",
|
|
120
|
+
reasonKey: "health.operations.list-items.dense-district.servedStaleCache",
|
|
121
|
+
},
|
|
122
|
+
],
|
|
123
|
+
stop: "scenario",
|
|
124
|
+
},
|
|
125
|
+
}
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
- `degraded`, not `down`: the cache policy is behaving as promised and callers
|
|
129
|
+
are still served; it is the upstream that is unavailable, and that has to be
|
|
130
|
+
visible instead of green.
|
|
131
|
+
- `expected_absence` is the only `reasonCode` the guard attribution schema
|
|
132
|
+
admits today, and it reads correctly here — the FRESH read is what was
|
|
133
|
+
absent. The tenant-facing wording lives in the localized `reasonKey`.
|
|
134
|
+
- Place the freshness guard BEFORE any row or emptiness guard. A stale serve
|
|
135
|
+
that happens to be empty would otherwise be attributed to the emptiness
|
|
136
|
+
reason, which is a confident answer to the wrong question.
|
|
137
|
+
- Do not change the handler for this. The envelope's `meta.stale` is written
|
|
138
|
+
by the serve layer, so `return result.value` discarding `result.meta` does
|
|
139
|
+
not suppress the signal.
|
|
35
140
|
|
|
36
141
|
## Fail-closed: envelope level
|
|
37
142
|
- Upstream error headers/codes → structured `ProviderError` with a stable
|
|
@@ -51,6 +156,14 @@ array is non-empty, but every row normalizes to nothing.
|
|
|
51
156
|
- Regression-test both layers separately: a bad envelope AND a good envelope
|
|
52
157
|
with unmappable rows.
|
|
53
158
|
|
|
159
|
+
## Self-test assertions (optional, not monitored)
|
|
160
|
+
A case without a `scenario` may carry an `assertions` lambda, executed only by
|
|
161
|
+
the provider runtime's own self-test. THROW to fail the case (recorded
|
|
162
|
+
`down`); return `{ status: "degraded", label }` to flag without failing;
|
|
163
|
+
return nothing for `ok`. There is no `"down"` return value. It MUST NOT touch
|
|
164
|
+
the scheduler, recorder, or any runtime type. Never use it as the only check
|
|
165
|
+
on an operation you need monitored.
|
|
166
|
+
|
|
54
167
|
## Error message hygiene
|
|
55
168
|
`ProviderError.message` reaches the tenant verbatim. Never interpolate
|
|
56
169
|
upstream free text that may contain personal data (names, phone numbers,
|
|
@@ -58,8 +171,10 @@ addresses); allowlist known code tokens and keep raw bodies in server-side
|
|
|
58
171
|
details/logs only.
|
|
59
172
|
|
|
60
173
|
## Checklist
|
|
61
|
-
- [ ] Every
|
|
174
|
+
- [ ] Every monitored operation's case carries `scenario` (not only `assertions`)
|
|
175
|
+
- [ ] Every list op probe fails on 0 rows for a dense query
|
|
62
176
|
- [ ] One semantic field asserted on a real row
|
|
177
|
+
- [ ] Every `staleIfErrorMs` read has a `served_stale_cache` guard, placed first
|
|
63
178
|
- [ ] Swallowed-error test exists (non-ok mock → handler rejects)
|
|
64
179
|
- [ ] Non-empty upstream → zero normalized rows throws
|
|
65
180
|
- [ ] No upstream free text in customer-facing error messages
|
|
@@ -137,8 +137,8 @@ Structured errors return an `error` object with `code`, `message`,
|
|
|
137
137
|
## Next steps
|
|
138
138
|
|
|
139
139
|
1. Replace the sample `ping` operation with real upstream logic.
|
|
140
|
-
2. Once the real operation
|
|
141
|
-
`
|
|
140
|
+
2. Once the real operation calls the upstream through `ctx.http` or `ctx.stealth`
|
|
141
|
+
(absolute URLs, or relative ones under its `upstream.baseUrl`), record a fixture with:
|
|
142
142
|
`bun run record -- --operation <operation> --params '<json-input>'`.
|
|
143
143
|
3. Replace the starter `healthCheckUnsupported` with a real `healthCheck` for read-only upstream operations when safe.
|
|
144
144
|
4. Extend tests and operation metadata until the provider is bounty-ready.
|
|
@@ -158,8 +158,14 @@ Every operation must declare exactly one of:
|
|
|
158
158
|
The generated `ping` operation uses `healthCheckUnsupported` only because it is
|
|
159
159
|
a local scaffold check, not a real upstream API probe.
|
|
160
160
|
|
|
161
|
-
`
|
|
162
|
-
|
|
161
|
+
A case carries either `scenario` or `assertions`, and only `scenario` is
|
|
162
|
+
executed in production: the platform health monitor runs a case's serialized
|
|
163
|
+
scenario and nothing else, because a closure cannot cross the registry's
|
|
164
|
+
serialization boundary. A case with only `assertions` still publishes a probe,
|
|
165
|
+
but its outcome is permanently `unknown` / `monitoring_unavailable` — the
|
|
166
|
+
operation reads as monitored while nothing is checked, and `apifuse check`
|
|
167
|
+
warns about it (`health-check-assertions-not-monitored`). Use this shape in
|
|
168
|
+
real operations:
|
|
163
169
|
|
|
164
170
|
```ts
|
|
165
171
|
healthCheck: {
|
|
@@ -167,11 +173,13 @@ healthCheck: {
|
|
|
167
173
|
cases: [{
|
|
168
174
|
name: "lookup baseline",
|
|
169
175
|
input: { q: "btc" },
|
|
170
|
-
|
|
171
|
-
if (status !== 200 || data.results.length === 0 || durationMs > 3000) {
|
|
172
|
-
return { status: "degraded", label: "lookup baseline changed" };
|
|
173
|
-
}
|
|
174
|
-
},
|
|
176
|
+
scenario: lookupBaselineScenario, // defineHealthScenario({ ... })
|
|
175
177
|
}],
|
|
176
178
|
}
|
|
177
179
|
```
|
|
180
|
+
|
|
181
|
+
If the operation reads through `ctx.cache.getOrSet(..., { staleIfErrorMs })`,
|
|
182
|
+
add a guard step on the operation step result's `served_stale_cache` as well —
|
|
183
|
+
otherwise an upstream outage is answered with a schema-valid 200 and the probe
|
|
184
|
+
reports `ok` for the whole stale window. See
|
|
185
|
+
`.agents/skills/health-checks-and-fail-closed/SKILL.md` for both shapes.
|
package/src/config/loader.ts
CHANGED
|
@@ -255,25 +255,62 @@ const SMARTPROXY_ALLOCATOR_RETRY_BASE_MS = 25;
|
|
|
255
255
|
const SMARTPROXY_EXTRACTION_CACHE_TTL_MS = 15_000;
|
|
256
256
|
const SMARTPROXY_EXTRACTION_SOFT_REFRESH_MS = 10_000;
|
|
257
257
|
|
|
258
|
+
/** Env names that can supply a provider store's Redis URL. */
|
|
259
|
+
export type ProviderRedisUrlEnvName =
|
|
260
|
+
| typeof PROVIDER_STATE_REDIS_URL_ENV
|
|
261
|
+
| typeof PROVIDER_CACHE_REDIS_URL_ENV
|
|
262
|
+
| typeof REDIS_URL_ENV;
|
|
263
|
+
|
|
264
|
+
/** Which env name supplied a store's Redis URL, alongside the URL itself. */
|
|
265
|
+
export type ProviderRedisUrlResolution = {
|
|
266
|
+
readonly url: string;
|
|
267
|
+
/** The env name that actually supplied `url`. */
|
|
268
|
+
readonly envName: ProviderRedisUrlEnvName;
|
|
269
|
+
/**
|
|
270
|
+
* True when `url` came from a shared fallback env rather than the store's
|
|
271
|
+
* own. A state store on `fallback: true` is sharing an instance that was
|
|
272
|
+
* provisioned for something else (apifuse#2144).
|
|
273
|
+
*/
|
|
274
|
+
readonly fallback: boolean;
|
|
275
|
+
};
|
|
276
|
+
|
|
277
|
+
// Precedence is declared once here so the URL accessors, the boot report and
|
|
278
|
+
// any future consumer can never disagree about which env wins.
|
|
279
|
+
const CACHE_REDIS_URL_PRECEDENCE = [PROVIDER_CACHE_REDIS_URL_ENV, REDIS_URL_ENV] as const;
|
|
280
|
+
const STATE_REDIS_URL_PRECEDENCE = [
|
|
281
|
+
PROVIDER_STATE_REDIS_URL_ENV,
|
|
282
|
+
PROVIDER_CACHE_REDIS_URL_ENV,
|
|
283
|
+
REDIS_URL_ENV,
|
|
284
|
+
] as const;
|
|
285
|
+
|
|
286
|
+
function resolveRedisUrlFromEnv(
|
|
287
|
+
precedence: readonly ProviderRedisUrlEnvName[],
|
|
288
|
+
): ProviderRedisUrlResolution | undefined {
|
|
289
|
+
for (const [index, envName] of precedence.entries()) {
|
|
290
|
+
const url = readDiagnosticEnv(envName)?.trim();
|
|
291
|
+
if (url) return { url, envName, fallback: index > 0 };
|
|
292
|
+
}
|
|
293
|
+
return undefined;
|
|
294
|
+
}
|
|
295
|
+
|
|
258
296
|
function redisUrlFromEnv(): string | undefined {
|
|
259
|
-
return (
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
);
|
|
297
|
+
return providerCacheRedisResolutionFromEnv()?.url;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
export function providerCacheRedisResolutionFromEnv(): ProviderRedisUrlResolution | undefined {
|
|
301
|
+
return resolveRedisUrlFromEnv(CACHE_REDIS_URL_PRECEDENCE);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
export function providerStateRedisResolutionFromEnv(): ProviderRedisUrlResolution | undefined {
|
|
305
|
+
return resolveRedisUrlFromEnv(STATE_REDIS_URL_PRECEDENCE);
|
|
264
306
|
}
|
|
265
307
|
|
|
266
308
|
export function providerCacheRedisUrlFromEnv(): string | undefined {
|
|
267
|
-
return
|
|
309
|
+
return providerCacheRedisResolutionFromEnv()?.url;
|
|
268
310
|
}
|
|
269
311
|
|
|
270
312
|
export function providerStateRedisUrlFromEnv(): string | undefined {
|
|
271
|
-
return (
|
|
272
|
-
readDiagnosticEnv(PROVIDER_STATE_REDIS_URL_ENV)?.trim() ||
|
|
273
|
-
readDiagnosticEnv(PROVIDER_CACHE_REDIS_URL_ENV)?.trim() ||
|
|
274
|
-
readDiagnosticEnv(REDIS_URL_ENV)?.trim() ||
|
|
275
|
-
undefined
|
|
276
|
-
);
|
|
313
|
+
return providerStateRedisResolutionFromEnv()?.url;
|
|
277
314
|
}
|
|
278
315
|
|
|
279
316
|
/** @internal Test-only hook for exercising shared proxy-cache behavior. */
|