@apifuse/provider-sdk 2.2.0-beta.66 → 2.2.0-beta.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +28 -1
- package/CHANGELOG.md +8 -0
- package/README.md +8 -4
- package/bin/apifuse-record.ts +207 -31
- package/bin/apifuse-submit-check.ts +273 -74
- package/dist/cli/local-connection.d.ts +111 -0
- package/dist/cli/local-connection.js +151 -0
- package/dist/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl +138 -23
- package/dist/cli/templates/provider/README.md.tpl +17 -9
- package/dist/config/loader.d.ts +16 -0
- package/dist/config/loader.js +25 -8
- package/dist/error-resolution.d.ts +13 -0
- package/dist/error-resolution.js +72 -35
- package/dist/lint.d.ts +2 -0
- package/dist/lint.js +493 -2
- package/dist/runtime/executor.js +3 -0
- package/dist/runtime/http.d.ts +2 -0
- package/dist/runtime/http.js +2 -1
- package/dist/runtime/provider.d.ts +7 -0
- package/dist/runtime/provider.js +14 -0
- package/dist/server/index.d.ts +1 -0
- package/dist/server/serve-implementation.d.ts +2 -1
- package/dist/server/serve-implementation.js +43 -29
- package/dist/server/state-backend-report.d.ts +70 -0
- package/dist/server/state-backend-report.js +113 -0
- package/package.json +1 -1
- package/src/cli/local-connection.ts +221 -0
- package/src/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl +138 -23
- package/src/cli/templates/provider/README.md.tpl +17 -9
- package/src/config/loader.ts +49 -12
- package/src/error-resolution.ts +89 -35
- package/src/lint.ts +541 -1
- package/src/runtime/executor.ts +3 -0
- package/src/runtime/http.ts +2 -1
- package/src/runtime/provider.ts +15 -0
- package/src/server/index.ts +4 -0
- package/src/server/serve-implementation.ts +56 -30
- package/src/server/state-backend-report.ts +185 -0
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JSON object of `connection.secrets`, keyed exactly as the provider reads
|
|
3
|
+
* them through `ctx.credential.get(...)`.
|
|
4
|
+
*
|
|
5
|
+
* A single JSON value rather than one variable per key on purpose:
|
|
6
|
+
* platform-managed providers must not declare `credential.keys` (the SDK lint
|
|
7
|
+
* rejects it), so the key names are known only to the provider's own source.
|
|
8
|
+
* There is no derivation from an env name that would be right for every
|
|
9
|
+
* provider, and guessing one would be a second, lossy contract next to the
|
|
10
|
+
* gateway's.
|
|
11
|
+
*/
|
|
12
|
+
export const LOCAL_CONNECTION_SECRETS_ENV = "APIFUSE__LOCAL_CONNECTION__SECRETS";
|
|
13
|
+
/** Stable local connection id, mirroring the CLIs' other `local-*` scopes. */
|
|
14
|
+
export const LOCAL_CONNECTION_ID = "local-connection";
|
|
15
|
+
/** Auth modes whose providers read a credential the gateway supplies. */
|
|
16
|
+
export function providerRequiresConnection(provider) {
|
|
17
|
+
const mode = provider.auth?.mode;
|
|
18
|
+
return mode !== undefined && mode !== "none";
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Parses repeated `--credential key=value` arguments.
|
|
22
|
+
*
|
|
23
|
+
* Splits on the first `=` only: a base64 or URL-shaped secret routinely
|
|
24
|
+
* contains more.
|
|
25
|
+
*/
|
|
26
|
+
export function parseCredentialOverrides(values) {
|
|
27
|
+
const secrets = {};
|
|
28
|
+
for (const [index, value] of values.entries()) {
|
|
29
|
+
// The offending argument is never echoed. `--credential =secret` and
|
|
30
|
+
// `--credential secret` are both malformed *and* carry the credential, so
|
|
31
|
+
// quoting the input to be helpful would print it to stderr — before any
|
|
32
|
+
// redaction context exists, so no caller could scrub it.
|
|
33
|
+
const separator = value.indexOf("=");
|
|
34
|
+
if (separator <= 0) {
|
|
35
|
+
return {
|
|
36
|
+
ok: false,
|
|
37
|
+
reason: `--credential #${index + 1} is not of the form key=value; its value is not echoed here because it may be the credential.`,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
const key = value.slice(0, separator).trim();
|
|
41
|
+
if (key.length === 0) {
|
|
42
|
+
return {
|
|
43
|
+
ok: false,
|
|
44
|
+
reason: `--credential #${index + 1} has an empty key; its value is not echoed here because it may be the credential.`,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
secrets[key] = value.slice(separator + 1);
|
|
48
|
+
}
|
|
49
|
+
return { ok: true, secrets };
|
|
50
|
+
}
|
|
51
|
+
/** Reads {@link LOCAL_CONNECTION_SECRETS_ENV} into a secrets map. */
|
|
52
|
+
export function parseEnvironmentSecrets(raw) {
|
|
53
|
+
if (raw === undefined || raw.trim().length === 0) {
|
|
54
|
+
return { ok: true, secrets: {} };
|
|
55
|
+
}
|
|
56
|
+
let parsed;
|
|
57
|
+
try {
|
|
58
|
+
parsed = JSON.parse(raw);
|
|
59
|
+
}
|
|
60
|
+
catch (error) {
|
|
61
|
+
return {
|
|
62
|
+
ok: false,
|
|
63
|
+
reason: `${LOCAL_CONNECTION_SECRETS_ENV} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
67
|
+
return {
|
|
68
|
+
ok: false,
|
|
69
|
+
reason: `${LOCAL_CONNECTION_SECRETS_ENV} must be a JSON object of credential key to string value.`,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
const secrets = {};
|
|
73
|
+
for (const [key, value] of Object.entries(parsed)) {
|
|
74
|
+
if (typeof value !== "string") {
|
|
75
|
+
return {
|
|
76
|
+
ok: false,
|
|
77
|
+
reason: `${LOCAL_CONNECTION_SECRETS_ENV}.${key} must be a string (received ${typeof value}); the value is not echoed here.`,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
secrets[key] = value;
|
|
81
|
+
}
|
|
82
|
+
return { ok: true, secrets };
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Resolves the connection to attach for one local invocation.
|
|
86
|
+
*
|
|
87
|
+
* Three outcomes, and the middle one is the point of the change:
|
|
88
|
+
*
|
|
89
|
+
* - the provider needs no credential (`auth.mode` absent or `"none"`) →
|
|
90
|
+
* no connection, exactly as before;
|
|
91
|
+
* - the provider needs one and none is configured → `ok: false` with a
|
|
92
|
+
* message naming both ways to supply it. The caller surfaces that instead
|
|
93
|
+
* of letting the provider fail deep inside a handler on a `MISSING_SECRET`
|
|
94
|
+
* that reads like an upstream problem;
|
|
95
|
+
* - the provider needs one and it is configured → the gateway's envelope
|
|
96
|
+
* shape.
|
|
97
|
+
*
|
|
98
|
+
* CLI overrides win over the environment so a one-off run does not need the
|
|
99
|
+
* shell variable unset.
|
|
100
|
+
*/
|
|
101
|
+
export function resolveLocalConnection(provider, options = {}) {
|
|
102
|
+
const fromEnv = parseEnvironmentSecrets((options.env ?? process.env)[LOCAL_CONNECTION_SECRETS_ENV]);
|
|
103
|
+
if (!fromEnv.ok)
|
|
104
|
+
return fromEnv;
|
|
105
|
+
const fromArgs = parseCredentialOverrides(options.overrides ?? []);
|
|
106
|
+
if (!fromArgs.ok)
|
|
107
|
+
return fromArgs;
|
|
108
|
+
const secrets = { ...fromEnv.secrets, ...fromArgs.secrets };
|
|
109
|
+
const mode = provider.auth?.mode;
|
|
110
|
+
if (!providerRequiresConnection(provider)) {
|
|
111
|
+
if (Object.keys(secrets).length > 0) {
|
|
112
|
+
return {
|
|
113
|
+
ok: false,
|
|
114
|
+
reason: `Provider "${provider.id}" declares auth.mode ${JSON.stringify(mode ?? "none")}, which takes no credential. Remove the supplied credential values.`,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
return { ok: true, connection: undefined };
|
|
118
|
+
}
|
|
119
|
+
if (Object.keys(secrets).length === 0) {
|
|
120
|
+
return {
|
|
121
|
+
ok: false,
|
|
122
|
+
reason: missingLocalConnectionMessage(provider.id, mode ?? "none"),
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
return {
|
|
126
|
+
ok: true,
|
|
127
|
+
connection: {
|
|
128
|
+
id: LOCAL_CONNECTION_ID,
|
|
129
|
+
// `mode` is narrowed by providerRequiresConnection above.
|
|
130
|
+
mode: mode ?? "none",
|
|
131
|
+
secrets,
|
|
132
|
+
metadata: {},
|
|
133
|
+
externalRef: "",
|
|
134
|
+
},
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
/** The message a caller prints when a needed credential is not configured. */
|
|
138
|
+
export function missingLocalConnectionMessage(providerId, mode) {
|
|
139
|
+
const platformManaged = mode === "platform-managed";
|
|
140
|
+
return [
|
|
141
|
+
`Provider "${providerId}" declares auth.mode "${mode}", so the gateway injects a credential on every invocation and this command has none.`,
|
|
142
|
+
platformManaged
|
|
143
|
+
? "The value is the platform-owned key an operator holds; it is not a provider secret and is not derivable from the checkout."
|
|
144
|
+
: "Supply the same credential the gateway would resolve for a connection.",
|
|
145
|
+
"Supply it with one of:",
|
|
146
|
+
" --credential <key>=<value> (repeatable)",
|
|
147
|
+
` ${LOCAL_CONNECTION_SECRETS_ENV}='{"<key>":"<value>"}'`,
|
|
148
|
+
"The keys are the ones the provider passes to ctx.credential.get(...).",
|
|
149
|
+
"Pass --no-credential to run without one anyway (the provider will fail the way it does in production when the credential is unavailable).",
|
|
150
|
+
].join("\n");
|
|
151
|
+
}
|
package/dist/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl
CHANGED
|
@@ -1,37 +1,142 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: health-checks-and-fail-closed
|
|
3
|
-
description: Writing health checks
|
|
3
|
+
description: Writing health checks the platform monitor actually executes, and fail-closed guards at envelope and row level. Load before writing healthCheck blocks or error handling.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Health checks and fail-closed guards
|
|
7
7
|
|
|
8
|
-
##
|
|
9
|
-
`Array.isArray(data.items)` alone can never fail. Every list operation's
|
|
10
|
-
health check must be able to detect the zero-rows regression.
|
|
8
|
+
## Only `scenario` is monitored in production
|
|
11
9
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
10
|
+
A `healthCheck` case takes one of two mutually exclusive shapes, and they do
|
|
11
|
+
not run in the same place:
|
|
12
|
+
|
|
13
|
+
| field | executed by | when |
|
|
14
|
+
|---|---|---|
|
|
15
|
+
| `scenario` | the platform health monitor | every suite interval, in production |
|
|
16
|
+
| `assertions` | the provider runtime's own self-test | only when that endpoint is called |
|
|
17
|
+
|
|
18
|
+
The monitor runs a case's SERIALIZED scenario and nothing else: a closure
|
|
19
|
+
cannot cross the registry's serialization boundary. A case that carries only
|
|
20
|
+
`assertions` is still published as a probe, but its outcome is permanently
|
|
21
|
+
`unknown` / `monitoring_unavailable` — the operation reads as monitored on the
|
|
22
|
+
status page while nothing is ever checked. `apifuse check` warns about this
|
|
23
|
+
(`health-check-assertions-not-monitored`).
|
|
24
|
+
|
|
25
|
+
Anything that must be caught in production belongs in `scenario`. Keep
|
|
26
|
+
`assertions` for a local self-test, or drop it.
|
|
27
|
+
|
|
28
|
+
## A scenario that can actually fail
|
|
29
|
+
|
|
30
|
+
`Array.isArray(data.items)` alone can never fail. Every list operation's probe
|
|
31
|
+
must be able to detect the zero-rows regression AND a mapper regression that
|
|
32
|
+
empties fields on a present row.
|
|
33
|
+
|
|
34
|
+
Step kinds: `operation` invokes, `assert` FAILS the probe (recorded `down`),
|
|
35
|
+
`guard` DEGRADES without failing it.
|
|
36
|
+
|
|
37
|
+
```ts
|
|
38
|
+
// health-scenarios.ts
|
|
39
|
+
import { type AssertionExpression, defineHealthScenario } from "@apifuse/provider-sdk/provider";
|
|
40
|
+
|
|
41
|
+
export const PROBE_INPUT = { district: "<a guaranteed-dense, stable id>" } as const;
|
|
42
|
+
|
|
43
|
+
const response = (path: readonly (string | number)[]) => ({
|
|
44
|
+
ref: { namespace: "steps" as const, binding: "response", path: [...path] },
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
const evidence: AssertionExpression = {
|
|
48
|
+
kind: "all",
|
|
49
|
+
clauses: [
|
|
50
|
+
{ kind: "predicate", operator: "status_2xx", actual: response(["status_code"]) },
|
|
51
|
+
// Dense query MUST return rows; zero rows = upstream contract drift.
|
|
52
|
+
{ kind: "predicate", operator: "array_length_gte", actual: response(["data", "items"]), expected: 1 },
|
|
53
|
+
// A semantic field on a real row, so an emptying mapper regression fails too.
|
|
54
|
+
{ kind: "predicate", operator: "non_empty", actual: response(["data", "items", 0, "name"]) },
|
|
55
|
+
],
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
export const listItemsScenario = defineHealthScenario({
|
|
59
|
+
scenarioVersion: 2,
|
|
60
|
+
id: "list-items.dense-district",
|
|
61
|
+
display: {
|
|
62
|
+
titleKey: "health.operations.list-items.dense-district.title",
|
|
63
|
+
descriptionKey: "health.operations.list-items.dense-district.description",
|
|
64
|
+
},
|
|
65
|
+
schedule: { kind: "interval", intervalMs: 60 * 60_000, jitterMs: 0 },
|
|
66
|
+
timeoutMs: 15_000,
|
|
67
|
+
coversOperations: ["list-items"],
|
|
68
|
+
credentialRefs: [],
|
|
69
|
+
steps: [
|
|
70
|
+
{ id: "read", result: "response", kind: "operation", operationId: "list-items", inputTemplate: { ...PROBE_INPUT } },
|
|
71
|
+
{ id: "verify", result: "verified", kind: "assert", coversOperations: ["list-items"], expression: evidence },
|
|
72
|
+
],
|
|
73
|
+
});
|
|
74
|
+
```
|
|
15
75
|
|
|
16
76
|
```ts
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
throw new Error("<op> missing items array");
|
|
23
|
-
}
|
|
24
|
-
// Dense query MUST return rows; zero rows = upstream contract drift
|
|
25
|
-
if (data.items.length === 0) {
|
|
26
|
-
return { status: "degraded", label: "<op> dense query returned 0 rows" };
|
|
27
|
-
}
|
|
77
|
+
// operations/list-items.ts
|
|
78
|
+
healthCheck: {
|
|
79
|
+
interval: "1h",
|
|
80
|
+
timeoutMs: 15_000,
|
|
81
|
+
cases: [{ name: "dense-district", input: { ...PROBE_INPUT }, scenario: listItemsScenario }],
|
|
28
82
|
}
|
|
29
83
|
```
|
|
30
84
|
|
|
31
|
-
- Choose
|
|
32
|
-
|
|
33
|
-
-
|
|
34
|
-
|
|
85
|
+
- Choose probe inputs that are guaranteed-dense (major city district, a stable
|
|
86
|
+
well-known entity id). Verify the id still exists when picking it.
|
|
87
|
+
- `scenario.coversOperations` may only name the operation that owns the case.
|
|
88
|
+
|
|
89
|
+
## Stale-if-error serves must be guarded, or an outage reads green
|
|
90
|
+
|
|
91
|
+
If the operation reads its upstream through `ctx.cache.getOrSet(..., {
|
|
92
|
+
staleIfErrorMs })`, an upstream outage is answered with HTTP 200 and a
|
|
93
|
+
well-formed, fully schema-valid body for the whole stale window. Every clause
|
|
94
|
+
over `status_code` and `data` still passes, so the probe rolls up `ok` while
|
|
95
|
+
the upstream is down, and only flips to `down` after the window expires — the
|
|
96
|
+
first minutes of every outage are reported as healthy.
|
|
97
|
+
|
|
98
|
+
`served_stale_cache` on the operation step result is the only operand that
|
|
99
|
+
separates a stale serve from a live read. It is always present, so the
|
|
100
|
+
reference resolves to `false` rather than raising `reference_unresolvable`.
|
|
101
|
+
Add ONE guard step per stale-if-error probe:
|
|
102
|
+
|
|
103
|
+
```ts
|
|
104
|
+
{
|
|
105
|
+
id: "guard-list-items-freshness",
|
|
106
|
+
result: "freshness-guarded",
|
|
107
|
+
kind: "guard",
|
|
108
|
+
condition: {
|
|
109
|
+
kind: "predicate",
|
|
110
|
+
operator: "not_equals",
|
|
111
|
+
actual: { ref: { namespace: "steps" as const, binding: "response", path: ["served_stale_cache"] } },
|
|
112
|
+
expected: true,
|
|
113
|
+
},
|
|
114
|
+
onFail: {
|
|
115
|
+
attribute: [
|
|
116
|
+
{
|
|
117
|
+
operationId: "list-items",
|
|
118
|
+
status: "degraded",
|
|
119
|
+
reasonCode: "expected_absence",
|
|
120
|
+
reasonKey: "health.operations.list-items.dense-district.servedStaleCache",
|
|
121
|
+
},
|
|
122
|
+
],
|
|
123
|
+
stop: "scenario",
|
|
124
|
+
},
|
|
125
|
+
}
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
- `degraded`, not `down`: the cache policy is behaving as promised and callers
|
|
129
|
+
are still served; it is the upstream that is unavailable, and that has to be
|
|
130
|
+
visible instead of green.
|
|
131
|
+
- `expected_absence` is the only `reasonCode` the guard attribution schema
|
|
132
|
+
admits today, and it reads correctly here — the FRESH read is what was
|
|
133
|
+
absent. The tenant-facing wording lives in the localized `reasonKey`.
|
|
134
|
+
- Place the freshness guard BEFORE any row or emptiness guard. A stale serve
|
|
135
|
+
that happens to be empty would otherwise be attributed to the emptiness
|
|
136
|
+
reason, which is a confident answer to the wrong question.
|
|
137
|
+
- Do not change the handler for this. The envelope's `meta.stale` is written
|
|
138
|
+
by the serve layer, so `return result.value` discarding `result.meta` does
|
|
139
|
+
not suppress the signal.
|
|
35
140
|
|
|
36
141
|
## Fail-closed: envelope level
|
|
37
142
|
- Upstream error headers/codes → structured `ProviderError` with a stable
|
|
@@ -51,6 +156,14 @@ array is non-empty, but every row normalizes to nothing.
|
|
|
51
156
|
- Regression-test both layers separately: a bad envelope AND a good envelope
|
|
52
157
|
with unmappable rows.
|
|
53
158
|
|
|
159
|
+
## Self-test assertions (optional, not monitored)
|
|
160
|
+
A case without a `scenario` may carry an `assertions` lambda, executed only by
|
|
161
|
+
the provider runtime's own self-test. THROW to fail the case (recorded
|
|
162
|
+
`down`); return `{ status: "degraded", label }` to flag without failing;
|
|
163
|
+
return nothing for `ok`. There is no `"down"` return value. It MUST NOT touch
|
|
164
|
+
the scheduler, recorder, or any runtime type. Never use it as the only check
|
|
165
|
+
on an operation you need monitored.
|
|
166
|
+
|
|
54
167
|
## Error message hygiene
|
|
55
168
|
`ProviderError.message` reaches the tenant verbatim. Never interpolate
|
|
56
169
|
upstream free text that may contain personal data (names, phone numbers,
|
|
@@ -58,8 +171,10 @@ addresses); allowlist known code tokens and keep raw bodies in server-side
|
|
|
58
171
|
details/logs only.
|
|
59
172
|
|
|
60
173
|
## Checklist
|
|
61
|
-
- [ ] Every
|
|
174
|
+
- [ ] Every monitored operation's case carries `scenario` (not only `assertions`)
|
|
175
|
+
- [ ] Every list op probe fails on 0 rows for a dense query
|
|
62
176
|
- [ ] One semantic field asserted on a real row
|
|
177
|
+
- [ ] Every `staleIfErrorMs` read has a `served_stale_cache` guard, placed first
|
|
63
178
|
- [ ] Swallowed-error test exists (non-ok mock → handler rejects)
|
|
64
179
|
- [ ] Non-empty upstream → zero normalized rows throws
|
|
65
180
|
- [ ] No upstream free text in customer-facing error messages
|
|
@@ -137,8 +137,8 @@ Structured errors return an `error` object with `code`, `message`,
|
|
|
137
137
|
## Next steps
|
|
138
138
|
|
|
139
139
|
1. Replace the sample `ping` operation with real upstream logic.
|
|
140
|
-
2. Once the real operation
|
|
141
|
-
`
|
|
140
|
+
2. Once the real operation calls the upstream through `ctx.http` or `ctx.stealth`
|
|
141
|
+
(absolute URLs, or relative ones under its `upstream.baseUrl`), record a fixture with:
|
|
142
142
|
`bun run record -- --operation <operation> --params '<json-input>'`.
|
|
143
143
|
3. Replace the starter `healthCheckUnsupported` with a real `healthCheck` for read-only upstream operations when safe.
|
|
144
144
|
4. Extend tests and operation metadata until the provider is bounty-ready.
|
|
@@ -158,8 +158,14 @@ Every operation must declare exactly one of:
|
|
|
158
158
|
The generated `ping` operation uses `healthCheckUnsupported` only because it is
|
|
159
159
|
a local scaffold check, not a real upstream API probe.
|
|
160
160
|
|
|
161
|
-
`
|
|
162
|
-
|
|
161
|
+
A case carries either `scenario` or `assertions`, and only `scenario` is
|
|
162
|
+
executed in production: the platform health monitor runs a case's serialized
|
|
163
|
+
scenario and nothing else, because a closure cannot cross the registry's
|
|
164
|
+
serialization boundary. A case with only `assertions` still publishes a probe,
|
|
165
|
+
but its outcome is permanently `unknown` / `monitoring_unavailable` — the
|
|
166
|
+
operation reads as monitored while nothing is checked, and `apifuse check`
|
|
167
|
+
warns about it (`health-check-assertions-not-monitored`). Use this shape in
|
|
168
|
+
real operations:
|
|
163
169
|
|
|
164
170
|
```ts
|
|
165
171
|
healthCheck: {
|
|
@@ -167,11 +173,13 @@ healthCheck: {
|
|
|
167
173
|
cases: [{
|
|
168
174
|
name: "lookup baseline",
|
|
169
175
|
input: { q: "btc" },
|
|
170
|
-
|
|
171
|
-
if (status !== 200 || data.results.length === 0 || durationMs > 3000) {
|
|
172
|
-
return { status: "degraded", label: "lookup baseline changed" };
|
|
173
|
-
}
|
|
174
|
-
},
|
|
176
|
+
scenario: lookupBaselineScenario, // defineHealthScenario({ ... })
|
|
175
177
|
}],
|
|
176
178
|
}
|
|
177
179
|
```
|
|
180
|
+
|
|
181
|
+
If the operation reads through `ctx.cache.getOrSet(..., { staleIfErrorMs })`,
|
|
182
|
+
add a guard step on the operation step result's `served_stale_cache` as well —
|
|
183
|
+
otherwise an upstream outage is answered with a schema-valid 200 and the probe
|
|
184
|
+
reports `ok` for the whole stale window. See
|
|
185
|
+
`.agents/skills/health-checks-and-fail-closed/SKILL.md` for both shapes.
|
package/dist/config/loader.d.ts
CHANGED
|
@@ -135,6 +135,22 @@ export declare class ProxyResolutionError extends Error {
|
|
|
135
135
|
});
|
|
136
136
|
}
|
|
137
137
|
type ProxyRedisClient = Pick<Redis, "connect" | "del" | "eval" | "get" | "on" | "pttl" | "set" | "status">;
|
|
138
|
+
/** Env names that can supply a provider store's Redis URL. */
|
|
139
|
+
export type ProviderRedisUrlEnvName = typeof PROVIDER_STATE_REDIS_URL_ENV | typeof PROVIDER_CACHE_REDIS_URL_ENV | typeof REDIS_URL_ENV;
|
|
140
|
+
/** Which env name supplied a store's Redis URL, alongside the URL itself. */
|
|
141
|
+
export type ProviderRedisUrlResolution = {
|
|
142
|
+
readonly url: string;
|
|
143
|
+
/** The env name that actually supplied `url`. */
|
|
144
|
+
readonly envName: ProviderRedisUrlEnvName;
|
|
145
|
+
/**
|
|
146
|
+
* True when `url` came from a shared fallback env rather than the store's
|
|
147
|
+
* own. A state store on `fallback: true` is sharing an instance that was
|
|
148
|
+
* provisioned for something else (apifuse#2144).
|
|
149
|
+
*/
|
|
150
|
+
readonly fallback: boolean;
|
|
151
|
+
};
|
|
152
|
+
export declare function providerCacheRedisResolutionFromEnv(): ProviderRedisUrlResolution | undefined;
|
|
153
|
+
export declare function providerStateRedisResolutionFromEnv(): ProviderRedisUrlResolution | undefined;
|
|
138
154
|
export declare function providerCacheRedisUrlFromEnv(): string | undefined;
|
|
139
155
|
export declare function providerStateRedisUrlFromEnv(): string | undefined;
|
|
140
156
|
/** @internal Test-only hook for exercising shared proxy-cache behavior. */
|
package/dist/config/loader.js
CHANGED
|
@@ -63,19 +63,36 @@ const SMARTPROXY_ALLOCATOR_RETRY_BASE_MS = 25;
|
|
|
63
63
|
// reusing stale raw CONNECT endpoints as if they were valid for `life` minutes.
|
|
64
64
|
const SMARTPROXY_EXTRACTION_CACHE_TTL_MS = 15_000;
|
|
65
65
|
const SMARTPROXY_EXTRACTION_SOFT_REFRESH_MS = 10_000;
|
|
66
|
+
// Precedence is declared once here so the URL accessors, the boot report and
|
|
67
|
+
// any future consumer can never disagree about which env wins.
|
|
68
|
+
const CACHE_REDIS_URL_PRECEDENCE = [PROVIDER_CACHE_REDIS_URL_ENV, REDIS_URL_ENV];
|
|
69
|
+
const STATE_REDIS_URL_PRECEDENCE = [
|
|
70
|
+
PROVIDER_STATE_REDIS_URL_ENV,
|
|
71
|
+
PROVIDER_CACHE_REDIS_URL_ENV,
|
|
72
|
+
REDIS_URL_ENV,
|
|
73
|
+
];
|
|
74
|
+
function resolveRedisUrlFromEnv(precedence) {
|
|
75
|
+
for (const [index, envName] of precedence.entries()) {
|
|
76
|
+
const url = readDiagnosticEnv(envName)?.trim();
|
|
77
|
+
if (url)
|
|
78
|
+
return { url, envName, fallback: index > 0 };
|
|
79
|
+
}
|
|
80
|
+
return undefined;
|
|
81
|
+
}
|
|
66
82
|
function redisUrlFromEnv() {
|
|
67
|
-
return (
|
|
68
|
-
|
|
69
|
-
|
|
83
|
+
return providerCacheRedisResolutionFromEnv()?.url;
|
|
84
|
+
}
|
|
85
|
+
export function providerCacheRedisResolutionFromEnv() {
|
|
86
|
+
return resolveRedisUrlFromEnv(CACHE_REDIS_URL_PRECEDENCE);
|
|
87
|
+
}
|
|
88
|
+
export function providerStateRedisResolutionFromEnv() {
|
|
89
|
+
return resolveRedisUrlFromEnv(STATE_REDIS_URL_PRECEDENCE);
|
|
70
90
|
}
|
|
71
91
|
export function providerCacheRedisUrlFromEnv() {
|
|
72
|
-
return
|
|
92
|
+
return providerCacheRedisResolutionFromEnv()?.url;
|
|
73
93
|
}
|
|
74
94
|
export function providerStateRedisUrlFromEnv() {
|
|
75
|
-
return (
|
|
76
|
-
readDiagnosticEnv(PROVIDER_CACHE_REDIS_URL_ENV)?.trim() ||
|
|
77
|
-
readDiagnosticEnv(REDIS_URL_ENV)?.trim() ||
|
|
78
|
-
undefined);
|
|
95
|
+
return providerStateRedisResolutionFromEnv()?.url;
|
|
79
96
|
}
|
|
80
97
|
/** @internal Test-only hook for exercising shared proxy-cache behavior. */
|
|
81
98
|
export function __setProxyRedisForTests(redis) {
|
|
@@ -1,5 +1,18 @@
|
|
|
1
|
+
import type { ProviderErrorCategory } from "./observability.js";
|
|
1
2
|
import type { ProviderErrorStatus } from "./types.js";
|
|
2
3
|
export declare const SDK_OWNED_PROVIDER_ERROR_CODES: Set<string>;
|
|
3
4
|
export declare const SDK_RUNTIME_OWNED_ERROR_CODES: Set<string>;
|
|
5
|
+
/** One registered upstream code: the status the SDK serves and who failed. */
|
|
6
|
+
export type SdkUpstreamErrorCodeRegistration = {
|
|
7
|
+
readonly status: ProviderErrorStatus;
|
|
8
|
+
readonly category: ProviderErrorCategory;
|
|
9
|
+
};
|
|
10
|
+
export declare const SDK_UPSTREAM_ERROR_CODE_REGISTRY: ReadonlyMap<string, SdkUpstreamErrorCodeRegistration>;
|
|
11
|
+
/**
|
|
12
|
+
* The category a registered upstream code carries when its `ProviderError`
|
|
13
|
+
* sets none. `undefined` for every other code, which keeps its existing
|
|
14
|
+
* default.
|
|
15
|
+
*/
|
|
16
|
+
export declare function registeredUpstreamErrorCategory(code: string | undefined): ProviderErrorCategory | undefined;
|
|
4
17
|
export declare const SDK_STATUS_MAPPED_PROVIDER_ERROR_CODES: ReadonlyMap<string, ProviderErrorStatus>;
|
|
5
18
|
export declare const SDK_CANONICAL_ERROR_CODE_RETRYABILITY: ReadonlyMap<string, boolean>;
|
package/dist/error-resolution.js
CHANGED
|
@@ -116,27 +116,82 @@ export const SDK_RUNTIME_OWNED_ERROR_CODES = new Set([
|
|
|
116
116
|
"NOT_FOUND",
|
|
117
117
|
"not_found",
|
|
118
118
|
]);
|
|
119
|
+
// Registered codes that name an upstream outcome. A row states the status the
|
|
120
|
+
// SDK serves and the observability category together, so an upstream code
|
|
121
|
+
// cannot be registered without saying who failed. Serve reads the category
|
|
122
|
+
// for a `ProviderError` that carries one of these codes and no explicit
|
|
123
|
+
// `category`, and the public `source` follows from it (`sourceForCategory`).
|
|
124
|
+
// Without a row the category defaults to `provider_error` → `source:
|
|
125
|
+
// "apifuse"`, and the health monitor cannot tie a real upstream failure to the
|
|
126
|
+
// upstream: it publishes `monitoring_unavailable` instead of `down`.
|
|
127
|
+
//
|
|
128
|
+
// The category is `categoryForStatus(status)` unless the code names a
|
|
129
|
+
// narrower class than its status can: a schema change, a bot wall, an
|
|
130
|
+
// upstream refusal of the platform-managed key (served 400, see below).
|
|
131
|
+
// Every row is a code the fleet throws for something the upstream did or
|
|
132
|
+
// said; caller-side and SDK-internal codes stay in the table below.
|
|
133
|
+
export const SDK_UPSTREAM_ERROR_CODE_REGISTRY = new Map([
|
|
134
|
+
// The SDK's own unknown-operation and unknown-route throws reuse the
|
|
135
|
+
// not-found spellings; they set an explicit category and stay APIFuse's.
|
|
136
|
+
["NOT_FOUND", { status: 404, category: "upstream_http" }],
|
|
137
|
+
["not_found", { status: 404, category: "upstream_http" }],
|
|
138
|
+
["NO_DATA", { status: 404, category: "upstream_http" }],
|
|
139
|
+
["RATE_LIMITED", { status: 429, category: "upstream_rate_limited" }],
|
|
140
|
+
["UPSTREAM_RATE_LIMIT", { status: 429, category: "upstream_rate_limited" }],
|
|
141
|
+
[
|
|
142
|
+
"LIMITED_NUMBER_OF_SERVICE_REQUESTS_EXCEEDS_ERROR",
|
|
143
|
+
{ status: 429, category: "upstream_rate_limited" },
|
|
144
|
+
],
|
|
145
|
+
// Deterministic upstream business refusal (honest-provider-error-
|
|
146
|
+
// contract): the upstream evaluated the request and said no under its
|
|
147
|
+
// own rules — a conflict with upstream state, never a 5xx.
|
|
148
|
+
["UPSTREAM_REJECTED", { status: 409, category: "upstream_rejected" }],
|
|
149
|
+
["UPSTREAM_ERROR", { status: 502, category: "upstream_http" }],
|
|
150
|
+
// The upstream's bot wall or request protection refused the request. The
|
|
151
|
+
// status says only "upstream"; the code says which kind.
|
|
152
|
+
["BLOCKED", { status: 502, category: "anti_bot_blocked" }],
|
|
153
|
+
// Fleet-consensus provider codes. These are thrown by provider code, not
|
|
154
|
+
// by the SDK, so they are registered here (status mapping) and not in
|
|
155
|
+
// SDK_RUNTIME_OWNED_ERROR_CODES: an operation-declared status still wins,
|
|
156
|
+
// exactly as it does for UPSTREAM_ERROR and BLOCKED above. Registering
|
|
157
|
+
// them stops the fleet from re-declaring the same rows on every
|
|
158
|
+
// operation, and stops an undeclared throw from being served as 500.
|
|
159
|
+
//
|
|
160
|
+
// A platform-managed upstream service key the upstream refuses. With
|
|
161
|
+
// `auth.mode: "platform-managed"` the caller holds no credential, so 401
|
|
162
|
+
// ("re-authenticate") tells the caller to do something it cannot do, and
|
|
163
|
+
// 502 ("upstream is sick") promises a recovery that will never come. It is
|
|
164
|
+
// a deployment/config defect — the same class as MISSING_SECRET below,
|
|
165
|
+
// which is already an explicit 400 for that reason. The upstream is still
|
|
166
|
+
// the one refusing, so the category is `upstream_auth`; the gateway
|
|
167
|
+
// records it as `provider_error` for every non-`required` connection mode
|
|
168
|
+
// so the refusal stays provider-alertable.
|
|
169
|
+
["UPSTREAM_AUTH_ERROR", { status: 400, category: "upstream_auth" }],
|
|
170
|
+
// The upstream changed its response shape and the provider cannot
|
|
171
|
+
// normalize it. 502 because the fault is upstream of us, non-retryable
|
|
172
|
+
// because a retry returns the same broken payload.
|
|
173
|
+
["UPSTREAM_SCHEMA_ERROR", { status: 502, category: "upstream_schema_drift" }],
|
|
174
|
+
]);
|
|
175
|
+
/**
|
|
176
|
+
* The category a registered upstream code carries when its `ProviderError`
|
|
177
|
+
* sets none. `undefined` for every other code, which keeps its existing
|
|
178
|
+
* default.
|
|
179
|
+
*/
|
|
180
|
+
export function registeredUpstreamErrorCategory(code) {
|
|
181
|
+
return code === undefined ? undefined : SDK_UPSTREAM_ERROR_CODE_REGISTRY.get(code)?.category;
|
|
182
|
+
}
|
|
119
183
|
// Canonical SDK status mapping for recognized provider-thrown error codes.
|
|
120
184
|
// serve.ts toStatusCode consults this map (after operation-declared overrides
|
|
121
185
|
// for non-SDK-owned codes), and the authoring lint treats these codes as
|
|
122
186
|
// SDK-registered. Add new codes here instead of duplicating literals in
|
|
123
|
-
// either consumer
|
|
187
|
+
// either consumer — an upstream-attributed code goes in
|
|
188
|
+
// SDK_UPSTREAM_ERROR_CODE_REGISTRY above, which feeds this map.
|
|
124
189
|
export const SDK_STATUS_MAPPED_PROVIDER_ERROR_CODES = new Map([
|
|
125
190
|
["AUTH_REQUIRED", 401],
|
|
126
191
|
["reauth_required", 401],
|
|
127
192
|
// Unprovisioned declared secret: a deployment/config defect, never an
|
|
128
193
|
// upstream failure — explicit 400.
|
|
129
194
|
["MISSING_SECRET", 400],
|
|
130
|
-
["NOT_FOUND", 404],
|
|
131
|
-
["not_found", 404],
|
|
132
|
-
["NO_DATA", 404],
|
|
133
|
-
["RATE_LIMITED", 429],
|
|
134
|
-
["UPSTREAM_RATE_LIMIT", 429],
|
|
135
|
-
["LIMITED_NUMBER_OF_SERVICE_REQUESTS_EXCEEDS_ERROR", 429],
|
|
136
|
-
// Deterministic upstream business refusal (honest-provider-error-
|
|
137
|
-
// contract): the upstream evaluated the request and said no under its
|
|
138
|
-
// own rules — a conflict with upstream state, never a 5xx.
|
|
139
|
-
["UPSTREAM_REJECTED", 409],
|
|
140
195
|
// Only an unverifiable or foreign handle is the caller's: key and binding
|
|
141
196
|
// faults stay unmapped (500) because the engine host, not the caller, owns them.
|
|
142
197
|
["EGRESS_LEASE_INVALID", 409],
|
|
@@ -152,30 +207,11 @@ export const SDK_STATUS_MAPPED_PROVIDER_ERROR_CODES = new Map([
|
|
|
152
207
|
["HANDLE_EXPIRED", 410],
|
|
153
208
|
["HANDLE_COMMITTED", 409],
|
|
154
209
|
["HANDLE_BUSY", 409],
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
//
|
|
158
|
-
//
|
|
159
|
-
//
|
|
160
|
-
// exactly as it does for UPSTREAM_ERROR and BLOCKED above. Registering
|
|
161
|
-
// them stops the fleet from re-declaring the same three rows on every
|
|
162
|
-
// operation, and stops an undeclared throw from being served as 500.
|
|
163
|
-
//
|
|
164
|
-
// A platform-managed upstream service key the upstream refuses. With
|
|
165
|
-
// `auth.mode: "platform-managed"` the caller holds no credential, so 401
|
|
166
|
-
// ("re-authenticate") tells the caller to do something it cannot do, and
|
|
167
|
-
// 502 ("upstream is sick") promises a recovery that will never come. It is
|
|
168
|
-
// a deployment/config defect — the same class as MISSING_SECRET above,
|
|
169
|
-
// which is already an explicit 400 for that reason.
|
|
170
|
-
["UPSTREAM_AUTH_ERROR", 400],
|
|
171
|
-
// The upstream changed its response shape and the provider cannot
|
|
172
|
-
// normalize it. 502 because the fault is upstream of us, non-retryable
|
|
173
|
-
// because a retry returns the same broken payload.
|
|
174
|
-
["UPSTREAM_SCHEMA_ERROR", 502],
|
|
175
|
-
// Caller-side bad input rejected by the provider. The minority spellings
|
|
176
|
-
// (INVALID_INPUT / VALIDATION_ERROR / BAD_REQUEST) are deliberately not
|
|
177
|
-
// registered: they migrate to this spelling on the contract track, and
|
|
178
|
-
// registering them here would freeze the divergence.
|
|
210
|
+
// Caller-side bad input rejected by the provider (fleet consensus, like
|
|
211
|
+
// UPSTREAM_AUTH_ERROR and UPSTREAM_SCHEMA_ERROR above). The minority
|
|
212
|
+
// spellings (INVALID_INPUT / VALIDATION_ERROR / BAD_REQUEST) are
|
|
213
|
+
// deliberately not registered: they migrate to this spelling on the
|
|
214
|
+
// contract track, and registering them here would freeze the divergence.
|
|
179
215
|
["INVALID_REQUEST", 400],
|
|
180
216
|
["OCR_UNAVAILABLE", 503],
|
|
181
217
|
["UNSUPPORTED_OCR_BACKEND", 503],
|
|
@@ -189,6 +225,7 @@ export const SDK_STATUS_MAPPED_PROVIDER_ERROR_CODES = new Map([
|
|
|
189
225
|
// retry can clear, and a retryable 5xx there would turn one bad rollout
|
|
190
226
|
// into gateway-driven load.
|
|
191
227
|
["PROVIDER_ENGINE_UNAVAILABLE", 503],
|
|
228
|
+
...Array.from(SDK_UPSTREAM_ERROR_CODE_REGISTRY, ([code, registration]) => [code, registration.status]),
|
|
192
229
|
]);
|
|
193
230
|
// Canonical retryability for the fleet-consensus codes registered above.
|
|
194
231
|
//
|
package/dist/lint.d.ts
CHANGED
|
@@ -83,6 +83,8 @@ export declare function lintProvider(provider: {
|
|
|
83
83
|
fixtures?: unknown;
|
|
84
84
|
handler?: unknown;
|
|
85
85
|
source?: string;
|
|
86
|
+
/** Read structurally by the health-check monitorability rules. */
|
|
87
|
+
healthCheck?: unknown;
|
|
86
88
|
errorCodes?: ReadonlyArray<{
|
|
87
89
|
code: string;
|
|
88
90
|
status?: number;
|