@apifuse/provider-sdk 2.2.0-beta.69 → 2.2.0-beta.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +17 -0
- package/CHANGELOG.md +4 -0
- package/README.md +2 -0
- package/dist/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl +7 -4
- package/dist/health-scenario.d.ts +34 -13
- package/dist/health-scenario.js +34 -5
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/lint.js +37 -1
- package/dist/provider.d.ts +11 -11
- package/dist/provider.js +6 -6
- package/dist/server/index.d.ts +1 -0
- package/dist/server/index.js +1 -0
- package/dist/server/runtime-diagnostics.d.ts +165 -0
- package/dist/server/runtime-diagnostics.js +360 -0
- package/dist/server/serve-implementation.js +16 -0
- package/package.json +1 -1
- package/src/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl +7 -4
- package/src/health-scenario.ts +60 -13
- package/src/index.ts +1 -0
- package/src/lint.ts +42 -1
- package/src/provider.ts +130 -129
- package/src/server/index.ts +28 -0
- package/src/server/runtime-diagnostics.ts +522 -0
- package/src/server/serve-implementation.ts +22 -0
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
import { readdirSync, readFileSync, readlinkSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { performance } from "node:perf_hooks";
|
|
4
|
+
import { SDK_VERSION } from "./sdk-version.js";
|
|
5
|
+
/**
|
|
6
|
+
* Internal runtime diagnostics route on the provider's primary listener, next
|
|
7
|
+
* to `/health` and `/readyz`. It answers one JSON report that tells an idle
|
|
8
|
+
* process that still burns CPU apart from a busy one without `exec`: process
|
|
9
|
+
* CPU sampled over a short window, memory, the Node event-loop and
|
|
10
|
+
* active-resource APIs, and on Linux the per-thread CPU and file-descriptor
|
|
11
|
+
* picture from `/proc/self`.
|
|
12
|
+
*
|
|
13
|
+
* Read-only and secret-free. It shares the boundary of `/health` and `/readyz`:
|
|
14
|
+
* the generated provider NetworkPolicy admits only the gateway on this port and
|
|
15
|
+
* the gateway dials canonical operation paths only, so a tenant cannot reach
|
|
16
|
+
* it; operators use `kubectl port-forward`.
|
|
17
|
+
*/
|
|
18
|
+
export const RUNTIME_DIAGNOSTICS_ROUTE = "/__apifuse/diagnostics/runtime";
|
|
19
|
+
export const RUNTIME_DIAGNOSTICS_SCHEMA_VERSION = 1;
|
|
20
|
+
/** Query parameter naming the sampling window; clamped, never rejected. */
|
|
21
|
+
export const RUNTIME_DIAGNOSTICS_WINDOW_QUERY = "windowMs";
|
|
22
|
+
export const DEFAULT_RUNTIME_DIAGNOSTICS_WINDOW_MS = 250;
|
|
23
|
+
export const MIN_RUNTIME_DIAGNOSTICS_WINDOW_MS = 50;
|
|
24
|
+
export const MAX_RUNTIME_DIAGNOSTICS_WINDOW_MS = 5_000;
|
|
25
|
+
/**
|
|
26
|
+
* Linux USER_HZ. The unit of `utime`/`stime` in `/proc` stat files is an ABI
|
|
27
|
+
* constant (100 Hz) independent of the kernel's CONFIG_HZ.
|
|
28
|
+
*/
|
|
29
|
+
export const PROCFS_CLOCK_TICKS_PER_SECOND = 100;
|
|
30
|
+
export const DEFAULT_PROCFS_SELF_ROOT = "/proc/self";
|
|
31
|
+
const defaultClock = {
|
|
32
|
+
now: () => performance.now(),
|
|
33
|
+
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
34
|
+
};
|
|
35
|
+
export function normalizeRuntimeDiagnosticsWindowMs(value) {
|
|
36
|
+
if (value === undefined || !Number.isFinite(value))
|
|
37
|
+
return DEFAULT_RUNTIME_DIAGNOSTICS_WINDOW_MS;
|
|
38
|
+
return Math.min(MAX_RUNTIME_DIAGNOSTICS_WINDOW_MS, Math.max(MIN_RUNTIME_DIAGNOSTICS_WINDOW_MS, Math.trunc(value)));
|
|
39
|
+
}
|
|
40
|
+
/** Parses the `windowMs` query value; anything unparsable falls back to the default. */
|
|
41
|
+
export function resolveRuntimeDiagnosticsWindowMs(raw) {
|
|
42
|
+
if (raw === undefined || raw === null || raw.trim() === "") {
|
|
43
|
+
return DEFAULT_RUNTIME_DIAGNOSTICS_WINDOW_MS;
|
|
44
|
+
}
|
|
45
|
+
const parsed = Number.parseInt(raw, 10);
|
|
46
|
+
return normalizeRuntimeDiagnosticsWindowMs(Number.isNaN(parsed) ? undefined : parsed);
|
|
47
|
+
}
|
|
48
|
+
export async function collectRuntimeDiagnostics(input) {
|
|
49
|
+
const clock = input.clock ?? defaultClock;
|
|
50
|
+
const procfsRoot = input.procfsRoot ?? DEFAULT_PROCFS_SELF_ROOT;
|
|
51
|
+
const requestedMs = normalizeRuntimeDiagnosticsWindowMs(input.windowMs);
|
|
52
|
+
// Two nested brackets. The procfs scans are synchronous work on the main
|
|
53
|
+
// thread, so the process-level CPU bracket starts after the first scan and
|
|
54
|
+
// closes before the second one: the scans' own CPU never lands inside it.
|
|
55
|
+
// The thread bracket encloses the scans and divides their tick deltas by
|
|
56
|
+
// its own elapsed time, so the scan cost is attributed over the interval it
|
|
57
|
+
// actually spans instead of inflating a short window.
|
|
58
|
+
const threadsStartedAt = clock.now();
|
|
59
|
+
const threadsStart = sampleThreads(procfsRoot);
|
|
60
|
+
const cpuStart = process.cpuUsage();
|
|
61
|
+
const eventLoopStart = sampleEventLoopUtilization();
|
|
62
|
+
const startedAt = clock.now();
|
|
63
|
+
await clock.sleep(requestedMs);
|
|
64
|
+
const observedMs = Math.max(0, clock.now() - startedAt);
|
|
65
|
+
const cpuWindow = process.cpuUsage(cpuStart);
|
|
66
|
+
const eventLoopEnd = sampleEventLoopUtilization();
|
|
67
|
+
const threadsEnd = sampleThreads(procfsRoot);
|
|
68
|
+
const threadsObservedMs = Math.max(0, clock.now() - threadsStartedAt);
|
|
69
|
+
const fileDescriptors = sampleFileDescriptors(procfsRoot);
|
|
70
|
+
const cpuLifetime = process.cpuUsage();
|
|
71
|
+
const uptimeSeconds = process.uptime();
|
|
72
|
+
const sampledAt = new Date().toISOString();
|
|
73
|
+
const memory = process.memoryUsage();
|
|
74
|
+
const activeResources = sampleActiveResources();
|
|
75
|
+
return {
|
|
76
|
+
schemaVersion: RUNTIME_DIAGNOSTICS_SCHEMA_VERSION,
|
|
77
|
+
provider: { id: input.provider.id, version: input.provider.version },
|
|
78
|
+
sdkVersion: SDK_VERSION,
|
|
79
|
+
runtime: {
|
|
80
|
+
bunVersion: process.versions.bun ?? null,
|
|
81
|
+
nodeCompatVersion: process.versions.node,
|
|
82
|
+
platform: process.platform,
|
|
83
|
+
arch: process.arch,
|
|
84
|
+
pid: process.pid,
|
|
85
|
+
},
|
|
86
|
+
sampledAt,
|
|
87
|
+
uptimeSeconds,
|
|
88
|
+
window: { requestedMs, observedMs },
|
|
89
|
+
cpu: {
|
|
90
|
+
lifetime: {
|
|
91
|
+
userMicros: cpuLifetime.user,
|
|
92
|
+
systemMicros: cpuLifetime.system,
|
|
93
|
+
averageCores: coresFromMicros(cpuLifetime.user + cpuLifetime.system, uptimeSeconds * 1000),
|
|
94
|
+
},
|
|
95
|
+
window: {
|
|
96
|
+
userMicros: cpuWindow.user,
|
|
97
|
+
systemMicros: cpuWindow.system,
|
|
98
|
+
userCores: coresFromMicros(cpuWindow.user, observedMs),
|
|
99
|
+
systemCores: coresFromMicros(cpuWindow.system, observedMs),
|
|
100
|
+
totalCores: coresFromMicros(cpuWindow.user + cpuWindow.system, observedMs),
|
|
101
|
+
},
|
|
102
|
+
},
|
|
103
|
+
memory: {
|
|
104
|
+
rss: memory.rss,
|
|
105
|
+
heapTotal: memory.heapTotal,
|
|
106
|
+
heapUsed: memory.heapUsed,
|
|
107
|
+
external: memory.external,
|
|
108
|
+
arrayBuffers: memory.arrayBuffers,
|
|
109
|
+
},
|
|
110
|
+
eventLoop: {
|
|
111
|
+
reported: eventLoopEnd.idle + eventLoopEnd.active > 0,
|
|
112
|
+
lifetime: eventLoopEnd,
|
|
113
|
+
window: eventLoopUtilizationDelta(eventLoopEnd, eventLoopStart),
|
|
114
|
+
},
|
|
115
|
+
activeResources,
|
|
116
|
+
procfs: threadsEnd === null || fileDescriptors === null
|
|
117
|
+
? null
|
|
118
|
+
: {
|
|
119
|
+
clockTicksPerSecond: PROCFS_CLOCK_TICKS_PER_SECOND,
|
|
120
|
+
observedMs: threadsObservedMs,
|
|
121
|
+
threads: summarizeThreads(threadsStart, threadsEnd, threadsObservedMs),
|
|
122
|
+
fileDescriptors,
|
|
123
|
+
},
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
function coresFromMicros(micros, elapsedMs) {
|
|
127
|
+
if (!(elapsedMs > 0))
|
|
128
|
+
return 0;
|
|
129
|
+
return micros / 1000 / elapsedMs;
|
|
130
|
+
}
|
|
131
|
+
function sampleEventLoopUtilization() {
|
|
132
|
+
const eventLoopUtilization = performance.eventLoopUtilization;
|
|
133
|
+
if (typeof eventLoopUtilization !== "function")
|
|
134
|
+
return { idle: 0, active: 0, utilization: 0 };
|
|
135
|
+
try {
|
|
136
|
+
const sample = eventLoopUtilization.call(performance);
|
|
137
|
+
return {
|
|
138
|
+
idle: numberProperty(sample, "idle"),
|
|
139
|
+
active: numberProperty(sample, "active"),
|
|
140
|
+
utilization: numberProperty(sample, "utilization"),
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
catch {
|
|
144
|
+
return { idle: 0, active: 0, utilization: 0 };
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
function eventLoopUtilizationDelta(end, start) {
|
|
148
|
+
const idle = Math.max(0, end.idle - start.idle);
|
|
149
|
+
const active = Math.max(0, end.active - start.active);
|
|
150
|
+
const total = idle + active;
|
|
151
|
+
return { idle, active, utilization: total > 0 ? active / total : 0 };
|
|
152
|
+
}
|
|
153
|
+
function sampleActiveResources() {
|
|
154
|
+
const getActiveResourcesInfo = process.getActiveResourcesInfo;
|
|
155
|
+
let types = [];
|
|
156
|
+
if (typeof getActiveResourcesInfo === "function") {
|
|
157
|
+
try {
|
|
158
|
+
const listed = getActiveResourcesInfo.call(process);
|
|
159
|
+
if (Array.isArray(listed)) {
|
|
160
|
+
types = listed.filter((entry) => typeof entry === "string");
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
catch {
|
|
164
|
+
types = [];
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
const byType = {};
|
|
168
|
+
for (const type of types)
|
|
169
|
+
byType[type] = (byType[type] ?? 0) + 1;
|
|
170
|
+
return { reported: types.length > 0, total: types.length, byType };
|
|
171
|
+
}
|
|
172
|
+
function numberProperty(value, key) {
|
|
173
|
+
if (value === null || typeof value !== "object")
|
|
174
|
+
return 0;
|
|
175
|
+
const candidate = value[key];
|
|
176
|
+
return typeof candidate === "number" && Number.isFinite(candidate) ? candidate : 0;
|
|
177
|
+
}
|
|
178
|
+
function sampleThreads(root) {
|
|
179
|
+
let entries;
|
|
180
|
+
try {
|
|
181
|
+
entries = readdirSync(join(root, "task"));
|
|
182
|
+
}
|
|
183
|
+
catch {
|
|
184
|
+
return null;
|
|
185
|
+
}
|
|
186
|
+
const samples = new Map();
|
|
187
|
+
for (const entry of entries) {
|
|
188
|
+
if (!/^\d+$/.test(entry))
|
|
189
|
+
continue;
|
|
190
|
+
const sample = sampleThread(join(root, "task", entry), Number.parseInt(entry, 10));
|
|
191
|
+
if (sample)
|
|
192
|
+
samples.set(sample.tid, sample);
|
|
193
|
+
}
|
|
194
|
+
return samples;
|
|
195
|
+
}
|
|
196
|
+
function sampleThread(taskDir, tid) {
|
|
197
|
+
let stat;
|
|
198
|
+
try {
|
|
199
|
+
stat = readFileSync(join(taskDir, "stat"), "utf8");
|
|
200
|
+
}
|
|
201
|
+
catch {
|
|
202
|
+
// The thread exited between the directory listing and this read.
|
|
203
|
+
return null;
|
|
204
|
+
}
|
|
205
|
+
const open = stat.indexOf("(");
|
|
206
|
+
const close = stat.lastIndexOf(")");
|
|
207
|
+
if (open < 0 || close < open)
|
|
208
|
+
return null;
|
|
209
|
+
const fields = stat
|
|
210
|
+
.slice(close + 2)
|
|
211
|
+
.trimEnd()
|
|
212
|
+
.split(" ");
|
|
213
|
+
const userTicks = Number.parseInt(fields[11] ?? "", 10);
|
|
214
|
+
const systemTicks = Number.parseInt(fields[12] ?? "", 10);
|
|
215
|
+
if (!Number.isFinite(userTicks) || !Number.isFinite(systemTicks))
|
|
216
|
+
return null;
|
|
217
|
+
const switches = readContextSwitches(join(taskDir, "status"));
|
|
218
|
+
return {
|
|
219
|
+
tid,
|
|
220
|
+
name: stat.slice(open + 1, close),
|
|
221
|
+
state: fields[0] ?? "?",
|
|
222
|
+
userTicks,
|
|
223
|
+
systemTicks,
|
|
224
|
+
voluntary: switches.voluntary,
|
|
225
|
+
nonvoluntary: switches.nonvoluntary,
|
|
226
|
+
waitChannel: readWaitChannel(join(taskDir, "wchan")),
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
function readContextSwitches(statusPath) {
|
|
230
|
+
let voluntary = 0;
|
|
231
|
+
let nonvoluntary = 0;
|
|
232
|
+
let status;
|
|
233
|
+
try {
|
|
234
|
+
status = readFileSync(statusPath, "utf8");
|
|
235
|
+
}
|
|
236
|
+
catch {
|
|
237
|
+
return { voluntary, nonvoluntary };
|
|
238
|
+
}
|
|
239
|
+
for (const line of status.split("\n")) {
|
|
240
|
+
const separator = line.indexOf(":");
|
|
241
|
+
if (separator < 0)
|
|
242
|
+
continue;
|
|
243
|
+
const key = line.slice(0, separator);
|
|
244
|
+
if (key !== "voluntary_ctxt_switches" && key !== "nonvoluntary_ctxt_switches")
|
|
245
|
+
continue;
|
|
246
|
+
const parsed = Number.parseInt(line.slice(separator + 1).trim(), 10);
|
|
247
|
+
if (!Number.isFinite(parsed))
|
|
248
|
+
continue;
|
|
249
|
+
if (key === "voluntary_ctxt_switches")
|
|
250
|
+
voluntary = parsed;
|
|
251
|
+
else
|
|
252
|
+
nonvoluntary = parsed;
|
|
253
|
+
}
|
|
254
|
+
return { voluntary, nonvoluntary };
|
|
255
|
+
}
|
|
256
|
+
function readWaitChannel(wchanPath) {
|
|
257
|
+
try {
|
|
258
|
+
const value = readFileSync(wchanPath, "utf8").trim();
|
|
259
|
+
return value === "" ? null : value;
|
|
260
|
+
}
|
|
261
|
+
catch {
|
|
262
|
+
return null;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
function summarizeThreads(start, end, observedMs) {
|
|
266
|
+
const byName = {};
|
|
267
|
+
const items = [];
|
|
268
|
+
for (const sample of end.values()) {
|
|
269
|
+
byName[sample.name] = (byName[sample.name] ?? 0) + 1;
|
|
270
|
+
const baseline = start?.get(sample.tid);
|
|
271
|
+
let window = null;
|
|
272
|
+
if (baseline) {
|
|
273
|
+
const userTicks = Math.max(0, sample.userTicks - baseline.userTicks);
|
|
274
|
+
const systemTicks = Math.max(0, sample.systemTicks - baseline.systemTicks);
|
|
275
|
+
window = {
|
|
276
|
+
userTicks,
|
|
277
|
+
systemTicks,
|
|
278
|
+
cores: coresFromTicks(userTicks + systemTicks, observedMs),
|
|
279
|
+
contextSwitches: {
|
|
280
|
+
voluntary: Math.max(0, sample.voluntary - baseline.voluntary),
|
|
281
|
+
nonvoluntary: Math.max(0, sample.nonvoluntary - baseline.nonvoluntary),
|
|
282
|
+
},
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
items.push({
|
|
286
|
+
tid: sample.tid,
|
|
287
|
+
name: sample.name,
|
|
288
|
+
state: sample.state,
|
|
289
|
+
waitChannel: sample.waitChannel,
|
|
290
|
+
lifetime: { userTicks: sample.userTicks, systemTicks: sample.systemTicks },
|
|
291
|
+
window,
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
items.sort(compareThreads);
|
|
295
|
+
return { total: items.length, byName, items };
|
|
296
|
+
}
|
|
297
|
+
function coresFromTicks(ticks, elapsedMs) {
|
|
298
|
+
if (!(elapsedMs > 0))
|
|
299
|
+
return 0;
|
|
300
|
+
return ((ticks / PROCFS_CLOCK_TICKS_PER_SECOND) * 1000) / elapsedMs;
|
|
301
|
+
}
|
|
302
|
+
function compareThreads(left, right) {
|
|
303
|
+
const leftCores = left.window?.cores ?? -1;
|
|
304
|
+
const rightCores = right.window?.cores ?? -1;
|
|
305
|
+
if (leftCores !== rightCores)
|
|
306
|
+
return rightCores - leftCores;
|
|
307
|
+
const leftLifetime = left.lifetime.userTicks + left.lifetime.systemTicks;
|
|
308
|
+
const rightLifetime = right.lifetime.userTicks + right.lifetime.systemTicks;
|
|
309
|
+
if (leftLifetime !== rightLifetime)
|
|
310
|
+
return rightLifetime - leftLifetime;
|
|
311
|
+
return left.tid - right.tid;
|
|
312
|
+
}
|
|
313
|
+
function sampleFileDescriptors(root) {
|
|
314
|
+
let entries;
|
|
315
|
+
try {
|
|
316
|
+
entries = readdirSync(join(root, "fd"));
|
|
317
|
+
}
|
|
318
|
+
catch {
|
|
319
|
+
return null;
|
|
320
|
+
}
|
|
321
|
+
const byKind = {
|
|
322
|
+
socket: 0,
|
|
323
|
+
eventpoll: 0,
|
|
324
|
+
timerfd: 0,
|
|
325
|
+
eventfd: 0,
|
|
326
|
+
pipe: 0,
|
|
327
|
+
file: 0,
|
|
328
|
+
other: 0,
|
|
329
|
+
};
|
|
330
|
+
let total = 0;
|
|
331
|
+
for (const entry of entries) {
|
|
332
|
+
let target;
|
|
333
|
+
try {
|
|
334
|
+
target = readlinkSync(join(root, "fd", entry));
|
|
335
|
+
}
|
|
336
|
+
catch {
|
|
337
|
+
// Closed between the listing and the readlink (the listing's own
|
|
338
|
+
// directory handle always lands here).
|
|
339
|
+
continue;
|
|
340
|
+
}
|
|
341
|
+
byKind[classifyFileDescriptor(target)] += 1;
|
|
342
|
+
total += 1;
|
|
343
|
+
}
|
|
344
|
+
return { total, byKind };
|
|
345
|
+
}
|
|
346
|
+
function classifyFileDescriptor(target) {
|
|
347
|
+
if (target.startsWith("socket:"))
|
|
348
|
+
return "socket";
|
|
349
|
+
if (target === "anon_inode:[eventpoll]")
|
|
350
|
+
return "eventpoll";
|
|
351
|
+
if (target === "anon_inode:[timerfd]")
|
|
352
|
+
return "timerfd";
|
|
353
|
+
if (target === "anon_inode:[eventfd]")
|
|
354
|
+
return "eventfd";
|
|
355
|
+
if (target.startsWith("pipe:"))
|
|
356
|
+
return "pipe";
|
|
357
|
+
if (target.startsWith("/"))
|
|
358
|
+
return "file";
|
|
359
|
+
return "other";
|
|
360
|
+
}
|
|
@@ -64,6 +64,7 @@ import { APIFUSE_STREAM_DONE_EVENT, APIFUSE_STREAM_ERROR_EVENT, encodeSseEvent,
|
|
|
64
64
|
import { sanitizeSpanNameForOutput, sanitizeTraceAttributes } from "../trace-sanitization.js";
|
|
65
65
|
import { VALID_OPERATION_ERROR_STATUSES } from "../types.js";
|
|
66
66
|
import { providerStateBackendLogEvent, } from "./state-backend-report.js";
|
|
67
|
+
import { collectRuntimeDiagnostics, resolveRuntimeDiagnosticsWindowMs, RUNTIME_DIAGNOSTICS_ROUTE, RUNTIME_DIAGNOSTICS_WINDOW_QUERY, } from "./runtime-diagnostics.js";
|
|
67
68
|
import { resolveSelfTestMasterSecrets } from "./self-test-token.js";
|
|
68
69
|
import { collectStaticDiagnosticSensitiveValues, createDiagnosticEnvObserver, } from "./sensitive-values.js";
|
|
69
70
|
import { resolveServerTraceContextOptions } from "./trace-output.js";
|
|
@@ -2594,6 +2595,21 @@ function createServerAppWithCapabilityModules(provider, serverOptions, capabilit
|
|
|
2594
2595
|
attached: engineAttached,
|
|
2595
2596
|
},
|
|
2596
2597
|
}, engineAttached ? 200 : 503));
|
|
2598
|
+
// Internal runtime diagnostics: process CPU over a short window, memory,
|
|
2599
|
+
// the Node event-loop/active-resource APIs and, on Linux, per-thread CPU and
|
|
2600
|
+
// file-descriptor counts from /proc/self. Tells an idle pod that still burns
|
|
2601
|
+
// CPU apart from a busy one without exec. Read-only and secret-free, behind
|
|
2602
|
+
// the same boundary as /health and /readyz: the provider NetworkPolicy only
|
|
2603
|
+
// admits the gateway on this port and the gateway dials canonical operation
|
|
2604
|
+
// paths only, so operators reach it through `kubectl port-forward`.
|
|
2605
|
+
app.get(RUNTIME_DIAGNOSTICS_ROUTE, async (c) => {
|
|
2606
|
+
const report = await collectRuntimeDiagnostics({
|
|
2607
|
+
provider: { id: provider.id, version: provider.version },
|
|
2608
|
+
windowMs: resolveRuntimeDiagnosticsWindowMs(c.req.query(RUNTIME_DIAGNOSTICS_WINDOW_QUERY)),
|
|
2609
|
+
});
|
|
2610
|
+
c.header("Cache-Control", "no-store");
|
|
2611
|
+
return c.json(report);
|
|
2612
|
+
});
|
|
2597
2613
|
app.post(STATEFUL_INTERNAL_OPERATIONS_ROUTE, async (c) => {
|
|
2598
2614
|
let rawBodyText = "";
|
|
2599
2615
|
let rawBody;
|
package/package.json
CHANGED
package/src/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl
CHANGED
|
@@ -116,7 +116,7 @@ Add ONE guard step per stale-if-error probe:
|
|
|
116
116
|
{
|
|
117
117
|
operationId: "list-items",
|
|
118
118
|
status: "degraded",
|
|
119
|
-
reasonCode: "
|
|
119
|
+
reasonCode: "served_stale_cache",
|
|
120
120
|
reasonKey: "health.operations.list-items.dense-district.servedStaleCache",
|
|
121
121
|
},
|
|
122
122
|
],
|
|
@@ -128,9 +128,12 @@ Add ONE guard step per stale-if-error probe:
|
|
|
128
128
|
- `degraded`, not `down`: the cache policy is behaving as promised and callers
|
|
129
129
|
are still served; it is the upstream that is unavailable, and that has to be
|
|
130
130
|
visible instead of green.
|
|
131
|
-
- `
|
|
132
|
-
|
|
133
|
-
|
|
131
|
+
- `served_stale_cache`, not `expected_absence`: the platform publishes a
|
|
132
|
+
stale serve as `degraded` (an upstream fault that opens incidents), while
|
|
133
|
+
`expected_absence` is published as a non-incident status — neither ok nor
|
|
134
|
+
degraded, excluded from uptime — for an absence the provider declares in
|
|
135
|
+
advance (no alert today, a closed shelf, a feed outside its hours). The
|
|
136
|
+
tenant-facing wording lives in the localized `reasonKey`.
|
|
134
137
|
- Place the freshness guard BEFORE any row or emptiness guard. A stale serve
|
|
135
138
|
that happens to be empty would otherwise be attributed to the emptiness
|
|
136
139
|
reason, which is a confident answer to the wrong question.
|
package/src/health-scenario.ts
CHANGED
|
@@ -102,11 +102,7 @@ export type CredentialReference = z.infer<typeof credentialReferenceSchema>;
|
|
|
102
102
|
export type EstablishedConnectionReference = CredentialReference & { field: "connection" };
|
|
103
103
|
export type AttemptReference = z.infer<typeof attemptReferenceSchema>;
|
|
104
104
|
export type CandidateReference = z.infer<typeof candidateReferenceSchema>;
|
|
105
|
-
export type Reference =
|
|
106
|
-
| StepReference
|
|
107
|
-
| CredentialReference
|
|
108
|
-
| AttemptReference
|
|
109
|
-
| CandidateReference;
|
|
105
|
+
export type Reference = StepReference | CredentialReference | AttemptReference | CandidateReference;
|
|
110
106
|
const referenceSchema = z.discriminatedUnion("namespace", [
|
|
111
107
|
stepReferenceSchema,
|
|
112
108
|
credentialReferenceSchema,
|
|
@@ -217,15 +213,31 @@ export const SafeRegexSchema = z
|
|
|
217
213
|
.strict()
|
|
218
214
|
.superRefine((value, ctx) => {
|
|
219
215
|
if (/\\[1-9kg]/.test(value.pattern))
|
|
220
|
-
ctx.addIssue({
|
|
216
|
+
ctx.addIssue({
|
|
217
|
+
code: "custom",
|
|
218
|
+
path: ["pattern"],
|
|
219
|
+
message: "backreferences are not supported by RE2",
|
|
220
|
+
});
|
|
221
221
|
if (/\(\?/.test(value.pattern))
|
|
222
|
-
ctx.addIssue({
|
|
222
|
+
ctx.addIssue({
|
|
223
|
+
code: "custom",
|
|
224
|
+
path: ["pattern"],
|
|
225
|
+
message: "extended groups are not supported by RE2",
|
|
226
|
+
});
|
|
223
227
|
if (containsNestedRegexQuantifier(value.pattern))
|
|
224
|
-
ctx.addIssue({
|
|
228
|
+
ctx.addIssue({
|
|
229
|
+
code: "custom",
|
|
230
|
+
path: ["pattern"],
|
|
231
|
+
message: "nested regex quantifiers are not permitted",
|
|
232
|
+
});
|
|
225
233
|
try {
|
|
226
234
|
new RegExp(value.pattern, value.flags);
|
|
227
235
|
} catch {
|
|
228
|
-
ctx.addIssue({
|
|
236
|
+
ctx.addIssue({
|
|
237
|
+
code: "custom",
|
|
238
|
+
path: ["pattern"],
|
|
239
|
+
message: "pattern must be valid regular-expression syntax",
|
|
240
|
+
});
|
|
229
241
|
}
|
|
230
242
|
});
|
|
231
243
|
|
|
@@ -532,7 +544,39 @@ const stepBaseSchema = z
|
|
|
532
544
|
timeoutMs: finiteInt(1, 600_000).optional(),
|
|
533
545
|
})
|
|
534
546
|
.strict();
|
|
535
|
-
|
|
547
|
+
/**
|
|
548
|
+
* Why a guard stopped the scenario. The platform decides what each reason
|
|
549
|
+
* publishes; the attribution's `status` is only the scenario rollup.
|
|
550
|
+
*
|
|
551
|
+
* - `expected_absence`: the upstream answered exactly as this provider says it
|
|
552
|
+
* does when there is nothing to return — no weather alert today, an empty
|
|
553
|
+
* shelf, a transit feed outside its operating hours. Published as the
|
|
554
|
+
* non-incident status `expected_absence`: neither ok nor degraded, excluded
|
|
555
|
+
* from uptime, never an incident. Use it only for an absence the provider
|
|
556
|
+
* can declare in advance. Data that must be there (a product that must have
|
|
557
|
+
* options) is a failing assertion, not an expected absence.
|
|
558
|
+
* - `served_stale_cache`: the operation step was answered from the
|
|
559
|
+
* stale-if-error window (`served_stale_cache` on the step result) instead
|
|
560
|
+
* of a live read. A measured upstream verdict: published `degraded`, opens
|
|
561
|
+
* incidents.
|
|
562
|
+
*/
|
|
563
|
+
export type GuardReasonCode = "expected_absence" | "served_stale_cache";
|
|
564
|
+
/**
|
|
565
|
+
* Every member of the union, pinned in both directions: a code missing here
|
|
566
|
+
* fails `satisfies Record<GuardReasonCode, true>`, a code added here without
|
|
567
|
+
* the union fails the excess-property check. The union itself stays a
|
|
568
|
+
* literal so the type-only entry points (contract, server, testing) do not
|
|
569
|
+
* have to re-export the runtime tuple.
|
|
570
|
+
*/
|
|
571
|
+
const GUARD_REASON_CODE_SET = {
|
|
572
|
+
expected_absence: true,
|
|
573
|
+
served_stale_cache: true,
|
|
574
|
+
} as const satisfies Record<GuardReasonCode, true>;
|
|
575
|
+
/** The closed vocabulary as a non-empty tuple, for schema and lint consumers. */
|
|
576
|
+
export const GUARD_REASON_CODES = Object.keys(GUARD_REASON_CODE_SET) as [
|
|
577
|
+
GuardReasonCode,
|
|
578
|
+
...GuardReasonCode[],
|
|
579
|
+
];
|
|
536
580
|
export type GuardAttribution = {
|
|
537
581
|
operationId: string;
|
|
538
582
|
status: "degraded";
|
|
@@ -562,7 +606,10 @@ export type OperationResult = {
|
|
|
562
606
|
* fails the step rather than projecting as fresh.
|
|
563
607
|
*
|
|
564
608
|
* Providers that serve stale-if-error should guard on it, so an outage the
|
|
565
|
-
* cache is absorbing stops reading as green
|
|
609
|
+
* cache is absorbing stops reading as green. Attribute it
|
|
610
|
+
* `served_stale_cache`, not `expected_absence`: a stale serve is an upstream
|
|
611
|
+
* fault the platform publishes as `degraded`, while an expected absence is
|
|
612
|
+
* published as a non-incident status.
|
|
566
613
|
*
|
|
567
614
|
* ```ts
|
|
568
615
|
* {
|
|
@@ -576,7 +623,7 @@ export type OperationResult = {
|
|
|
576
623
|
* expected: true,
|
|
577
624
|
* },
|
|
578
625
|
* onFail: {
|
|
579
|
-
* attribute: [{ operationId, status: "degraded", reasonCode: "
|
|
626
|
+
* attribute: [{ operationId, status: "degraded", reasonCode: "served_stale_cache", reasonKey }],
|
|
580
627
|
* stop: "scenario",
|
|
581
628
|
* },
|
|
582
629
|
* }
|
|
@@ -601,7 +648,7 @@ const attributionSchema = z
|
|
|
601
648
|
.object({
|
|
602
649
|
operationId: z.string().min(1),
|
|
603
650
|
status: z.literal("degraded"),
|
|
604
|
-
reasonCode: z.
|
|
651
|
+
reasonCode: z.enum(GUARD_REASON_CODES),
|
|
605
652
|
reasonKey: z.string().min(1),
|
|
606
653
|
})
|
|
607
654
|
.strict();
|
package/src/index.ts
CHANGED
package/src/lint.ts
CHANGED
|
@@ -3306,6 +3306,35 @@ function resultBindingChecksFreshness(
|
|
|
3306
3306
|
return false;
|
|
3307
3307
|
}
|
|
3308
3308
|
|
|
3309
|
+
/**
|
|
3310
|
+
* Whether a freshness guard for this operation still attributes
|
|
3311
|
+
* `expected_absence` — the only code the attribution schema admitted before
|
|
3312
|
+
* `served_stale_cache` existed, so earlier guidance recommended it. It IS
|
|
3313
|
+
* freshness coverage (the guard runs and stops the scenario), which is why
|
|
3314
|
+
* `scenarioChecksFreshness` accepts it; but the platform publishes
|
|
3315
|
+
* `expected_absence` as a non-incident status, so the stale serve it catches
|
|
3316
|
+
* would read as "nothing to return" instead of `degraded`.
|
|
3317
|
+
*/
|
|
3318
|
+
function freshnessGuardAttributesExpectedAbsence(scenario: unknown, operationId: string): boolean {
|
|
3319
|
+
if (!isLintRecord(scenario) || !Array.isArray(scenario.steps)) return false;
|
|
3320
|
+
const bindings = operationStepBindings(scenario.steps, operationId);
|
|
3321
|
+
if (bindings.size === 0) return false;
|
|
3322
|
+
return scenario.steps.some(
|
|
3323
|
+
(step) =>
|
|
3324
|
+
isLintRecord(step) &&
|
|
3325
|
+
step.kind === "guard" &&
|
|
3326
|
+
rejectsStaleServe(step.condition, bindings) &&
|
|
3327
|
+
isLintRecord(step.onFail) &&
|
|
3328
|
+
Array.isArray(step.onFail.attribute) &&
|
|
3329
|
+
step.onFail.attribute.some(
|
|
3330
|
+
(entry) =>
|
|
3331
|
+
isLintRecord(entry) &&
|
|
3332
|
+
entry.operationId === operationId &&
|
|
3333
|
+
entry.reasonCode === "expected_absence",
|
|
3334
|
+
),
|
|
3335
|
+
);
|
|
3336
|
+
}
|
|
3337
|
+
|
|
3309
3338
|
/** How many uncovered probes the stale-serve message names before eliding. */
|
|
3310
3339
|
const STALE_SERVE_MESSAGE_PROBE_LIMIT = 8;
|
|
3311
3340
|
|
|
@@ -3345,6 +3374,7 @@ function lintHealthCheckMonitorability(provider: {
|
|
|
3345
3374
|
for (const [operationKey, operation] of Object.entries(provider.operations ?? {})) {
|
|
3346
3375
|
const cases = readHealthCheckCases(operation.healthCheck);
|
|
3347
3376
|
const unmonitored: string[] = [];
|
|
3377
|
+
const absenceAttributed: string[] = [];
|
|
3348
3378
|
for (const [caseIndex, healthCase] of cases.entries()) {
|
|
3349
3379
|
const caseLabel =
|
|
3350
3380
|
typeof healthCase.name === "string" && healthCase.name.length > 0
|
|
@@ -3358,6 +3388,17 @@ function lintHealthCheckMonitorability(provider: {
|
|
|
3358
3388
|
uncovered.push(`${operationKey} ${caseLabel}`);
|
|
3359
3389
|
uncoveredByOperation.set(operationKey, uncovered);
|
|
3360
3390
|
}
|
|
3391
|
+
if (freshnessGuardAttributesExpectedAbsence(healthCase.scenario, operationKey)) {
|
|
3392
|
+
absenceAttributed.push(caseLabel);
|
|
3393
|
+
}
|
|
3394
|
+
}
|
|
3395
|
+
if (absenceAttributed.length > 0) {
|
|
3396
|
+
diagnostics.push({
|
|
3397
|
+
rule: "health-check-stale-serve-reason",
|
|
3398
|
+
level: "warn",
|
|
3399
|
+
field: `operations.${operationKey}.healthCheck.cases`,
|
|
3400
|
+
message: `${providerLabel} operation "${operationKey}" health-check ${absenceAttributed.length === 1 ? "case" : "cases"} ${absenceAttributed.join(", ")} guard ${SERVED_STALE_CACHE_OPERAND} but attribute reasonCode "expected_absence". The platform publishes expected_absence as a non-incident status (neither ok nor degraded, excluded from uptime and incidents), so an upstream outage this guard catches would read as "nothing to return" instead of degraded. Attribute reasonCode "served_stale_cache" instead; the reasonKey and its copy can stay.`,
|
|
3401
|
+
});
|
|
3361
3402
|
}
|
|
3362
3403
|
if (unmonitored.length > 0) {
|
|
3363
3404
|
diagnostics.push({
|
|
@@ -3413,7 +3454,7 @@ function lintHealthCheckMonitorability(provider: {
|
|
|
3413
3454
|
rule: "health-check-stale-serve-unguarded",
|
|
3414
3455
|
level: "warn",
|
|
3415
3456
|
field: "healthCheck",
|
|
3416
|
-
message: `${providerLabel} serves stale-if-error (staleIfErrorMs in ${[...sources].sort().join(", ")}) and these health-check cases never check ${SERVED_STALE_CACHE_OPERAND}: ${named}. During an upstream outage the cache answers HTTP 200 with a schema-valid body, so every clause over status_code and data still passes and those probes report ok for the whole stale window. Add one guard step per affected probe on { ref: { namespace: "steps", binding: "<operation step result>", path: ["${SERVED_STALE_CACHE_OPERAND}"] } } attributing status "degraded" with reasonCode "expected_absence", placed before any row or emptiness guard.${caveat}`,
|
|
3457
|
+
message: `${providerLabel} serves stale-if-error (staleIfErrorMs in ${[...sources].sort().join(", ")}) and these health-check cases never check ${SERVED_STALE_CACHE_OPERAND}: ${named}. During an upstream outage the cache answers HTTP 200 with a schema-valid body, so every clause over status_code and data still passes and those probes report ok for the whole stale window. Add one guard step per affected probe on { ref: { namespace: "steps", binding: "<operation step result>", path: ["${SERVED_STALE_CACHE_OPERAND}"] } } attributing status "degraded" with reasonCode "served_stale_cache" (a stale serve is an upstream fault; "expected_absence" is the non-incident status for an absence the provider declares in advance), placed before any row or emptiness guard.${caveat}`,
|
|
3417
3458
|
});
|
|
3418
3459
|
return diagnostics;
|
|
3419
3460
|
}
|