@bitbaum/ai-kit 0.6.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +100 -14
- package/dist/complete.d.ts +160 -0
- package/dist/complete.js +237 -0
- package/dist/index.d.ts +16 -7
- package/dist/index.js +16 -7
- package/dist/liveness.d.ts +112 -0
- package/dist/liveness.js +198 -0
- package/package.json +9 -7
- package/src/complete.ts +340 -0
- package/src/index.ts +33 -7
- package/src/liveness.ts +268 -0
package/src/complete.ts
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The call itself — the one thing this package refused to ship, and the reason
|
|
3
|
+
* the rest of it went unused.
|
|
4
|
+
*
|
|
5
|
+
* ── WHY THIS REVERSES A STATED RULE ──────────────────────────────────────────
|
|
6
|
+
* Every module here was written against a real outage, and every one of them is
|
|
7
|
+
* correct. None of that reached the apps that were actually failing. Measured
|
|
8
|
+
* across the fleet on 2026-09-05:
|
|
9
|
+
*
|
|
10
|
+
* this package's decisions adopted by 2 repos
|
|
11
|
+
* `ai-forms`, which ships a working handler adopted by 5 repos
|
|
12
|
+
* hand-rolled LLM clients still in service 8, ~1400-1700 lines each
|
|
13
|
+
*
|
|
14
|
+
* The pattern is not about quality, it is about shape. `ai-forms` was adopted
|
|
15
|
+
* because `createFormAssistHandler` does the job; this package was not, because
|
|
16
|
+
* it hands back advice the caller must then wire up. The old rule — "every app
|
|
17
|
+
* has its own calling conventions, replacing them is a rewrite rather than an
|
|
18
|
+
* adoption" — describes the duplication accurately and then protects it. The
|
|
19
|
+
* conventions differ because nothing ever offered to own them.
|
|
20
|
+
*
|
|
21
|
+
* The cost of that is not theoretical. Of the 8 hand-rolled clients, 2 tell the
|
|
22
|
+
* three kinds of 429 apart; the other 6 treat a spent daily budget as a busy
|
|
23
|
+
* minute — `limits.ts` has explained why that is harmful since 2026-08-14, in a
|
|
24
|
+
* module those 6 apps do not import. `tryChain` says it plainly: a chain nobody
|
|
25
|
+
* walks is a list, not a fallback. A decision nobody calls is a comment.
|
|
26
|
+
*
|
|
27
|
+
* So: this owns the fetch. `tryChain` stays for callers with a genuinely
|
|
28
|
+
* unusual request to make; this is the answer for everyone else.
|
|
29
|
+
*
|
|
30
|
+
* ── WHAT IT KNOWS THAT A HAND-ROLLED LOOP DOES NOT ───────────────────────────
|
|
31
|
+
* Walking the chain is the easy half. Three behaviours below are the ones every
|
|
32
|
+
* hand-rolled client in this fleet got wrong, each traced to an incident:
|
|
33
|
+
*
|
|
34
|
+
* HTTP 200 IS NOT SUCCESS. `nvidia/nemotron-nano-12b-v2-vl` returns 200 with
|
|
35
|
+
* empty content, and `gemini-2.5-flash` does the same after a tool call — it
|
|
36
|
+
* spends its whole budget on internal thinking and emits no text part. A
|
|
37
|
+
* client that checks `res.ok` returns "" to the user and reports success, so
|
|
38
|
+
* the chain never advances and health stays green through a total outage.
|
|
39
|
+
* Empty content is a FAILURE here, and it demotes to the next link.
|
|
40
|
+
*
|
|
41
|
+
* A DAILY 429 CONDEMNS THE WHOLE VENDOR, not one model. The budget is
|
|
42
|
+
* org-wide and shared across models, so every remaining link at that provider
|
|
43
|
+
* is already dead. Walking them costs a dead round trip each and reaches the
|
|
44
|
+
* same failure. They are skipped.
|
|
45
|
+
*
|
|
46
|
+
* A SIZE 429 ENDS THE WALK. One request exceeded the entire per-minute
|
|
47
|
+
* allowance; the next model down has a SMALLER ceiling (measured: 12000 TPM
|
|
48
|
+
* vs 6000), so demoting makes it strictly worse. The only cure is a shorter
|
|
49
|
+
* prompt, and the caller is told exactly that instead of watching the chain
|
|
50
|
+
* burn itself down to reach a worse version of the same error.
|
|
51
|
+
*
|
|
52
|
+
* ── AND ONE IT INHERITS ──────────────────────────────────────────────────────
|
|
53
|
+
* The response BODY is kept in every error. A status-only message ("groq 429")
|
|
54
|
+
* makes an exhausted day indistinguishable from a momentary burst, and the
|
|
55
|
+
* obvious remedy for the latter — wait and retry — can never work for the
|
|
56
|
+
* former. That misdiagnosis cost an hour once; it is not free to repeat.
|
|
57
|
+
*/
|
|
58
|
+
|
|
59
|
+
import { type Env, type Link, type Provider, chainFrom, freeChain, usableChain } from "./chain.js";
|
|
60
|
+
import { ChainExhaustedError, type ChainAttemptFailure } from "./attempt.js";
|
|
61
|
+
import type { HealthTracker } from "./health.js";
|
|
62
|
+
import { classifyRateLimit, retryAfterSeconds, type RateLimitKind } from "./limits.js";
|
|
63
|
+
|
|
64
|
+
/** One message in the OpenAI chat-completions shape every provider here speaks. */
|
|
65
|
+
export interface ChatMessage {
|
|
66
|
+
role: "system" | "user" | "assistant" | "tool";
|
|
67
|
+
content: string;
|
|
68
|
+
/** Present on `role: "tool"` replies; passed through untouched. */
|
|
69
|
+
tool_call_id?: string;
|
|
70
|
+
name?: string;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* A tool call the model asked for, normalised across the two protocols models
|
|
75
|
+
* actually answer on.
|
|
76
|
+
*
|
|
77
|
+
* Both exist in the default chain: of nine free models probed live, four
|
|
78
|
+
* answered with native `tool_calls` and five only in text. Callers get the
|
|
79
|
+
* native shape here; parsing the text protocol is the caller's business,
|
|
80
|
+
* because its convention differs per app.
|
|
81
|
+
*/
|
|
82
|
+
export interface ToolCall {
|
|
83
|
+
id: string;
|
|
84
|
+
name: string;
|
|
85
|
+
/** Raw JSON string as the model emitted it — NOT parsed, because a model can emit invalid JSON and the caller decides what to do about that. */
|
|
86
|
+
args: string;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export interface CompleteOptions {
|
|
90
|
+
messages: ChatMessage[];
|
|
91
|
+
/**
|
|
92
|
+
* Links to try, in order. Defaults to `usableChain(freeChain())` — every free
|
|
93
|
+
* provider that has a key in `env`.
|
|
94
|
+
*/
|
|
95
|
+
chain?: Link[];
|
|
96
|
+
/** Providers to derive the chain from when `chain` is not given. */
|
|
97
|
+
providers?: Provider[];
|
|
98
|
+
/**
|
|
99
|
+
* Start the chain at this model rather than the front, falling through to the
|
|
100
|
+
* rest. The usual home for an app's "use this model" env var.
|
|
101
|
+
*/
|
|
102
|
+
model?: string;
|
|
103
|
+
env?: Env;
|
|
104
|
+
health?: HealthTracker;
|
|
105
|
+
signal?: AbortSignal;
|
|
106
|
+
/**
|
|
107
|
+
* Set this GENEROUSLY, or a healthy model looks dead.
|
|
108
|
+
*
|
|
109
|
+
* The default chain leads with reasoning models, which spend this budget
|
|
110
|
+
* thinking before emitting a visible token. Set it too low and the vendor
|
|
111
|
+
* returns 200 with empty content — which this module correctly treats as a
|
|
112
|
+
* failure and demotes, so a small `maxTokens` silently walks the whole chain
|
|
113
|
+
* and reports every link broken. Measured 2026-09-05: groq/openai/gpt-oss-20b
|
|
114
|
+
* answered EMPTY at 16 and answered correctly at 256, for the same one-word
|
|
115
|
+
* question.
|
|
116
|
+
*/
|
|
117
|
+
maxTokens?: number;
|
|
118
|
+
temperature?: number;
|
|
119
|
+
/** Tool definitions in the OpenAI shape; passed through untouched. */
|
|
120
|
+
tools?: unknown[];
|
|
121
|
+
/** Extra body fields for a vendor-specific parameter. Merged last, so it can override. */
|
|
122
|
+
extraBody?: Record<string, unknown>;
|
|
123
|
+
/** Called on each link's failure before moving on — e.g. to log which id rotted. */
|
|
124
|
+
onLinkFailure?: (link: Link, error: Error) => void;
|
|
125
|
+
/** Injected for tests. Defaults to global `fetch`. */
|
|
126
|
+
fetchImpl?: typeof fetch;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export interface CompleteResult {
|
|
130
|
+
/** The assistant's text. Never empty — an empty completion is treated as a failure. */
|
|
131
|
+
text: string;
|
|
132
|
+
/** `provider/model`, the id worth logging: it says which link actually served the turn. */
|
|
133
|
+
id: string;
|
|
134
|
+
link: Link;
|
|
135
|
+
toolCalls: ToolCall[];
|
|
136
|
+
/** The parsed response body, for a caller that needs a field this does not surface. */
|
|
137
|
+
raw: unknown;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* A link failed in a way that says something about the WALK, not just this link.
|
|
142
|
+
*
|
|
143
|
+
* `kind` is what the walker acts on; it is carried on the error so a caller
|
|
144
|
+
* reading `ChainExhaustedError.failures` can see why the walk stopped where it
|
|
145
|
+
* did rather than inferring it from prose.
|
|
146
|
+
*/
|
|
147
|
+
export class LinkFailure extends Error {
|
|
148
|
+
readonly link: Link;
|
|
149
|
+
readonly status?: number;
|
|
150
|
+
readonly kind?: RateLimitKind;
|
|
151
|
+
readonly retryAfter?: number | null;
|
|
152
|
+
|
|
153
|
+
constructor(
|
|
154
|
+
link: Link,
|
|
155
|
+
message: string,
|
|
156
|
+
init: { status?: number; kind?: RateLimitKind; retryAfter?: number | null } = {},
|
|
157
|
+
) {
|
|
158
|
+
super(message);
|
|
159
|
+
this.name = "LinkFailure";
|
|
160
|
+
this.link = link;
|
|
161
|
+
this.status = init.status;
|
|
162
|
+
this.kind = init.kind;
|
|
163
|
+
this.retryAfter = init.retryAfter;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** `provider/model` — the id worth logging, because the model alone does not say whose meter it drew on. */
|
|
168
|
+
export function linkId(link: Link): string {
|
|
169
|
+
return `${link.provider.id}/${link.model}`;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function firstText(message: Record<string, unknown> | undefined): string {
|
|
173
|
+
if (!message) return "";
|
|
174
|
+
const content = message.content;
|
|
175
|
+
if (typeof content === "string") return content;
|
|
176
|
+
// Some vendors return content as an array of parts. Concatenate the text ones
|
|
177
|
+
// rather than stringifying the array, which would hand the caller JSON.
|
|
178
|
+
if (Array.isArray(content)) {
|
|
179
|
+
return content
|
|
180
|
+
.map((part) =>
|
|
181
|
+
part && typeof part === "object" && typeof (part as { text?: unknown }).text === "string"
|
|
182
|
+
? (part as { text: string }).text
|
|
183
|
+
: "",
|
|
184
|
+
)
|
|
185
|
+
.join("");
|
|
186
|
+
}
|
|
187
|
+
return "";
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function toolCallsFrom(message: Record<string, unknown> | undefined): ToolCall[] {
|
|
191
|
+
const raw = message?.tool_calls;
|
|
192
|
+
if (!Array.isArray(raw)) return [];
|
|
193
|
+
const out: ToolCall[] = [];
|
|
194
|
+
for (const entry of raw) {
|
|
195
|
+
if (!entry || typeof entry !== "object") continue;
|
|
196
|
+
const fn = (entry as { function?: { name?: unknown; arguments?: unknown } }).function;
|
|
197
|
+
if (!fn || typeof fn.name !== "string") continue;
|
|
198
|
+
out.push({
|
|
199
|
+
id: String((entry as { id?: unknown }).id ?? ""),
|
|
200
|
+
name: fn.name,
|
|
201
|
+
args: typeof fn.arguments === "string" ? fn.arguments : JSON.stringify(fn.arguments ?? {}),
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
return out;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Truncated so a failure message stays readable in a log line, but long enough
|
|
209
|
+
* to carry the sentence that matters: Groq states the real reset ~90 characters
|
|
210
|
+
* into a daily-cap body, and cutting before it throws away the one number the
|
|
211
|
+
* user can act on.
|
|
212
|
+
*/
|
|
213
|
+
function excerpt(body: string, limit = 300): string {
|
|
214
|
+
const flat = body.replace(/\s+/g, " ").trim();
|
|
215
|
+
return flat.length > limit ? `${flat.slice(0, limit)}…` : flat;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
async function callLink(
|
|
219
|
+
link: Link,
|
|
220
|
+
options: CompleteOptions,
|
|
221
|
+
key: string,
|
|
222
|
+
): Promise<CompleteResult> {
|
|
223
|
+
const doFetch = options.fetchImpl ?? globalThis.fetch;
|
|
224
|
+
const body: Record<string, unknown> = {
|
|
225
|
+
model: link.model,
|
|
226
|
+
messages: options.messages,
|
|
227
|
+
...(options.maxTokens === undefined ? {} : { max_tokens: options.maxTokens }),
|
|
228
|
+
...(options.temperature === undefined ? {} : { temperature: options.temperature }),
|
|
229
|
+
...(options.tools === undefined ? {} : { tools: options.tools }),
|
|
230
|
+
...options.extraBody,
|
|
231
|
+
};
|
|
232
|
+
|
|
233
|
+
let res: Response;
|
|
234
|
+
try {
|
|
235
|
+
res = await doFetch(`${link.provider.baseUrl}/chat/completions`, {
|
|
236
|
+
method: "POST",
|
|
237
|
+
headers: { "content-type": "application/json", authorization: `Bearer ${key}` },
|
|
238
|
+
body: JSON.stringify(body),
|
|
239
|
+
signal: options.signal,
|
|
240
|
+
});
|
|
241
|
+
} catch (error) {
|
|
242
|
+
// A transport failure (DNS, TLS, abort) is not the vendor's answer, so it
|
|
243
|
+
// carries no rate-limit kind — it demotes like any other link failure.
|
|
244
|
+
throw new LinkFailure(link, `${linkId(link)}: ${(error as Error).message}`);
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
const text = await res.text();
|
|
248
|
+
|
|
249
|
+
if (!res.ok) {
|
|
250
|
+
if (res.status === 429) {
|
|
251
|
+
const kind = classifyRateLimit(text);
|
|
252
|
+
const retryAfter = retryAfterSeconds(text);
|
|
253
|
+
throw new LinkFailure(link, `${linkId(link)}: 429 ${kind} — ${excerpt(text)}`, {
|
|
254
|
+
status: 429,
|
|
255
|
+
kind,
|
|
256
|
+
retryAfter,
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
throw new LinkFailure(link, `${linkId(link)}: ${res.status} — ${excerpt(text)}`, {
|
|
260
|
+
status: res.status,
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
let parsed: unknown;
|
|
265
|
+
try {
|
|
266
|
+
parsed = JSON.parse(text);
|
|
267
|
+
} catch {
|
|
268
|
+
throw new LinkFailure(link, `${linkId(link)}: 200 with unparseable body — ${excerpt(text)}`, {
|
|
269
|
+
status: res.status,
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const choice = (parsed as { choices?: Array<{ message?: Record<string, unknown> }> })
|
|
274
|
+
?.choices?.[0];
|
|
275
|
+
const content = firstText(choice?.message);
|
|
276
|
+
const toolCalls = toolCallsFrom(choice?.message);
|
|
277
|
+
|
|
278
|
+
// A 200 that carries neither text nor a tool call is an outage wearing a
|
|
279
|
+
// success code — see the header. Demote, so the chain gets its chance.
|
|
280
|
+
if (content.trim() === "" && toolCalls.length === 0) {
|
|
281
|
+
throw new LinkFailure(
|
|
282
|
+
link,
|
|
283
|
+
`${linkId(link)}: 200 with empty content — model produced no output`,
|
|
284
|
+
{
|
|
285
|
+
status: res.status,
|
|
286
|
+
},
|
|
287
|
+
);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
return { text: content, id: linkId(link), link, toolCalls, raw: parsed };
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Call the first link that works, and return what it said.
|
|
295
|
+
*
|
|
296
|
+
* Throws `ChainExhaustedError` carrying every link's failure, so a log shows
|
|
297
|
+
* what was actually tried — the failure that explains an outage is usually not
|
|
298
|
+
* the last one.
|
|
299
|
+
*/
|
|
300
|
+
export async function complete(options: CompleteOptions): Promise<CompleteResult> {
|
|
301
|
+
const env = options.env ?? process.env;
|
|
302
|
+
const base = options.chain ?? usableChain(options.providers ?? freeChain(), env);
|
|
303
|
+
const chain = chainFrom(options.model, base);
|
|
304
|
+
|
|
305
|
+
const failures: ChainAttemptFailure[] = [];
|
|
306
|
+
const deadProviders = new Set<string>();
|
|
307
|
+
|
|
308
|
+
for (const link of chain) {
|
|
309
|
+
// A daily cap already condemned this vendor earlier in the walk. Its other
|
|
310
|
+
// models draw on the same exhausted budget, so trying them buys a dead
|
|
311
|
+
// round trip and the identical error.
|
|
312
|
+
if (deadProviders.has(link.provider.id)) continue;
|
|
313
|
+
|
|
314
|
+
const key = env[link.provider.keyEnv]?.trim();
|
|
315
|
+
if (!key) {
|
|
316
|
+
failures.push({ link, message: `${linkId(link)}: no ${link.provider.keyEnv} in env` });
|
|
317
|
+
continue;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
try {
|
|
321
|
+
const result = await callLink(link, options, key);
|
|
322
|
+
options.health?.recordSuccess();
|
|
323
|
+
return result;
|
|
324
|
+
} catch (error) {
|
|
325
|
+
const failure = error as LinkFailure;
|
|
326
|
+
failures.push({ link, message: failure.message });
|
|
327
|
+
options.onLinkFailure?.(link, failure);
|
|
328
|
+
|
|
329
|
+
if (failure.kind === "daily") deadProviders.add(link.provider.id);
|
|
330
|
+
|
|
331
|
+
// Stepping down after a size 429 reaches a model with a smaller ceiling —
|
|
332
|
+
// strictly worse. Stop, and let the caller shorten the prompt.
|
|
333
|
+
if (failure.kind === "size") break;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
const exhausted = new ChainExhaustedError(failures);
|
|
338
|
+
options.health?.recordFailure(exhausted);
|
|
339
|
+
throw exhausted;
|
|
340
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -34,13 +34,20 @@
|
|
|
34
34
|
* retired model id — the exact failure the `chain` and `catalog` modules exist
|
|
35
35
|
* to prevent. "Ration" described one of five modules and buried the other four.
|
|
36
36
|
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
37
|
+
* NOW INCLUDED: an HTTP client. `complete()` makes the call.
|
|
38
|
+
*
|
|
39
|
+
* The old rule was "every app has its own calling conventions, and replacing
|
|
40
|
+
* those is a rewrite rather than an adoption" — accurate about the fleet, and
|
|
41
|
+
* it protected the very duplication it described. The conventions differed
|
|
42
|
+
* because nothing ever offered to own them. Measured 2026-09-05: 8 hand-rolled
|
|
43
|
+
* clients in service, 2 of which tell the three kinds of 429 apart, while this
|
|
44
|
+
* package explained the distinction to the 2 repos that imported it.
|
|
45
|
+
* `ai-forms` — adopted by 5 — is not better code, it is code that does the job
|
|
46
|
+
* rather than advising on it. See `complete.ts` for the full argument.
|
|
47
|
+
*
|
|
48
|
+
* `tryChain` remains for a caller with a genuinely unusual request to make: it
|
|
49
|
+
* walks the chain and lets the caller keep the fetch. `complete` is the answer
|
|
50
|
+
* for everyone else.
|
|
44
51
|
*/
|
|
45
52
|
|
|
46
53
|
export {
|
|
@@ -75,6 +82,16 @@ export {
|
|
|
75
82
|
tryChain,
|
|
76
83
|
} from "./attempt.js";
|
|
77
84
|
|
|
85
|
+
export {
|
|
86
|
+
type ChatMessage,
|
|
87
|
+
type ToolCall,
|
|
88
|
+
type CompleteOptions,
|
|
89
|
+
type CompleteResult,
|
|
90
|
+
LinkFailure,
|
|
91
|
+
complete,
|
|
92
|
+
linkId,
|
|
93
|
+
} from "./complete.js";
|
|
94
|
+
|
|
78
95
|
export {
|
|
79
96
|
type HealthStatus,
|
|
80
97
|
type Health,
|
|
@@ -83,6 +100,15 @@ export {
|
|
|
83
100
|
createHealthTracker,
|
|
84
101
|
} from "./health.js";
|
|
85
102
|
|
|
103
|
+
export {
|
|
104
|
+
type LivenessResult,
|
|
105
|
+
type LivenessOptions,
|
|
106
|
+
type LivenessProbe,
|
|
107
|
+
type AiHealthHandlerOptions,
|
|
108
|
+
createLivenessProbe,
|
|
109
|
+
createAiHealthHandler,
|
|
110
|
+
} from "./liveness.js";
|
|
111
|
+
|
|
86
112
|
export {
|
|
87
113
|
type RateLimitKind,
|
|
88
114
|
classifyRateLimit,
|
package/src/liveness.ts
ADDED
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Can this deployment reach a model RIGHT NOW?
|
|
3
|
+
*
|
|
4
|
+
* ── The gap this closes ──────────────────────────────────────────────────────
|
|
5
|
+
* A health route that reports `createHealthTracker().getHealth()` reports what
|
|
6
|
+
* happened the last time the app happened to call a model. Straight after a
|
|
7
|
+
* deploy that is `"unknown"` — no calls yet — and "unknown" is what it stays
|
|
8
|
+
* until real traffic arrives. So the one question a deploy needs answered ("did
|
|
9
|
+
* I just ship a working AI path?") is exactly the one it cannot answer.
|
|
10
|
+
*
|
|
11
|
+
* Observed 2026-09-05, converting the first app to `complete()`: the deploy was
|
|
12
|
+
* green, the bundle provably contained the new code, both keys were present,
|
|
13
|
+
* `/api/health` returned 200 — and `llm.status` was `"unknown"`. Every signal
|
|
14
|
+
* available said "probably fine". The only paths that would have produced a
|
|
15
|
+
* real answer were an admin-authenticated form and two cron jobs that EMAIL
|
|
16
|
+
* REAL USERS. Verifying a deploy must never require spamming somebody.
|
|
17
|
+
*
|
|
18
|
+
* ── Why it is a probe and not a passive read ─────────────────────────────────
|
|
19
|
+
* Absence of failure is not evidence of success. A tracker that has recorded
|
|
20
|
+
* nothing looks identical whether the chain is perfect or every key is missing.
|
|
21
|
+
* The only thing that distinguishes them is making a call — so this makes one,
|
|
22
|
+
* deliberately, on demand.
|
|
23
|
+
*
|
|
24
|
+
* ── Why it must be gated and cached ──────────────────────────────────────────
|
|
25
|
+
* This spends real tokens from a free daily budget shared across the whole org.
|
|
26
|
+
* An ungated probe on a health route is a self-inflicted outage: a monitor
|
|
27
|
+
* polling every 30s would drain a 100k/day allowance and take the app's actual
|
|
28
|
+
* AI features down with it. So:
|
|
29
|
+
*
|
|
30
|
+
* - a probe runs ONLY when asked for explicitly (`?probe=1`) AND the caller
|
|
31
|
+
* proves it is allowed to (a secret), never on an ordinary health poll;
|
|
32
|
+
* - a successful probe is CACHED for `minIntervalMs` (default 10 minutes),
|
|
33
|
+
* so even an authorised caller in a retry loop cannot burn the budget. The
|
|
34
|
+
* cached answer is returned with `cached: true` and the age, because a
|
|
35
|
+
* nine-minute-old success is a different claim from a fresh one and the
|
|
36
|
+
* reader deserves to know which they got.
|
|
37
|
+
*
|
|
38
|
+
* The prompt is deliberately tiny — a handful of tokens — because the question
|
|
39
|
+
* is "does the pipe carry water", not "is the model any good".
|
|
40
|
+
*/
|
|
41
|
+
|
|
42
|
+
import { complete, type CompleteOptions } from "./complete.js";
|
|
43
|
+
import { ChainExhaustedError } from "./attempt.js";
|
|
44
|
+
import type { HealthTracker } from "./health.js";
|
|
45
|
+
|
|
46
|
+
export interface LivenessResult {
|
|
47
|
+
/** Did a model answer? */
|
|
48
|
+
ok: boolean;
|
|
49
|
+
/** `provider/model` that served it, when one did. */
|
|
50
|
+
servedBy?: string;
|
|
51
|
+
/** What the model actually said, trimmed — proof of a real generation, not a 200. */
|
|
52
|
+
answer?: string;
|
|
53
|
+
/** Round-trip milliseconds for a fresh probe. */
|
|
54
|
+
ms?: number;
|
|
55
|
+
/** True when this is a remembered result rather than a call made just now. */
|
|
56
|
+
cached: boolean;
|
|
57
|
+
/** Age of a cached result, in milliseconds. */
|
|
58
|
+
cachedAgeMs?: number;
|
|
59
|
+
/** Every link's failure, when the whole chain was exhausted. */
|
|
60
|
+
failures?: string[];
|
|
61
|
+
/** Why no call was attempted at all (no keys, no links). */
|
|
62
|
+
skipped?: string;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface LivenessOptions extends Omit<
|
|
66
|
+
CompleteOptions,
|
|
67
|
+
"messages" | "maxTokens" | "temperature"
|
|
68
|
+
> {
|
|
69
|
+
/**
|
|
70
|
+
* Don't call again within this window; return the last successful result.
|
|
71
|
+
* Default 10 minutes. Set 0 to disable caching — only for a test.
|
|
72
|
+
*/
|
|
73
|
+
minIntervalMs?: number;
|
|
74
|
+
/** Injected for tests. Defaults to `Date.now`. */
|
|
75
|
+
now?: () => number;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const DEFAULT_MIN_INTERVAL_MS = 10 * 60 * 1000;
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* The budget the probe asks for.
|
|
82
|
+
*
|
|
83
|
+
* NOT small, despite the tiny prompt. The chain leads with REASONING models,
|
|
84
|
+
* which spend this on hidden thinking before emitting a visible token: measured
|
|
85
|
+
* 2026-09-05, groq/openai/gpt-oss-20b answered EMPTY at 16 and correctly at 256
|
|
86
|
+
* for the same one-word question. An empty completion is a failure here (see
|
|
87
|
+
* complete.ts), so a mean budget would make a perfectly healthy deployment
|
|
88
|
+
* report itself dead — the exact false alarm this module exists to prevent.
|
|
89
|
+
*/
|
|
90
|
+
const PROBE_MAX_TOKENS = 256;
|
|
91
|
+
|
|
92
|
+
/** A question with one short right answer, cheap to ask and easy to sanity-check. */
|
|
93
|
+
const PROBE_MESSAGES = [
|
|
94
|
+
{ role: "system" as const, content: "Answer with a single word, no punctuation." },
|
|
95
|
+
{ role: "user" as const, content: "What colour is a clear midday sky? Answer in one word." },
|
|
96
|
+
];
|
|
97
|
+
|
|
98
|
+
export interface LivenessProbe {
|
|
99
|
+
/** Make a call (or return a cached success). */
|
|
100
|
+
run(): Promise<LivenessResult>;
|
|
101
|
+
/** Forget any cached success — the next `run` will really call. */
|
|
102
|
+
reset(): void;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Build a probe with its own cache.
|
|
107
|
+
*
|
|
108
|
+
* The cache lives on the instance rather than in a module global so that two
|
|
109
|
+
* apps in one process, or a test, cannot silently share (and satisfy) each
|
|
110
|
+
* other's probe.
|
|
111
|
+
*/
|
|
112
|
+
export function createLivenessProbe(options: LivenessOptions = {}): LivenessProbe {
|
|
113
|
+
const minIntervalMs = options.minIntervalMs ?? DEFAULT_MIN_INTERVAL_MS;
|
|
114
|
+
const now = options.now ?? Date.now;
|
|
115
|
+
|
|
116
|
+
let lastOk: { at: number; result: LivenessResult } | null = null;
|
|
117
|
+
|
|
118
|
+
return {
|
|
119
|
+
reset() {
|
|
120
|
+
lastOk = null;
|
|
121
|
+
},
|
|
122
|
+
|
|
123
|
+
async run(): Promise<LivenessResult> {
|
|
124
|
+
if (lastOk && minIntervalMs > 0) {
|
|
125
|
+
const age = now() - lastOk.at;
|
|
126
|
+
if (age < minIntervalMs) {
|
|
127
|
+
return { ...lastOk.result, cached: true, cachedAgeMs: age };
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const started = now();
|
|
132
|
+
try {
|
|
133
|
+
const result = await complete({
|
|
134
|
+
...options,
|
|
135
|
+
messages: PROBE_MESSAGES,
|
|
136
|
+
maxTokens: PROBE_MAX_TOKENS,
|
|
137
|
+
temperature: 0,
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
const answer = result.text.trim();
|
|
141
|
+
const fresh: LivenessResult = {
|
|
142
|
+
ok: true,
|
|
143
|
+
servedBy: result.id,
|
|
144
|
+
answer,
|
|
145
|
+
ms: now() - started,
|
|
146
|
+
cached: false,
|
|
147
|
+
};
|
|
148
|
+
lastOk = { at: now(), result: fresh };
|
|
149
|
+
return fresh;
|
|
150
|
+
} catch (error) {
|
|
151
|
+
// A failure is deliberately NOT cached. Caching it would keep reporting
|
|
152
|
+
// an outage after the vendor recovered, and the whole point is to tell
|
|
153
|
+
// the truth about right now.
|
|
154
|
+
if (error instanceof ChainExhaustedError) {
|
|
155
|
+
return {
|
|
156
|
+
ok: false,
|
|
157
|
+
cached: false,
|
|
158
|
+
ms: now() - started,
|
|
159
|
+
failures: error.failures.map((f) => f.message),
|
|
160
|
+
...(error.failures.length === 0
|
|
161
|
+
? { skipped: "No usable link — every provider is missing its key or has no models." }
|
|
162
|
+
: {}),
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
return {
|
|
166
|
+
ok: false,
|
|
167
|
+
cached: false,
|
|
168
|
+
ms: now() - started,
|
|
169
|
+
failures: [error instanceof Error ? error.message : String(error)],
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
},
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
export interface AiHealthHandlerOptions extends LivenessOptions {
|
|
177
|
+
/**
|
|
178
|
+
* Shared secret authorising a probe. Compared against the `x-probe-secret`
|
|
179
|
+
* header or a `secret` query parameter.
|
|
180
|
+
*
|
|
181
|
+
* When absent, the handler NEVER probes — it only reports passive health.
|
|
182
|
+
* That default is deliberate: an app that forgets to configure a secret gets
|
|
183
|
+
* a route that cannot spend money, rather than an open endpoint that can.
|
|
184
|
+
*/
|
|
185
|
+
secret?: string;
|
|
186
|
+
/** Passive health to report alongside. Optional. */
|
|
187
|
+
health?: HealthTracker;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* A framework-neutral `Request -> Response` handler for an AI health route.
|
|
192
|
+
*
|
|
193
|
+
* Web-standard on purpose: Next's App Router, Hono, Deno and Bun all accept
|
|
194
|
+
* this shape directly, so adopting it is an export line rather than a port.
|
|
195
|
+
*
|
|
196
|
+
* GET /api/health/ai passive — what happened last time. Free.
|
|
197
|
+
* GET /api/health/ai?probe=1 makes a real call. Requires the secret.
|
|
198
|
+
*
|
|
199
|
+
* A probe without a valid secret is 401 and does NOT fall back to probing.
|
|
200
|
+
*
|
|
201
|
+
* Status codes are chosen so an uptime monitor can watch this URL directly:
|
|
202
|
+
* 200 when the answer is good, 503 when a probe was attempted and the chain
|
|
203
|
+
* could not answer.
|
|
204
|
+
*/
|
|
205
|
+
export function createAiHealthHandler(
|
|
206
|
+
options: AiHealthHandlerOptions = {},
|
|
207
|
+
): (request: Request) => Promise<Response> {
|
|
208
|
+
const { secret, health, ...probeOptions } = options;
|
|
209
|
+
const probe = createLivenessProbe(probeOptions);
|
|
210
|
+
|
|
211
|
+
return async function handler(request: Request): Promise<Response> {
|
|
212
|
+
const url = new URL(request.url);
|
|
213
|
+
const wantsProbe = url.searchParams.get("probe") === "1";
|
|
214
|
+
const offered =
|
|
215
|
+
request.headers.get("x-probe-secret") ?? url.searchParams.get("secret") ?? undefined;
|
|
216
|
+
|
|
217
|
+
const passive = health ? { health: health.getHealth() } : {};
|
|
218
|
+
|
|
219
|
+
if (!wantsProbe) {
|
|
220
|
+
return json(200, { probed: false, ...passive });
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// No secret configured means probing is switched off, which is a different
|
|
224
|
+
// answer from "your secret is wrong" — say so, rather than implying the
|
|
225
|
+
// caller could retry with a better credential.
|
|
226
|
+
if (!secret) {
|
|
227
|
+
return json(501, {
|
|
228
|
+
probed: false,
|
|
229
|
+
error: "Probing is not configured on this deployment (no secret set).",
|
|
230
|
+
...passive,
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
if (!offered || !timingSafeEqual(offered, secret)) {
|
|
234
|
+
return json(401, { probed: false, error: "Bad or missing probe secret.", ...passive });
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
const result = await probe.run();
|
|
238
|
+
// Record into passive health too, so one probe also answers the next
|
|
239
|
+
// ordinary health poll — otherwise the probe's knowledge dies with it.
|
|
240
|
+
if (health) {
|
|
241
|
+
if (result.ok) health.recordSuccess();
|
|
242
|
+
else health.recordFailure(new Error(result.failures?.join("; ") ?? "probe failed"));
|
|
243
|
+
}
|
|
244
|
+
return json(result.ok ? 200 : 503, {
|
|
245
|
+
probed: true,
|
|
246
|
+
...result,
|
|
247
|
+
...(health ? { health: health.getHealth() } : {}),
|
|
248
|
+
});
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function json(status: number, body: unknown): Response {
|
|
253
|
+
return new Response(JSON.stringify(body), {
|
|
254
|
+
status,
|
|
255
|
+
headers: { "content-type": "application/json", "cache-control": "no-store" },
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Constant-time comparison, so a wrong secret cannot be discovered one
|
|
261
|
+
* character at a time by timing the 401.
|
|
262
|
+
*/
|
|
263
|
+
function timingSafeEqual(a: string, b: string): boolean {
|
|
264
|
+
if (a.length !== b.length) return false;
|
|
265
|
+
let diff = 0;
|
|
266
|
+
for (let i = 0; i < a.length; i += 1) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
|
|
267
|
+
return diff === 0;
|
|
268
|
+
}
|