@bitbaum/ai-kit 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +216 -0
- package/dist/attempt.d.ts +48 -0
- package/dist/attempt.js +59 -0
- package/dist/catalog.d.ts +65 -0
- package/dist/catalog.js +115 -0
- package/dist/chain.d.ts +204 -0
- package/dist/chain.js +261 -0
- package/dist/fair-share.d.ts +120 -0
- package/dist/fair-share.js +127 -0
- package/dist/forms.d.ts +15 -0
- package/dist/forms.js +15 -0
- package/dist/grounding/contract.d.ts +101 -0
- package/dist/grounding/contract.js +138 -0
- package/dist/grounding/facts.d.ts +107 -0
- package/dist/grounding/facts.js +134 -0
- package/dist/grounding/index.d.ts +24 -0
- package/dist/grounding/index.js +24 -0
- package/dist/grounding/verify.d.ts +91 -0
- package/dist/grounding/verify.js +372 -0
- package/dist/health.d.ts +52 -0
- package/dist/health.js +64 -0
- package/dist/index.d.ts +50 -0
- package/dist/index.js +70 -0
- package/dist/limits.d.ts +102 -0
- package/dist/limits.js +136 -0
- package/dist/react.d.ts +8 -0
- package/dist/react.js +8 -0
- package/dist/registry.d.ts +133 -0
- package/dist/registry.js +126 -0
- package/dist/server.d.ts +10 -0
- package/dist/server.js +10 -0
- package/dist-cjs/grounding/contract.js +146 -0
- package/dist-cjs/grounding/facts.js +143 -0
- package/dist-cjs/grounding/index.js +43 -0
- package/dist-cjs/grounding/verify.js +376 -0
- package/dist-cjs/package.json +1 -0
- package/dist-cjs/registry.js +131 -0
- package/package.json +102 -0
- package/src/attempt.ts +82 -0
- package/src/catalog.ts +155 -0
- package/src/chain.ts +318 -0
- package/src/fair-share.ts +183 -0
- package/src/forms.ts +15 -0
- package/src/grounding/contract.ts +176 -0
- package/src/grounding/facts.ts +170 -0
- package/src/grounding/index.ts +50 -0
- package/src/grounding/verify.ts +429 -0
- package/src/health.ts +92 -0
- package/src/index.ts +124 -0
- package/src/limits.ts +137 -0
- package/src/react.ts +8 -0
- package/src/registry.ts +207 -0
- package/src/server.ts +10 -0
|
@@ -0,0 +1,376 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.verifyAnswer = verifyAnswer;
|
|
4
|
+
exports.buildRepairPrompt = buildRepairPrompt;
|
|
5
|
+
/**
|
|
6
|
+
* Groundedness verifier — MIRRORED MODULE (see core/README.md).
|
|
7
|
+
*
|
|
8
|
+
* Runs on the generated answer and reports claims the fact set does not support.
|
|
9
|
+
* Deliberately deterministic: no second model call, no embedding round-trip, no
|
|
10
|
+
* added cost or latency. That is a requirement, not a shortcut — this must run
|
|
11
|
+
* on every turn including the free-tier ones, and a verifier that costs a
|
|
12
|
+
* frontier call is one that gets disabled exactly where it is needed most.
|
|
13
|
+
*
|
|
14
|
+
* The insight that makes a cheap check work: fabrication is overwhelmingly
|
|
15
|
+
* NOMINAL. Models invent organisations, titles, people, file paths, phone
|
|
16
|
+
* numbers and dates — tokens that are mechanically recognisable and that must,
|
|
17
|
+
* if genuine, have appeared in the retrieved records or in what the user said.
|
|
18
|
+
* Grammar and hedging are hard to check; proper nouns and digits are easy.
|
|
19
|
+
*
|
|
20
|
+
* Scored against the real failure this was built from, every fabricated claim
|
|
21
|
+
* is caught by the proper-noun or numeric rule:
|
|
22
|
+
*
|
|
23
|
+
* "Ilya Druzhnikov (UZH)" → UZH: novel acronym
|
|
24
|
+
* "Accelerator & Bridge Program Manager" → novel proper-noun run
|
|
25
|
+
* "University of Liechtenstein", "START Summit" → novel proper-noun runs
|
|
26
|
+
* "/opt/fleetcrown/runner/.env" → novel path
|
|
27
|
+
*
|
|
28
|
+
* while the true parts ("Elena Weber SINGA Switzerland", "+41774730093") appear
|
|
29
|
+
* verbatim in the records and pass clean.
|
|
30
|
+
*/
|
|
31
|
+
const facts_js_1 = require("./facts.js");
|
|
32
|
+
/**
|
|
33
|
+
* Words that are capitalised for reasons other than being a proper noun, or
|
|
34
|
+
* that are part of this system's own vocabulary. Kept deliberately small —
|
|
35
|
+
* every entry is a hole in the check, so add only what demonstrably causes
|
|
36
|
+
* false positives, never to silence a true one.
|
|
37
|
+
*/
|
|
38
|
+
const COMMON = new Set([
|
|
39
|
+
// Sentence/structural
|
|
40
|
+
"the",
|
|
41
|
+
"a",
|
|
42
|
+
"an",
|
|
43
|
+
"and",
|
|
44
|
+
"or",
|
|
45
|
+
"but",
|
|
46
|
+
"if",
|
|
47
|
+
"then",
|
|
48
|
+
"so",
|
|
49
|
+
"because",
|
|
50
|
+
"not",
|
|
51
|
+
"this",
|
|
52
|
+
"that",
|
|
53
|
+
"these",
|
|
54
|
+
"those",
|
|
55
|
+
"it",
|
|
56
|
+
"its",
|
|
57
|
+
"your",
|
|
58
|
+
"you",
|
|
59
|
+
"i",
|
|
60
|
+
"we",
|
|
61
|
+
"there",
|
|
62
|
+
"here",
|
|
63
|
+
"what",
|
|
64
|
+
"which",
|
|
65
|
+
"who",
|
|
66
|
+
"when",
|
|
67
|
+
"where",
|
|
68
|
+
"why",
|
|
69
|
+
"how",
|
|
70
|
+
"no",
|
|
71
|
+
"yes",
|
|
72
|
+
"none",
|
|
73
|
+
"nothing",
|
|
74
|
+
"today",
|
|
75
|
+
"tomorrow",
|
|
76
|
+
"yesterday",
|
|
77
|
+
"now",
|
|
78
|
+
"next",
|
|
79
|
+
"last",
|
|
80
|
+
"first",
|
|
81
|
+
"one",
|
|
82
|
+
"two",
|
|
83
|
+
"three",
|
|
84
|
+
"primary",
|
|
85
|
+
"focus",
|
|
86
|
+
"task",
|
|
87
|
+
"tasks",
|
|
88
|
+
"outreach",
|
|
89
|
+
"note",
|
|
90
|
+
"notes",
|
|
91
|
+
"summary",
|
|
92
|
+
"status",
|
|
93
|
+
"update",
|
|
94
|
+
// Days / months — real words, never evidence of a fabricated entity
|
|
95
|
+
"monday",
|
|
96
|
+
"tuesday",
|
|
97
|
+
"wednesday",
|
|
98
|
+
"thursday",
|
|
99
|
+
"friday",
|
|
100
|
+
"saturday",
|
|
101
|
+
"sunday",
|
|
102
|
+
"january",
|
|
103
|
+
"february",
|
|
104
|
+
"march",
|
|
105
|
+
"april",
|
|
106
|
+
"may",
|
|
107
|
+
"june",
|
|
108
|
+
"july",
|
|
109
|
+
"august",
|
|
110
|
+
"september",
|
|
111
|
+
"october",
|
|
112
|
+
"november",
|
|
113
|
+
"december",
|
|
114
|
+
// This system's own nouns
|
|
115
|
+
"loki",
|
|
116
|
+
"cat",
|
|
117
|
+
"fleetcrown",
|
|
118
|
+
"orangecat",
|
|
119
|
+
"not",
|
|
120
|
+
"recorded",
|
|
121
|
+
].map((w) => w.toLowerCase()));
|
|
122
|
+
/** Normalise for containment tests: casefold, collapse punctuation and space. */
|
|
123
|
+
function norm(s) {
|
|
124
|
+
return s
|
|
125
|
+
.toLowerCase()
|
|
126
|
+
.replace(/[^a-z0-9+]+/g, " ")
|
|
127
|
+
.replace(/\s+/g, " ")
|
|
128
|
+
.trim();
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Everything the model was legitimately given this turn: record values, record
|
|
132
|
+
* subjects, and the user's own message (a name the user typed is fair to
|
|
133
|
+
* repeat). This is the corpus a claim must be traceable to.
|
|
134
|
+
*/
|
|
135
|
+
function buildEvidence(facts, userMessage, extra) {
|
|
136
|
+
const parts = [userMessage, ...extra];
|
|
137
|
+
for (const f of facts) {
|
|
138
|
+
parts.push(f.subject, f.kind, f.source);
|
|
139
|
+
for (const v of Object.values(f.fields))
|
|
140
|
+
if (v)
|
|
141
|
+
parts.push(v);
|
|
142
|
+
}
|
|
143
|
+
return norm(parts.join(" "));
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Lowercase words that legitimately sit INSIDE a proper name and must not break
|
|
147
|
+
* it up: "University of Zurich", "Bank für Handel", "Institute for the Study of
|
|
148
|
+
* Complexity". Without these, the run splits at the connector and the check
|
|
149
|
+
* only ever sees the harmless halves ("University", "Zurich") while the actual
|
|
150
|
+
* fabricated entity slips through unnamed.
|
|
151
|
+
*/
|
|
152
|
+
const NAME_CONNECTORS = new Set([
|
|
153
|
+
"of",
|
|
154
|
+
"the",
|
|
155
|
+
"for",
|
|
156
|
+
"and",
|
|
157
|
+
"de",
|
|
158
|
+
"der",
|
|
159
|
+
"des",
|
|
160
|
+
"van",
|
|
161
|
+
"von",
|
|
162
|
+
"du",
|
|
163
|
+
"da",
|
|
164
|
+
"di",
|
|
165
|
+
"für",
|
|
166
|
+
"el",
|
|
167
|
+
"al",
|
|
168
|
+
]);
|
|
169
|
+
/**
|
|
170
|
+
* Named-entity candidates: ALL-CAPS acronyms, capitalised words, and the
|
|
171
|
+
* multi-word runs they form (connectors allowed strictly between two
|
|
172
|
+
* capitalised tokens, never at an edge).
|
|
173
|
+
*
|
|
174
|
+
* Both the run AND its individual tokens are emitted, deliberately. The run
|
|
175
|
+
* catches composite inventions ("University of Zurich") that no single token
|
|
176
|
+
* reveals; the individual tokens catch an invented acronym sitting next to a
|
|
177
|
+
* real name ("Druzhnikov UZH"), where reporting only the run would name the
|
|
178
|
+
* real person in the violation and produce a repair prompt that deletes the
|
|
179
|
+
* true claim along with the false one.
|
|
180
|
+
*
|
|
181
|
+
* Sentence-initial single words are skipped — otherwise "Rotate the key" flags
|
|
182
|
+
* "Rotate". That costs a little recall at sentence starts and removes the
|
|
183
|
+
* dominant source of false positives; a fabricated name at a sentence start is
|
|
184
|
+
* still caught by its remaining tokens.
|
|
185
|
+
*/
|
|
186
|
+
function properNounRuns(text) {
|
|
187
|
+
const out = [];
|
|
188
|
+
// Strip fenced and inline code — quoted identifiers are usually the user's
|
|
189
|
+
// own or a literal under discussion, not a claim about the world.
|
|
190
|
+
const prose = text.replace(/```[\s\S]*?```/g, " ").replace(/`[^`]*`/g, " ");
|
|
191
|
+
for (const sentence of prose.split(/(?<=[.!?:\n])\s+/)) {
|
|
192
|
+
const tokens = sentence.match(/[A-Za-z][A-Za-z0-9&.'’-]*/g) ?? [];
|
|
193
|
+
let run = [];
|
|
194
|
+
const flush = () => {
|
|
195
|
+
// Trim trailing connectors so "University of" never stands as a run.
|
|
196
|
+
while (run.length > 0 && NAME_CONNECTORS.has((run[run.length - 1] ?? "").toLowerCase()))
|
|
197
|
+
run.pop();
|
|
198
|
+
if (run.length > 1)
|
|
199
|
+
out.push(run.join(" "));
|
|
200
|
+
run = [];
|
|
201
|
+
};
|
|
202
|
+
tokens.forEach((tok, i) => {
|
|
203
|
+
const bare = tok.replace(/[.'’-]+$/, "");
|
|
204
|
+
const isAcronym = /^[A-Z]{2,}$/.test(bare);
|
|
205
|
+
const isCapitalised = /^[A-Z][a-z]/.test(bare);
|
|
206
|
+
const isConnector = NAME_CONNECTORS.has(bare.toLowerCase());
|
|
207
|
+
if (isAcronym || (isCapitalised && i > 0)) {
|
|
208
|
+
run.push(bare);
|
|
209
|
+
out.push(bare); // individually checkable
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
// A connector only continues a run that has already started.
|
|
213
|
+
if (isConnector && run.length > 0) {
|
|
214
|
+
run.push(bare);
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
flush();
|
|
218
|
+
});
|
|
219
|
+
flush();
|
|
220
|
+
}
|
|
221
|
+
return out;
|
|
222
|
+
}
|
|
223
|
+
/** Digit groups worth checking: phone numbers, years, percentages, counts ≥ 2 digits. */
|
|
224
|
+
function numericClaims(text) {
|
|
225
|
+
const prose = text.replace(/```[\s\S]*?```/g, " ").replace(/`[^`]*`/g, " ");
|
|
226
|
+
return (prose.match(/\+?\d[\d\s().-]{3,}\d|\b\d{2,}%?\b/g) ?? []).map((s) => s.trim());
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* File and path references — a favourite fabrication, and an unusually
|
|
230
|
+
* damaging one because naming a file implies the model READ it.
|
|
231
|
+
*
|
|
232
|
+
* Covers absolute paths (`/opt/fleetcrown/runner/.env`), relative paths
|
|
233
|
+
* (`data/contact-resolver.json`), and bare filenames with a data/config
|
|
234
|
+
* extension. The relative form matters: when challenged on the UZH claim, the
|
|
235
|
+
* model "corrected" itself by asserting what `data/contact-resolver.json`
|
|
236
|
+
* contained — a file it was never given. That reads as citing a source, which
|
|
237
|
+
* is precisely why an unverified correction is more corrosive than the
|
|
238
|
+
* original error: it spends the credibility the user was trying to restore.
|
|
239
|
+
*/
|
|
240
|
+
function pathClaims(text) {
|
|
241
|
+
const patterns = [
|
|
242
|
+
/(?:^|[\s("'`])(\/[A-Za-z0-9_.\-/]{4,})/g, // absolute
|
|
243
|
+
/(?:^|[\s("'`])([A-Za-z0-9_.-]+\/[A-Za-z0-9_.\-/]*[A-Za-z0-9_-]\.[a-z]{2,5})/g, // relative w/ extension
|
|
244
|
+
/(?:^|[\s("'`])([A-Za-z0-9_-]+\.(?:json|env|ya?ml|sql|toml|ini|conf|log))\b/g, // bare config filename
|
|
245
|
+
];
|
|
246
|
+
const out = new Set();
|
|
247
|
+
for (const re of patterns) {
|
|
248
|
+
for (const m of text.matchAll(re))
|
|
249
|
+
if (m[1])
|
|
250
|
+
out.add(m[1]);
|
|
251
|
+
}
|
|
252
|
+
return [...out];
|
|
253
|
+
}
|
|
254
|
+
/** Does this sentence talk about one of the user's own records? */
|
|
255
|
+
function mentionsSubject(sentence, subjects) {
|
|
256
|
+
const s = norm(sentence);
|
|
257
|
+
return subjects.some((sub) => {
|
|
258
|
+
const n = norm(sub);
|
|
259
|
+
return n.length > 2 && s.includes(n);
|
|
260
|
+
});
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Verify an answer against the facts it was supposed to come from.
|
|
264
|
+
*
|
|
265
|
+
* `extraEvidence` lets a caller admit sources outside the fact set — computed
|
|
266
|
+
* directive output, a tool result the model legitimately saw this turn.
|
|
267
|
+
* Anything not in facts, the user's message, or extraEvidence is unsupported
|
|
268
|
+
* by construction.
|
|
269
|
+
*
|
|
270
|
+
* `subjects` (entity-attribution mode) names the user's own records, so the
|
|
271
|
+
* check can tell "your contact Elena works at X" from "Lightning is instant".
|
|
272
|
+
*/
|
|
273
|
+
function verifyAnswer(input) {
|
|
274
|
+
const { answer, facts, userMessage } = input;
|
|
275
|
+
const mode = input.mode ?? "closed-world";
|
|
276
|
+
const subjects = input.subjects ?? facts.map((f) => f.subject);
|
|
277
|
+
const evidence = buildEvidence(facts, userMessage, input.extraEvidence ?? []);
|
|
278
|
+
const legalIds = new Set([
|
|
279
|
+
...facts.map((f) => f.id.toUpperCase()),
|
|
280
|
+
...(input.extraCitationIds ?? []).map((id) => id.toUpperCase()),
|
|
281
|
+
]);
|
|
282
|
+
const violations = [];
|
|
283
|
+
/**
|
|
284
|
+
* In entity-attribution mode, only sentences about the user's own records are
|
|
285
|
+
* subject to the name check. Built once so the per-token loop stays cheap.
|
|
286
|
+
*/
|
|
287
|
+
const attributionScope = mode === "entity-attribution"
|
|
288
|
+
? answer
|
|
289
|
+
.split(/(?<=[.!?:\n])\s+/)
|
|
290
|
+
.filter((s) => mentionsSubject(s, subjects))
|
|
291
|
+
.join(" ")
|
|
292
|
+
: answer;
|
|
293
|
+
// 1. Citations must resolve. A citation to a record that does not exist is
|
|
294
|
+
// the strongest possible signal of fabrication — it invents its own proof.
|
|
295
|
+
for (const cite of answer.match(/\[[FD]\d+\]/g) ?? []) {
|
|
296
|
+
const id = cite.slice(1, -1).toUpperCase();
|
|
297
|
+
if (!legalIds.has(id)) {
|
|
298
|
+
violations.push({
|
|
299
|
+
kind: "unknown-citation",
|
|
300
|
+
text: cite,
|
|
301
|
+
detail: `${cite} is not a record in this turn's context. Cite only ids that were provided, or say there is no record.`,
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
// 2. Named entities must be traceable. This is the anti-"UZH" rule.
|
|
306
|
+
const seen = new Set();
|
|
307
|
+
for (const run of properNounRuns(attributionScope)) {
|
|
308
|
+
const n = norm(run);
|
|
309
|
+
if (!n || seen.has(n))
|
|
310
|
+
continue;
|
|
311
|
+
seen.add(n);
|
|
312
|
+
// Single common words are noise; multi-word runs always checked.
|
|
313
|
+
const words = n.split(" ");
|
|
314
|
+
if (words.length === 1 && (COMMON.has(words[0] ?? "") || (words[0] ?? "").length < 2))
|
|
315
|
+
continue;
|
|
316
|
+
if (words.every((w) => COMMON.has(w)))
|
|
317
|
+
continue;
|
|
318
|
+
if (evidence.includes(n))
|
|
319
|
+
continue;
|
|
320
|
+
// A multi-word run whose every word is individually attested is fine —
|
|
321
|
+
// it is a rephrasing, not a new entity.
|
|
322
|
+
if (words.length > 1 && words.every((w) => COMMON.has(w) || evidence.includes(w)))
|
|
323
|
+
continue;
|
|
324
|
+
violations.push({
|
|
325
|
+
kind: "novel-proper-noun",
|
|
326
|
+
text: run,
|
|
327
|
+
detail: `"${run}" does not appear in any record or in the operator's message. If it is an organisation, role, or place you associated with someone, the relevant field is ${facts_js_1.NOT_RECORDED} — remove the claim.`,
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
// 3. Numbers must be traceable — invented phone numbers and dates read as
|
|
331
|
+
// authoritative precisely because they are specific.
|
|
332
|
+
for (const num of numericClaims(answer)) {
|
|
333
|
+
const n = norm(num);
|
|
334
|
+
if (!n || n.length < 2)
|
|
335
|
+
continue;
|
|
336
|
+
if (evidence.includes(n))
|
|
337
|
+
continue;
|
|
338
|
+
// Compare digits-only too: "+41 77 473 00 93" vs stored "+41774730093".
|
|
339
|
+
const digits = num.replace(/\D/g, "");
|
|
340
|
+
if (digits.length >= 4 && evidence.replace(/\D/g, "").includes(digits))
|
|
341
|
+
continue;
|
|
342
|
+
if (digits.length < 4)
|
|
343
|
+
continue; // small counts ("3 tasks") are rhetorical
|
|
344
|
+
violations.push({
|
|
345
|
+
kind: "novel-number",
|
|
346
|
+
text: num,
|
|
347
|
+
detail: `The number "${num}" is not in any record. Do not state contact details, dates, or metrics that were not provided.`,
|
|
348
|
+
});
|
|
349
|
+
}
|
|
350
|
+
// 4. Paths — "update the key in /opt/fleetcrown/runner/.env" was invented
|
|
351
|
+
// wholesale, and its specificity is what made it convincing.
|
|
352
|
+
for (const p of pathClaims(answer)) {
|
|
353
|
+
if (evidence.includes(norm(p)))
|
|
354
|
+
continue;
|
|
355
|
+
violations.push({
|
|
356
|
+
kind: "novel-path",
|
|
357
|
+
text: p,
|
|
358
|
+
detail: `The path "${p}" is not in any record. Do not state file locations you were not given.`,
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
return { ok: violations.length === 0, violations };
|
|
362
|
+
}
|
|
363
|
+
/**
|
|
364
|
+
* Turn violations into a repair instruction. One cheap retry with this appended
|
|
365
|
+
* fixes most turns, because the model is not being asked to know more — only to
|
|
366
|
+
* delete claims it cannot support.
|
|
367
|
+
*/
|
|
368
|
+
function buildRepairPrompt(violations, noBasisPhrase) {
|
|
369
|
+
return [
|
|
370
|
+
"Your previous answer contained claims not supported by the records. Rewrite it.",
|
|
371
|
+
"",
|
|
372
|
+
...violations.map((v) => `- ${v.detail}`),
|
|
373
|
+
"",
|
|
374
|
+
`Remove every unsupported claim. Where removing one empties a requested item, write "${noBasisPhrase}" for that item instead of substituting something else. Keep everything that was supported, unchanged.`,
|
|
375
|
+
].join("\n");
|
|
376
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"type":"commonjs"}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* The model REGISTRY — one SSOT for every model id an app may call.
|
|
4
|
+
*
|
|
5
|
+
* This module exists because the fleet paid for its absence twice, in two
|
|
6
|
+
* different currencies:
|
|
7
|
+
*
|
|
8
|
+
* OUTAGE — on 2026-08-18 Groq removed `llama-3.3-70b-versatile` and one app
|
|
9
|
+
* kept asking for it for eight days. A rot checker already existed, but it
|
|
10
|
+
* probed only the chains it knew about; the id that died was pinned
|
|
11
|
+
* elsewhere. A checker that does not enumerate its subjects cannot report
|
|
12
|
+
* the one it never knew about. The registry IS the enumeration: a model id
|
|
13
|
+
* is callable only if it appears here, and the catalog check walks exactly
|
|
14
|
+
* this list.
|
|
15
|
+
*
|
|
16
|
+
* MONEY — three apps silently billed real money on fallback, because the
|
|
17
|
+
* only thing separating the free variant from the paid one was a `:free`
|
|
18
|
+
* suffix on the id string. A billing boundary that lives in a naming
|
|
19
|
+
* convention is one typo away from a paid call. Here it is a FIELD, and the
|
|
20
|
+
* validator refuses an entry whose flag contradicts its own cost or suffix —
|
|
21
|
+
* so the contradiction is a build failure, not an invoice.
|
|
22
|
+
*
|
|
23
|
+
* What deliberately does NOT live here: which model to PREFER (that is the
|
|
24
|
+
* chain's job), UI presentation (labels, badges — app concern), and anything
|
|
25
|
+
* that knows where data lives. Same boundary as the rest of this package:
|
|
26
|
+
* meaning in core, adapters in the app.
|
|
27
|
+
*
|
|
28
|
+
* ── Vendor vs author ─────────────────────────────────────────────────────────
|
|
29
|
+
* A registry row is a CALLABLE id at a VENDOR — the place a request goes —
|
|
30
|
+
* because that is the unit that rots, meters, and bills. The AUTHOR (who
|
|
31
|
+
* trained it) is metadata. The two were conflated in one app's registry
|
|
32
|
+
* ("provider: Anthropic" on a row served by OpenRouter), which made "who do we
|
|
33
|
+
* pay" unanswerable by query. Here they are separate fields.
|
|
34
|
+
*/
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.defineRegistry = defineRegistry;
|
|
37
|
+
exports.freeOnly = freeOnly;
|
|
38
|
+
exports.toolCapable = toolCapable;
|
|
39
|
+
/** A `:free`-suffixed id claiming to be paid, or a "free" entry with a price —
|
|
40
|
+
* each one is the 2026 billing incident waiting to recur. */
|
|
41
|
+
function validateEntry(e) {
|
|
42
|
+
if (!e.id.trim())
|
|
43
|
+
return "entry has an empty id";
|
|
44
|
+
if (!e.vendor.trim())
|
|
45
|
+
return `"${e.id}": empty vendor`;
|
|
46
|
+
const cost = (e.inputCostPer1M ?? 0) + (e.outputCostPer1M ?? 0);
|
|
47
|
+
if (!e.paid && cost > 0) {
|
|
48
|
+
return `"${e.id}": declared free but carries a cost (${cost}/1M) — the flag or the price is lying`;
|
|
49
|
+
}
|
|
50
|
+
if (e.paid && e.id.endsWith(":free")) {
|
|
51
|
+
return `"${e.id}": declared paid but the id says :free — the flag or the id is lying`;
|
|
52
|
+
}
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Build a registry from entries. Throws on the first contradiction — a
|
|
57
|
+
* registry that loads is a registry whose billing boundary can be trusted.
|
|
58
|
+
*/
|
|
59
|
+
function defineRegistry(entries) {
|
|
60
|
+
const seen = new Set();
|
|
61
|
+
for (const e of entries) {
|
|
62
|
+
const problem = validateEntry(e);
|
|
63
|
+
if (problem)
|
|
64
|
+
throw new Error(`ai-kit registry: ${problem}`);
|
|
65
|
+
const key = `${e.vendor}:${e.id}`;
|
|
66
|
+
if (seen.has(key)) {
|
|
67
|
+
throw new Error(`ai-kit registry: duplicate entry ${key} — two rows for one callable id is two sources of truth`);
|
|
68
|
+
}
|
|
69
|
+
seen.add(key);
|
|
70
|
+
}
|
|
71
|
+
const frozen = Object.freeze(entries.map((e) => ({ ...e })));
|
|
72
|
+
const find = (id, vendor) => frozen.find((e) => e.id === id && (vendor === undefined || e.vendor === vendor));
|
|
73
|
+
return {
|
|
74
|
+
entries: frozen,
|
|
75
|
+
find,
|
|
76
|
+
require(id, vendor) {
|
|
77
|
+
const hit = find(id, vendor);
|
|
78
|
+
if (!hit) {
|
|
79
|
+
const scope = vendor ? ` at ${vendor}` : "";
|
|
80
|
+
throw new Error(`ai-kit registry: "${id}"${scope} is not registered — a model id is callable only if it appears in the registry (add it with its paid flag, or stop calling it)`);
|
|
81
|
+
}
|
|
82
|
+
return hit;
|
|
83
|
+
},
|
|
84
|
+
idsForVendor: (vendor) => frozen.filter((e) => e.vendor === vendor).map((e) => e.id),
|
|
85
|
+
vendors: () => [...new Set(frozen.map((e) => e.vendor))],
|
|
86
|
+
freeEntries: () => frozen.filter((e) => !e.paid),
|
|
87
|
+
paidEntries: () => frozen.filter((e) => e.paid),
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* The platform-key guard: the ids from `requested` that a platform-funded
|
|
92
|
+
* call may serve. Registered-and-free passes; paid is dropped; an UNKNOWN id
|
|
93
|
+
* is dropped too — an id nobody registered has an unknown price, and "unknown"
|
|
94
|
+
* spends someone's money only when a person decides it does.
|
|
95
|
+
*
|
|
96
|
+
* Returns the dropped ids alongside, because a silently narrowed chain reads
|
|
97
|
+
* as "covered everything" when it didn't.
|
|
98
|
+
*/
|
|
99
|
+
function freeOnly(registry, requested) {
|
|
100
|
+
const allowed = [];
|
|
101
|
+
const dropped = [];
|
|
102
|
+
for (const id of requested) {
|
|
103
|
+
const entry = registry.find(id);
|
|
104
|
+
if (!entry)
|
|
105
|
+
dropped.push({ id, why: "unregistered" });
|
|
106
|
+
else if (entry.paid)
|
|
107
|
+
dropped.push({ id, why: "paid" });
|
|
108
|
+
else
|
|
109
|
+
allowed.push(id);
|
|
110
|
+
}
|
|
111
|
+
return { allowed, dropped };
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* A tool-driving chain may only contain models that can drive a tool loop.
|
|
115
|
+
* "unprobed" entries are reported, not silently trusted — the probe table is
|
|
116
|
+
* one `npm run probe:models` away, and a chain built on guesses loses turns
|
|
117
|
+
* exactly on the models most likely to serve free traffic.
|
|
118
|
+
*/
|
|
119
|
+
function toolCapable(registry, requested) {
|
|
120
|
+
const usable = [];
|
|
121
|
+
const refused = [];
|
|
122
|
+
for (const id of requested) {
|
|
123
|
+
const entry = registry.find(id);
|
|
124
|
+
const protocol = entry?.toolProtocol ?? "unprobed";
|
|
125
|
+
if (protocol === "native" || protocol === "text")
|
|
126
|
+
usable.push(id);
|
|
127
|
+
else
|
|
128
|
+
refused.push({ id, protocol });
|
|
129
|
+
}
|
|
130
|
+
return { usable, refused };
|
|
131
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@bitbaum/ai-kit",
|
|
3
|
+
"version": "0.6.2",
|
|
4
|
+
"description": "One install for the AI layer of an app: which model to call, what to do when the vendor retires it, how to walk the fallback chain and know when none of it worked, how to read the three kinds of 429, a fair daily budget across users, headless AI form filling — and now the model registry (one SSOT for every callable id, with the paid/free boundary as a field) and the grounding harness (facts, contract, deterministic fabrication check).",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"author": "Mao Nakamoto",
|
|
7
|
+
"homepage": "https://github.com/bitbaum/ai-kit#readme",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/bitbaum/ai-kit.git"
|
|
11
|
+
},
|
|
12
|
+
"bugs": {
|
|
13
|
+
"url": "https://github.com/bitbaum/ai-kit/issues"
|
|
14
|
+
},
|
|
15
|
+
"keywords": [
|
|
16
|
+
"ai",
|
|
17
|
+
"llm",
|
|
18
|
+
"form-fill",
|
|
19
|
+
"fallback",
|
|
20
|
+
"failover",
|
|
21
|
+
"health-check",
|
|
22
|
+
"free-tier",
|
|
23
|
+
"rate-limit",
|
|
24
|
+
"quota",
|
|
25
|
+
"fair-share",
|
|
26
|
+
"groq",
|
|
27
|
+
"openrouter"
|
|
28
|
+
],
|
|
29
|
+
"type": "module",
|
|
30
|
+
"sideEffects": false,
|
|
31
|
+
"engines": {
|
|
32
|
+
"node": ">=20"
|
|
33
|
+
},
|
|
34
|
+
"publishConfig": {
|
|
35
|
+
"access": "public",
|
|
36
|
+
"provenance": true
|
|
37
|
+
},
|
|
38
|
+
"files": [
|
|
39
|
+
"dist",
|
|
40
|
+
"dist-cjs",
|
|
41
|
+
"src"
|
|
42
|
+
],
|
|
43
|
+
"exports": {
|
|
44
|
+
".": {
|
|
45
|
+
"types": "./dist/index.d.ts",
|
|
46
|
+
"default": "./dist/index.js"
|
|
47
|
+
},
|
|
48
|
+
"./forms": {
|
|
49
|
+
"types": "./dist/forms.d.ts",
|
|
50
|
+
"default": "./dist/forms.js"
|
|
51
|
+
},
|
|
52
|
+
"./react": {
|
|
53
|
+
"types": "./dist/react.d.ts",
|
|
54
|
+
"default": "./dist/react.js"
|
|
55
|
+
},
|
|
56
|
+
"./server": {
|
|
57
|
+
"types": "./dist/server.d.ts",
|
|
58
|
+
"default": "./dist/server.js"
|
|
59
|
+
},
|
|
60
|
+
"./registry": {
|
|
61
|
+
"types": "./dist/registry.d.ts",
|
|
62
|
+
"require": "./dist-cjs/registry.js",
|
|
63
|
+
"default": "./dist/registry.js"
|
|
64
|
+
},
|
|
65
|
+
"./grounding": {
|
|
66
|
+
"types": "./dist/grounding/index.d.ts",
|
|
67
|
+
"require": "./dist-cjs/grounding/index.js",
|
|
68
|
+
"default": "./dist/grounding/index.js"
|
|
69
|
+
}
|
|
70
|
+
},
|
|
71
|
+
"scripts": {
|
|
72
|
+
"build": "tsc -p tsconfig.json && tsc -p tsconfig.cjs.json && node -e \"require('fs').writeFileSync('dist-cjs/package.json', JSON.stringify({type:'commonjs'})+'\\n')\"",
|
|
73
|
+
"lint": "eslint .",
|
|
74
|
+
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
75
|
+
"test": "node --test test/*.test.js",
|
|
76
|
+
"check:catalog": "npm run build && node scripts/check-catalog.mjs",
|
|
77
|
+
"verify": "npm run format:check && npm run lint && npm run typecheck && npm run build && npm test",
|
|
78
|
+
"prepare": "npm run build",
|
|
79
|
+
"format": "prettier --write .",
|
|
80
|
+
"format:check": "prettier --check ."
|
|
81
|
+
},
|
|
82
|
+
"devDependencies": {
|
|
83
|
+
"@eslint/js": "^10.0.1",
|
|
84
|
+
"@types/node": "^26.4.0",
|
|
85
|
+
"eslint": "^10.9.1",
|
|
86
|
+
"globals": "^17.11.0",
|
|
87
|
+
"prettier": "3.9.6",
|
|
88
|
+
"typescript": "^6.0.3",
|
|
89
|
+
"typescript-eslint": "^8.68.0"
|
|
90
|
+
},
|
|
91
|
+
"dependencies": {
|
|
92
|
+
"ai-forms": "^0.1.2"
|
|
93
|
+
},
|
|
94
|
+
"peerDependencies": {
|
|
95
|
+
"react": ">=18"
|
|
96
|
+
},
|
|
97
|
+
"peerDependenciesMeta": {
|
|
98
|
+
"react": {
|
|
99
|
+
"optional": true
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
package/src/attempt.ts
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Walk a chain, never own the fetch.
|
|
3
|
+
*
|
|
4
|
+
* `usableChain`/`chainFrom` (chain.ts) already answer WHICH links exist and in
|
|
5
|
+
* what order. What was still missing — in every app that hand-rolled it, and
|
|
6
|
+
* inconsistently — is the loop that tries link one, and on failure tries link
|
|
7
|
+
* two, rather than picking the first link and calling it once. A chain nobody
|
|
8
|
+
* walks is a list, not a fallback: it was found sitting unused next to a
|
|
9
|
+
* single-shot caller in the same app that this package's `freeChain` already
|
|
10
|
+
* protected from a retired model but not from a dead key, because ordering the
|
|
11
|
+
* links and walking them were still two different jobs and only one had a
|
|
12
|
+
* home.
|
|
13
|
+
*
|
|
14
|
+
* This still ships no HTTP client. `attempt` is supplied by the caller and
|
|
15
|
+
* does the actual request; this only decides which link goes next, and
|
|
16
|
+
* records the outcome if a `HealthTracker` is given.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import type { Link } from "./chain.js";
|
|
20
|
+
import type { HealthTracker } from "./health.js";
|
|
21
|
+
|
|
22
|
+
export interface ChainAttemptFailure {
|
|
23
|
+
link: Link;
|
|
24
|
+
message: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Every link in the chain was tried and failed (or the chain was empty). */
|
|
28
|
+
export class ChainExhaustedError extends Error {
|
|
29
|
+
readonly failures: ChainAttemptFailure[];
|
|
30
|
+
|
|
31
|
+
constructor(failures: ChainAttemptFailure[]) {
|
|
32
|
+
super(
|
|
33
|
+
failures.length === 0
|
|
34
|
+
? "No usable link in the chain — every provider is missing its key, or has no models configured."
|
|
35
|
+
: `All ${failures.length} link(s) failed — ${failures
|
|
36
|
+
.map((f) => `${f.link.provider.id}/${f.link.model}: ${f.message}`)
|
|
37
|
+
.join("; ")}`,
|
|
38
|
+
);
|
|
39
|
+
this.name = "ChainExhaustedError";
|
|
40
|
+
this.failures = failures;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface TryChainOptions<T> {
|
|
45
|
+
/** Makes the actual call for one link. Throw to demote to the next link. */
|
|
46
|
+
attempt: (link: Link) => Promise<T>;
|
|
47
|
+
/** Records one success or one failure for the WHOLE walk, not per link. */
|
|
48
|
+
health?: HealthTracker;
|
|
49
|
+
/** Called on each link's failure, before moving to the next — e.g. to log it. */
|
|
50
|
+
onLinkFailure?: (link: Link, error: unknown) => void;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Try each link in order; return the first success.
|
|
55
|
+
*
|
|
56
|
+
* Health is recorded once per call — a success on link two is still a success
|
|
57
|
+
* for the app, and a health check that flagged it "degraded" because the FIRST
|
|
58
|
+
* link failed would be reporting its own fallback working as a problem.
|
|
59
|
+
*
|
|
60
|
+
* Throws `ChainExhaustedError` (carrying every link's failure) when none
|
|
61
|
+
* succeed, so a caller can log exactly what was tried rather than only the
|
|
62
|
+
* last error — the failure that matters is often not the last one.
|
|
63
|
+
*/
|
|
64
|
+
export async function tryChain<T>(chain: Link[], options: TryChainOptions<T>): Promise<T> {
|
|
65
|
+
const failures: ChainAttemptFailure[] = [];
|
|
66
|
+
|
|
67
|
+
for (const link of chain) {
|
|
68
|
+
try {
|
|
69
|
+
const result = await options.attempt(link);
|
|
70
|
+
options.health?.recordSuccess();
|
|
71
|
+
return result;
|
|
72
|
+
} catch (error) {
|
|
73
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
74
|
+
failures.push({ link, message });
|
|
75
|
+
options.onLinkFailure?.(link, error);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const exhausted = new ChainExhaustedError(failures);
|
|
80
|
+
options.health?.recordFailure(exhausted);
|
|
81
|
+
throw exhausted;
|
|
82
|
+
}
|