@bitbaum/ai-kit 1.4.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog-fetch.d.ts +77 -0
- package/dist/catalog-fetch.js +109 -0
- package/dist/catalog.js +16 -22
- package/dist/index.d.ts +2 -0
- package/dist/index.js +5 -0
- package/dist/resolve.d.ts +159 -0
- package/dist/resolve.js +218 -0
- package/package.json +1 -1
- package/src/catalog-fetch.ts +146 -0
- package/src/catalog.ts +16 -19
- package/src/index.ts +16 -0
- package/src/resolve.ts +294 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One GET /models, parsed once, for everything that needs to know what a vendor
|
|
3
|
+
* currently offers.
|
|
4
|
+
*
|
|
5
|
+
* `catalog.ts` already fetched this list to answer "has a pinned id rotted".
|
|
6
|
+
* `resolve.ts` needs the SAME request to answer "what else is on the shelf, and
|
|
7
|
+
* what does the vendor say about it" — and the ids alone cannot answer that.
|
|
8
|
+
*
|
|
9
|
+
* Rather than issue a second, slightly different GET (the exact failure
|
|
10
|
+
* catalog.ts's own header complains about — "the app that wrote its own first
|
|
11
|
+
* wrote it slightly differently"), both read this.
|
|
12
|
+
*
|
|
13
|
+
* ── Two vendors, two schemas, one shape ──────────────────────────────────────
|
|
14
|
+
* Verified against live responses on 2026-09-13, and they do NOT agree:
|
|
15
|
+
*
|
|
16
|
+
* OpenRouter architecture.output_modalities supported_parameters: ["tools"]
|
|
17
|
+
* Groq output_modalities (top level) supported_features: ["json_mode"]
|
|
18
|
+
*
|
|
19
|
+
* So a reader that knows only OpenRouter's shape reports every Groq model as
|
|
20
|
+
* having no text output and no tools — which, in a filter, silently removes a
|
|
21
|
+
* working vendor. Normalising here means each consumer states its requirement
|
|
22
|
+
* once instead of learning both schemas.
|
|
23
|
+
*
|
|
24
|
+
* ── Unknown is a value, not a default ────────────────────────────────────────
|
|
25
|
+
* Every normalised field is nullable, and null means "the vendor did not say".
|
|
26
|
+
* That is deliberately distinct from false. A filter that treats "did not say"
|
|
27
|
+
* as "does not support" narrows silently as vendors change their schemas; one
|
|
28
|
+
* that treats it as "supports" invents capability. Callers must choose, and the
|
|
29
|
+
* choice is visible at the call site because the type forces it.
|
|
30
|
+
*/
|
|
31
|
+
/** What a vendor's catalogue says about one model, in one shape. */
|
|
32
|
+
export type ModelRecord = {
|
|
33
|
+
id: string;
|
|
34
|
+
/** Output modalities, or null when the vendor does not publish them. */
|
|
35
|
+
outputModalities: string[] | null;
|
|
36
|
+
/**
|
|
37
|
+
* True only when the vendor publishes a price and every component is zero.
|
|
38
|
+
* Null when it publishes no price at all.
|
|
39
|
+
*
|
|
40
|
+
* Not the same question as "does the `:free` suffix appear". Checked live on
|
|
41
|
+
* 2026-09-13: OpenRouter lists 19 ids ending `:free` and 22 priced at zero —
|
|
42
|
+
* and the three zero-priced ids WITHOUT the suffix include `openrouter/free`,
|
|
43
|
+
* the auto-router that is the most rot-resistant entry in the whole chain.
|
|
44
|
+
* A suffix check misses it. Price is the vendor telling you who pays.
|
|
45
|
+
*/
|
|
46
|
+
costsNothing: boolean | null;
|
|
47
|
+
/** True when the vendor declares tool/function calling. Null when unstated. */
|
|
48
|
+
tools: boolean | null;
|
|
49
|
+
/** Vendor-declared context window, when published. */
|
|
50
|
+
contextLength: number | null;
|
|
51
|
+
/**
|
|
52
|
+
* Date the vendor says this id stops working (ISO yyyy-mm-dd), when it says so.
|
|
53
|
+
*
|
|
54
|
+
* Rare but real: of 445 OpenRouter models on 2026-09-13, five carried one, and
|
|
55
|
+
* one of those was a FREE model expiring in 17 days. Where present this turns
|
|
56
|
+
* rot from something detected afterwards into something known in advance, so
|
|
57
|
+
* it is worth surfacing even though most models omit it.
|
|
58
|
+
*/
|
|
59
|
+
expiresOn: string | null;
|
|
60
|
+
};
|
|
61
|
+
export type FetchCatalogOptions = {
|
|
62
|
+
fetchImpl?: typeof fetch;
|
|
63
|
+
timeoutMs?: number;
|
|
64
|
+
};
|
|
65
|
+
/**
|
|
66
|
+
* Every model one vendor lists, normalised — or NULL when the catalogue could
|
|
67
|
+
* not be read.
|
|
68
|
+
*
|
|
69
|
+
* Null covers no key, a network failure, a non-200, an unparseable body, and a
|
|
70
|
+
* body that parses but lists nothing. It never means "this vendor has no
|
|
71
|
+
* models", and the distinction is not academic: while building this, a local
|
|
72
|
+
* checkout missing GROQ_API_KEY answered 401, and a reader that collapsed that
|
|
73
|
+
* into an empty list reported BOTH live Groq models as retired. Had resolution
|
|
74
|
+
* trusted it, an expired key would have emptied the chain rather than failing a
|
|
75
|
+
* single call.
|
|
76
|
+
*/
|
|
77
|
+
export declare function fetchCatalog(baseUrl: string, key: string | undefined, opts?: FetchCatalogOptions): Promise<ModelRecord[] | null>;
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One GET /models, parsed once, for everything that needs to know what a vendor
|
|
3
|
+
* currently offers.
|
|
4
|
+
*
|
|
5
|
+
* `catalog.ts` already fetched this list to answer "has a pinned id rotted".
|
|
6
|
+
* `resolve.ts` needs the SAME request to answer "what else is on the shelf, and
|
|
7
|
+
* what does the vendor say about it" — and the ids alone cannot answer that.
|
|
8
|
+
*
|
|
9
|
+
* Rather than issue a second, slightly different GET (the exact failure
|
|
10
|
+
* catalog.ts's own header complains about — "the app that wrote its own first
|
|
11
|
+
* wrote it slightly differently"), both read this.
|
|
12
|
+
*
|
|
13
|
+
* ── Two vendors, two schemas, one shape ──────────────────────────────────────
|
|
14
|
+
* Verified against live responses on 2026-09-13, and they do NOT agree:
|
|
15
|
+
*
|
|
16
|
+
* OpenRouter architecture.output_modalities supported_parameters: ["tools"]
|
|
17
|
+
* Groq output_modalities (top level) supported_features: ["json_mode"]
|
|
18
|
+
*
|
|
19
|
+
* So a reader that knows only OpenRouter's shape reports every Groq model as
|
|
20
|
+
* having no text output and no tools — which, in a filter, silently removes a
|
|
21
|
+
* working vendor. Normalising here means each consumer states its requirement
|
|
22
|
+
* once instead of learning both schemas.
|
|
23
|
+
*
|
|
24
|
+
* ── Unknown is a value, not a default ────────────────────────────────────────
|
|
25
|
+
* Every normalised field is nullable, and null means "the vendor did not say".
|
|
26
|
+
* That is deliberately distinct from false. A filter that treats "did not say"
|
|
27
|
+
* as "does not support" narrows silently as vendors change their schemas; one
|
|
28
|
+
* that treats it as "supports" invents capability. Callers must choose, and the
|
|
29
|
+
* choice is visible at the call site because the type forces it.
|
|
30
|
+
*/
|
|
31
|
+
function asStringArray(v) {
|
|
32
|
+
if (!Array.isArray(v))
|
|
33
|
+
return null;
|
|
34
|
+
const out = v.filter((x) => typeof x === "string");
|
|
35
|
+
return out.length > 0 ? out : null;
|
|
36
|
+
}
|
|
37
|
+
/** Zero only when a price is published AND every numeric component is zero. */
|
|
38
|
+
function priceIsZero(pricing) {
|
|
39
|
+
if (!pricing || typeof pricing !== "object")
|
|
40
|
+
return null;
|
|
41
|
+
const entries = Object.entries(pricing);
|
|
42
|
+
const numbers = entries
|
|
43
|
+
.map(([, v]) => (typeof v === "string" || typeof v === "number" ? Number(v) : NaN))
|
|
44
|
+
.filter((n) => Number.isFinite(n));
|
|
45
|
+
if (numbers.length === 0)
|
|
46
|
+
return null;
|
|
47
|
+
return numbers.every((n) => n === 0);
|
|
48
|
+
}
|
|
49
|
+
/** Tool support as DECLARED by the vendor, across both known schemas. */
|
|
50
|
+
function declaresTools(m) {
|
|
51
|
+
const params = asStringArray(m.supported_parameters);
|
|
52
|
+
if (params)
|
|
53
|
+
return params.includes("tools");
|
|
54
|
+
const features = asStringArray(m.supported_features);
|
|
55
|
+
if (features)
|
|
56
|
+
return features.some((f) => f === "tools" || f === "tool_use");
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
function normalise(m) {
|
|
60
|
+
const id = typeof m.id === "string" ? m.id : "";
|
|
61
|
+
if (!id)
|
|
62
|
+
return null;
|
|
63
|
+
const arch = (m.architecture ?? {});
|
|
64
|
+
const expires = m.expiration_date;
|
|
65
|
+
return {
|
|
66
|
+
id,
|
|
67
|
+
outputModalities: asStringArray(arch.output_modalities) ?? asStringArray(m.output_modalities),
|
|
68
|
+
costsNothing: priceIsZero(m.pricing),
|
|
69
|
+
tools: declaresTools(m),
|
|
70
|
+
contextLength: typeof m.context_length === "number" ? m.context_length : null,
|
|
71
|
+
expiresOn: typeof expires === "string" && expires.trim() ? expires.trim() : null,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Every model one vendor lists, normalised — or NULL when the catalogue could
|
|
76
|
+
* not be read.
|
|
77
|
+
*
|
|
78
|
+
* Null covers no key, a network failure, a non-200, an unparseable body, and a
|
|
79
|
+
* body that parses but lists nothing. It never means "this vendor has no
|
|
80
|
+
* models", and the distinction is not academic: while building this, a local
|
|
81
|
+
* checkout missing GROQ_API_KEY answered 401, and a reader that collapsed that
|
|
82
|
+
* into an empty list reported BOTH live Groq models as retired. Had resolution
|
|
83
|
+
* trusted it, an expired key would have emptied the chain rather than failing a
|
|
84
|
+
* single call.
|
|
85
|
+
*/
|
|
86
|
+
export async function fetchCatalog(baseUrl, key, opts = {}) {
|
|
87
|
+
if (!key?.trim())
|
|
88
|
+
return null;
|
|
89
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
90
|
+
const timeoutMs = opts.timeoutMs ?? 20_000;
|
|
91
|
+
try {
|
|
92
|
+
const res = await fetchImpl(`${baseUrl.replace(/\/$/, "")}/models`, {
|
|
93
|
+
headers: { Authorization: `Bearer ${key.trim()}` },
|
|
94
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
95
|
+
});
|
|
96
|
+
if (!res.ok)
|
|
97
|
+
return null;
|
|
98
|
+
const body = (await res.json());
|
|
99
|
+
if (!Array.isArray(body?.data))
|
|
100
|
+
return null;
|
|
101
|
+
const records = body.data
|
|
102
|
+
.map((m) => normalise((m ?? {})))
|
|
103
|
+
.filter((m) => m !== null);
|
|
104
|
+
return records.length > 0 ? records : null;
|
|
105
|
+
}
|
|
106
|
+
catch {
|
|
107
|
+
return null;
|
|
108
|
+
}
|
|
109
|
+
}
|
package/dist/catalog.js
CHANGED
|
@@ -22,29 +22,23 @@
|
|
|
22
22
|
* costs real tokens and cannot run on a timer; existence can.
|
|
23
23
|
*/
|
|
24
24
|
import { providerModels } from "./chain.js";
|
|
25
|
-
|
|
25
|
+
import { fetchCatalog } from "./catalog-fetch.js";
|
|
26
|
+
/**
|
|
27
|
+
* Ids listed by one provider, or null when the catalogue could not be read.
|
|
28
|
+
*
|
|
29
|
+
* The request itself now lives in catalog-fetch.ts, because `resolve.ts` needs
|
|
30
|
+
* the SAME GET and the ids alone cannot tell it what a model costs or can do.
|
|
31
|
+
* Two near-identical fetches drifting apart is the failure this module's own
|
|
32
|
+
* header objects to; one fetch, two readers.
|
|
33
|
+
*
|
|
34
|
+
* Every null case is preserved exactly — no key, network failure, non-200,
|
|
35
|
+
* unparseable body, and a body that parses but lists nothing (a malformed
|
|
36
|
+
* answer, not a vendor with no models: refusing it keeps a bad response from
|
|
37
|
+
* reading as total rot).
|
|
38
|
+
*/
|
|
26
39
|
async function liveIds(provider, key, fetchImpl, timeoutMs) {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
headers: { Authorization: `Bearer ${key}` },
|
|
30
|
-
signal: AbortSignal.timeout(timeoutMs),
|
|
31
|
-
});
|
|
32
|
-
if (!res.ok)
|
|
33
|
-
return null;
|
|
34
|
-
const body = (await res.json());
|
|
35
|
-
if (!Array.isArray(body?.data))
|
|
36
|
-
return null;
|
|
37
|
-
const ids = body.data
|
|
38
|
-
.map((m) => (typeof m?.id === "string" ? m.id : ""))
|
|
39
|
-
.filter((id) => id.length > 0);
|
|
40
|
-
// A catalogue that parses but lists nothing is a malformed answer, not a
|
|
41
|
-
// vendor with no models. Refusing it keeps a bad response from reading as
|
|
42
|
-
// total rot.
|
|
43
|
-
return ids.length > 0 ? ids : null;
|
|
44
|
-
}
|
|
45
|
-
catch {
|
|
46
|
-
return null;
|
|
47
|
-
}
|
|
40
|
+
const records = await fetchCatalog(provider.baseUrl, key, { fetchImpl, timeoutMs });
|
|
41
|
+
return records ? records.map((r) => r.id) : null;
|
|
48
42
|
}
|
|
49
43
|
/**
|
|
50
44
|
* Check every model a chain would try against what its vendor still lists.
|
package/dist/index.d.ts
CHANGED
|
@@ -51,6 +51,8 @@
|
|
|
51
51
|
*/
|
|
52
52
|
export { type Provider, type Env, type Link, type CostVerdict, providerModels, withEnvPrefix, freeChain, modelCost, modelCostAt, paidModelsIn, dayCapacityTokens, usableChain, chainFrom, } from "./chain.js";
|
|
53
53
|
export { type CatalogVerdict, type CheckCatalogOptions, checkCatalog, hasRot, deadProviders, catalogReport, } from "./catalog.js";
|
|
54
|
+
export { type ModelRecord, type FetchCatalogOptions, fetchCatalog } from "./catalog-fetch.js";
|
|
55
|
+
export { type Requirements, type ResolveOptions, type ProviderResolution, resolveChain, applyResolution, routedAroundRot, emptyProviders, resolutionReport, } from "./resolve.js";
|
|
54
56
|
export { type ChainAttemptFailure, type TryChainOptions, ChainExhaustedError, tryChain, } from "./attempt.js";
|
|
55
57
|
export { type ChatMessage, type ContentPart, type ToolCall, type CompleteOptions, type CompleteResult, LinkFailure, complete, linkId, } from "./complete.js";
|
|
56
58
|
export { type HealthStatus, type Health, type HealthTrackerOptions, type HealthTracker, createHealthTracker, } from "./health.js";
|
package/dist/index.js
CHANGED
|
@@ -51,6 +51,11 @@
|
|
|
51
51
|
*/
|
|
52
52
|
export { providerModels, withEnvPrefix, freeChain, modelCost, modelCostAt, paidModelsIn, dayCapacityTokens, usableChain, chainFrom, } from "./chain.js";
|
|
53
53
|
export { checkCatalog, hasRot, deadProviders, catalogReport, } from "./catalog.js";
|
|
54
|
+
export { fetchCatalog } from "./catalog-fetch.js";
|
|
55
|
+
// Resolution is the answer to the question `catalog` only asks. checkCatalog
|
|
56
|
+
// REPORTS that a pinned id is gone; resolveChain drops it and calls the next
|
|
57
|
+
// one, so a vendor retirement stops needing a pull request to survive.
|
|
58
|
+
export { resolveChain, applyResolution, routedAroundRot, emptyProviders, resolutionReport, } from "./resolve.js";
|
|
54
59
|
export { ChainExhaustedError, tryChain, } from "./attempt.js";
|
|
55
60
|
export { LinkFailure, complete, linkId, } from "./complete.js";
|
|
56
61
|
export { createHealthTracker, } from "./health.js";
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* resolve — stop writing model ids down.
|
|
3
|
+
*
|
|
4
|
+
* ── The problem the chain did not solve ──────────────────────────────────────
|
|
5
|
+
* `chain.ts` replaced one pinned model with a list of pinned models, and says so
|
|
6
|
+
* itself: "a single pinned free model is not a configuration, it is a scheduled
|
|
7
|
+
* outage." `catalog.ts` then noticed the list rots too, and added a check. But a
|
|
8
|
+
* check only produces a SENTENCE. Somebody still has to read it, edit a file,
|
|
9
|
+
* open a pull request, wait for CI, and deploy — and until they do, the app is
|
|
10
|
+
* handing users a model that answers 404.
|
|
11
|
+
*
|
|
12
|
+
* That round trip is not hypothetical, it is this module's origin. On
|
|
13
|
+
* 2026-09-13 a freshly scheduled health check alerted within minutes of its
|
|
14
|
+
* first run: two ids in a consumer's registry were gone from OpenRouter. The
|
|
15
|
+
* alert was correct, fast, and completely dependent on a human being available
|
|
16
|
+
* to act on it. The vendor had published the truth the whole time.
|
|
17
|
+
*
|
|
18
|
+
* So: keep the declared list, but treat it as a PREFERENCE, and let the vendor's
|
|
19
|
+
* own catalogue decide what is actually callable. A retirement stops being an
|
|
20
|
+
* incident and becomes a no-op.
|
|
21
|
+
*
|
|
22
|
+
* ── Three rules, each paid for by a real failure ─────────────────────────────
|
|
23
|
+
*
|
|
24
|
+
* 1. NEVER SHRINK ON IGNORANCE. An unreadable catalogue leaves the declared
|
|
25
|
+
* list exactly as written and sets `unverified`. While building this, a
|
|
26
|
+
* checkout without GROQ_API_KEY got a 401, and the first draft of the probe
|
|
27
|
+
* printed "GONE" for both Groq models — both of which were live. Resolution
|
|
28
|
+
* that trusted that would let an expired key empty the chain.
|
|
29
|
+
*
|
|
30
|
+
* 2. DISCOVERY EXTENDS THE TAIL, NEVER THE HEAD. Declared ids were curated —
|
|
31
|
+
* in this package's case probed with a real tool call, because five of nine
|
|
32
|
+
* free models turned out to emit tool calls only as text. A discovered id
|
|
33
|
+
* has no such evidence behind it, so it may be a last resort and never a
|
|
34
|
+
* first choice.
|
|
35
|
+
*
|
|
36
|
+
* 3. DISCOVER ONLY WHAT THE VENDOR CALLS FREE, AND ONLY WHAT IT CALLS USABLE.
|
|
37
|
+
* Both halves matter, and both are read from the vendor rather than guessed:
|
|
38
|
+
*
|
|
39
|
+
* - Price. Not the `:free` suffix — the published number. Checked live on
|
|
40
|
+
* 2026-09-13, OpenRouter listed 19 ids ending `:free` and 22 priced at
|
|
41
|
+
* zero. Groq publishes prices too, and NONE of its models are zero, so
|
|
42
|
+
* discovery there correctly finds nothing and that vendor stays curated.
|
|
43
|
+
* A suffix heuristic would have gone looking for `:free` at a vendor
|
|
44
|
+
* that does not use the convention; a price check just declines.
|
|
45
|
+
*
|
|
46
|
+
* - Suitability. Of the 19 free OpenRouter ids, one was
|
|
47
|
+
* `nemotron-3.5-content-safety` — a classifier, and the only one of the
|
|
48
|
+
* 19 declaring `tools: false`. Two more zero-priced ids were
|
|
49
|
+
* `lyria-3-*`, which emit AUDIO. Adding all zero-priced models to a chat
|
|
50
|
+
* chain would have put a safety classifier and a music generator in
|
|
51
|
+
* front of a user asking a question. Requiring text output and declared
|
|
52
|
+
* tool support removes exactly those three and nothing else.
|
|
53
|
+
*
|
|
54
|
+
* ── What this is worth ───────────────────────────────────────────────────────
|
|
55
|
+
* Measured against the live catalogue on 2026-09-13: `freeChain()` names five
|
|
56
|
+
* OpenRouter ids by hand; nineteen zero-priced, text-out, tool-capable ids were
|
|
57
|
+
* available. Same key, same account, no human — a free pool nearly four times
|
|
58
|
+
* larger, which grows when the vendor adds models and shrinks when it removes
|
|
59
|
+
* them, without anyone being told.
|
|
60
|
+
*
|
|
61
|
+
* ── What this is NOT ─────────────────────────────────────────────────────────
|
|
62
|
+
* Presence in a catalogue is not proof a model works. This package already
|
|
63
|
+
* records two counter-examples: an id that answered "Provider returned error"
|
|
64
|
+
* on probe, and one that returned HTTP 200 with EMPTY content. Resolution fixes
|
|
65
|
+
* "the id does not exist"; it cannot fix "the id exists and misbehaves." That
|
|
66
|
+
* is what the chain's own fallback, and observed health, are for. Declared
|
|
67
|
+
* capability is a starting hypothesis — traffic is the evidence.
|
|
68
|
+
*/
|
|
69
|
+
import { type Env, type Provider } from "./chain.js";
|
|
70
|
+
/** What a discovered model must prove about itself before it joins the tail. */
|
|
71
|
+
export type Requirements = {
|
|
72
|
+
/** Vendor must publish a price and it must be zero. Default true. */
|
|
73
|
+
free?: boolean;
|
|
74
|
+
/** Vendor must declare text as its only output modality. Default true. */
|
|
75
|
+
textOnly?: boolean;
|
|
76
|
+
/** Vendor must declare tool/function calling. Default true. */
|
|
77
|
+
tools?: boolean;
|
|
78
|
+
/** Minimum published context window, when the vendor publishes one. */
|
|
79
|
+
minContext?: number;
|
|
80
|
+
};
|
|
81
|
+
export type ResolveOptions = {
|
|
82
|
+
env?: Env;
|
|
83
|
+
fetchImpl?: typeof fetch;
|
|
84
|
+
timeoutMs?: number;
|
|
85
|
+
/**
|
|
86
|
+
* Append live models the declaration never named. Default true — that is the
|
|
87
|
+
* half that makes the pool GROW without a human. Set false to verify the
|
|
88
|
+
* declared list against the catalogue and nothing more.
|
|
89
|
+
*/
|
|
90
|
+
discover?: boolean;
|
|
91
|
+
/** What a discovered model must declare. See Requirements. */
|
|
92
|
+
require?: Requirements;
|
|
93
|
+
/**
|
|
94
|
+
* Cap on discovered ids appended per provider. Default 10.
|
|
95
|
+
*
|
|
96
|
+
* A cap, not a preference: the chain is walked in order on failure, so an
|
|
97
|
+
* unbounded tail turns one bad minute at a vendor into dozens of sequential
|
|
98
|
+
* requests before the caller sees an error.
|
|
99
|
+
*/
|
|
100
|
+
maxDiscovered?: number;
|
|
101
|
+
/** Clock injection for tests. Default `Date.now`. */
|
|
102
|
+
now?: () => number;
|
|
103
|
+
};
|
|
104
|
+
export type ProviderResolution = {
|
|
105
|
+
provider: string;
|
|
106
|
+
/** The ids to actually try, in order: kept declarations, then discovered. */
|
|
107
|
+
models: string[];
|
|
108
|
+
/** Declared ids the vendor still lists. */
|
|
109
|
+
kept: string[];
|
|
110
|
+
/**
|
|
111
|
+
* Declared ids the vendor no longer lists. Dropped from `models` — this is
|
|
112
|
+
* the rot, already routed around rather than merely reported.
|
|
113
|
+
*/
|
|
114
|
+
dropped: string[];
|
|
115
|
+
/** Live ids the declaration never named, appended after the kept ones. */
|
|
116
|
+
discovered: string[];
|
|
117
|
+
/**
|
|
118
|
+
* True when the catalogue could not be read, so `models` is the declaration
|
|
119
|
+
* verbatim and NOTHING here was verified. `dropped` is empty because no id
|
|
120
|
+
* was confirmed gone — not because none is.
|
|
121
|
+
*/
|
|
122
|
+
unverified: boolean;
|
|
123
|
+
/** Ids in `models` the vendor says will stop working, with the date. */
|
|
124
|
+
expiring: Array<{
|
|
125
|
+
model: string;
|
|
126
|
+
on: string;
|
|
127
|
+
}>;
|
|
128
|
+
};
|
|
129
|
+
/**
|
|
130
|
+
* Resolve one chain against what its vendors currently offer.
|
|
131
|
+
*
|
|
132
|
+
* Costs one GET /models per provider and ZERO tokens — the same property that
|
|
133
|
+
* made the catalogue check schedulable makes this callable on a warm path, and
|
|
134
|
+
* consumers are expected to cache the result rather than resolve per request.
|
|
135
|
+
*/
|
|
136
|
+
export declare function resolveChain(chain: Provider[], opts?: ResolveOptions): Promise<ProviderResolution[]>;
|
|
137
|
+
/**
|
|
138
|
+
* Fold a resolution back into providers, ready for `usableChain`.
|
|
139
|
+
*
|
|
140
|
+
* A provider whose every model resolved away is left with an EMPTY model list
|
|
141
|
+
* rather than being dropped from the chain. `usableChain` already skips links
|
|
142
|
+
* with nothing to try, and keeping the row means the vendor still appears in
|
|
143
|
+
* reports — a silently vanished provider is how a chain quietly becomes a
|
|
144
|
+
* single point of failure without anyone noticing.
|
|
145
|
+
*/
|
|
146
|
+
export declare function applyResolution(chain: Provider[], resolved: ProviderResolution[]): Provider[];
|
|
147
|
+
/** True when any declared id was confirmed gone and routed around. */
|
|
148
|
+
export declare function routedAroundRot(resolved: ProviderResolution[]): boolean;
|
|
149
|
+
/**
|
|
150
|
+
* Providers left with nothing to try, despite a readable catalogue.
|
|
151
|
+
*
|
|
152
|
+
* Kept separate from "could not look" on purpose: a vendor whose catalogue
|
|
153
|
+
* answered and contained none of our models is a real lost link, while an
|
|
154
|
+
* unreadable one is an unknown. Collapsing them produces either a false alarm
|
|
155
|
+
* every time a key expires, or silence when a vendor actually goes away.
|
|
156
|
+
*/
|
|
157
|
+
export declare function emptyProviders(resolved: ProviderResolution[]): string[];
|
|
158
|
+
/** Human-readable report. Could-not-look stays visibly distinct from a pass. */
|
|
159
|
+
export declare function resolutionReport(resolved: ProviderResolution[]): string;
|
package/dist/resolve.js
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* resolve — stop writing model ids down.
|
|
3
|
+
*
|
|
4
|
+
* ── The problem the chain did not solve ──────────────────────────────────────
|
|
5
|
+
* `chain.ts` replaced one pinned model with a list of pinned models, and says so
|
|
6
|
+
* itself: "a single pinned free model is not a configuration, it is a scheduled
|
|
7
|
+
* outage." `catalog.ts` then noticed the list rots too, and added a check. But a
|
|
8
|
+
* check only produces a SENTENCE. Somebody still has to read it, edit a file,
|
|
9
|
+
* open a pull request, wait for CI, and deploy — and until they do, the app is
|
|
10
|
+
* handing users a model that answers 404.
|
|
11
|
+
*
|
|
12
|
+
* That round trip is not hypothetical, it is this module's origin. On
|
|
13
|
+
* 2026-09-13 a freshly scheduled health check alerted within minutes of its
|
|
14
|
+
* first run: two ids in a consumer's registry were gone from OpenRouter. The
|
|
15
|
+
* alert was correct, fast, and completely dependent on a human being available
|
|
16
|
+
* to act on it. The vendor had published the truth the whole time.
|
|
17
|
+
*
|
|
18
|
+
* So: keep the declared list, but treat it as a PREFERENCE, and let the vendor's
|
|
19
|
+
* own catalogue decide what is actually callable. A retirement stops being an
|
|
20
|
+
* incident and becomes a no-op.
|
|
21
|
+
*
|
|
22
|
+
* ── Three rules, each paid for by a real failure ─────────────────────────────
|
|
23
|
+
*
|
|
24
|
+
* 1. NEVER SHRINK ON IGNORANCE. An unreadable catalogue leaves the declared
|
|
25
|
+
* list exactly as written and sets `unverified`. While building this, a
|
|
26
|
+
* checkout without GROQ_API_KEY got a 401, and the first draft of the probe
|
|
27
|
+
* printed "GONE" for both Groq models — both of which were live. Resolution
|
|
28
|
+
* that trusted that would let an expired key empty the chain.
|
|
29
|
+
*
|
|
30
|
+
* 2. DISCOVERY EXTENDS THE TAIL, NEVER THE HEAD. Declared ids were curated —
|
|
31
|
+
* in this package's case probed with a real tool call, because five of nine
|
|
32
|
+
* free models turned out to emit tool calls only as text. A discovered id
|
|
33
|
+
* has no such evidence behind it, so it may be a last resort and never a
|
|
34
|
+
* first choice.
|
|
35
|
+
*
|
|
36
|
+
* 3. DISCOVER ONLY WHAT THE VENDOR CALLS FREE, AND ONLY WHAT IT CALLS USABLE.
|
|
37
|
+
* Both halves matter, and both are read from the vendor rather than guessed:
|
|
38
|
+
*
|
|
39
|
+
* - Price. Not the `:free` suffix — the published number. Checked live on
|
|
40
|
+
* 2026-09-13, OpenRouter listed 19 ids ending `:free` and 22 priced at
|
|
41
|
+
* zero. Groq publishes prices too, and NONE of its models are zero, so
|
|
42
|
+
* discovery there correctly finds nothing and that vendor stays curated.
|
|
43
|
+
* A suffix heuristic would have gone looking for `:free` at a vendor
|
|
44
|
+
* that does not use the convention; a price check just declines.
|
|
45
|
+
*
|
|
46
|
+
* - Suitability. Of the 19 free OpenRouter ids, one was
|
|
47
|
+
* `nemotron-3.5-content-safety` — a classifier, and the only one of the
|
|
48
|
+
* 19 declaring `tools: false`. Two more zero-priced ids were
|
|
49
|
+
* `lyria-3-*`, which emit AUDIO. Adding all zero-priced models to a chat
|
|
50
|
+
* chain would have put a safety classifier and a music generator in
|
|
51
|
+
* front of a user asking a question. Requiring text output and declared
|
|
52
|
+
* tool support removes exactly those three and nothing else.
|
|
53
|
+
*
|
|
54
|
+
* ── What this is worth ───────────────────────────────────────────────────────
|
|
55
|
+
* Measured against the live catalogue on 2026-09-13: `freeChain()` names five
|
|
56
|
+
* OpenRouter ids by hand; nineteen zero-priced, text-out, tool-capable ids were
|
|
57
|
+
* available. Same key, same account, no human — a free pool nearly four times
|
|
58
|
+
* larger, which grows when the vendor adds models and shrinks when it removes
|
|
59
|
+
* them, without anyone being told.
|
|
60
|
+
*
|
|
61
|
+
* ── What this is NOT ─────────────────────────────────────────────────────────
|
|
62
|
+
* Presence in a catalogue is not proof a model works. This package already
|
|
63
|
+
* records two counter-examples: an id that answered "Provider returned error"
|
|
64
|
+
* on probe, and one that returned HTTP 200 with EMPTY content. Resolution fixes
|
|
65
|
+
* "the id does not exist"; it cannot fix "the id exists and misbehaves." That
|
|
66
|
+
* is what the chain's own fallback, and observed health, are for. Declared
|
|
67
|
+
* capability is a starting hypothesis — traffic is the evidence.
|
|
68
|
+
*/
|
|
69
|
+
import { providerModels } from "./chain.js";
|
|
70
|
+
import { fetchCatalog } from "./catalog-fetch.js";
|
|
71
|
+
const DEFAULTS = {
|
|
72
|
+
free: true,
|
|
73
|
+
textOnly: true,
|
|
74
|
+
tools: true,
|
|
75
|
+
};
|
|
76
|
+
/**
|
|
77
|
+
* Does this record satisfy the requirement, reading "the vendor did not say" as
|
|
78
|
+
* a failure?
|
|
79
|
+
*
|
|
80
|
+
* Silence is refused rather than assumed. A model that does not declare a price
|
|
81
|
+
* might be free; if it is not, discovery has quietly started spending money,
|
|
82
|
+
* which is the one failure here with a bill attached. The cost of being wrong is
|
|
83
|
+
* asymmetric, so the default leans to declining.
|
|
84
|
+
*/
|
|
85
|
+
function meets(m, req) {
|
|
86
|
+
const want = { ...DEFAULTS, ...req };
|
|
87
|
+
if (want.free && m.costsNothing !== true)
|
|
88
|
+
return false;
|
|
89
|
+
if (want.textOnly && !(m.outputModalities?.length === 1 && m.outputModalities[0] === "text"))
|
|
90
|
+
return false;
|
|
91
|
+
if (want.tools && m.tools !== true)
|
|
92
|
+
return false;
|
|
93
|
+
if (req.minContext !== undefined && (m.contextLength ?? 0) < req.minContext)
|
|
94
|
+
return false;
|
|
95
|
+
return true;
|
|
96
|
+
}
|
|
97
|
+
/** Has the vendor's own stated end-date already passed? */
|
|
98
|
+
function expired(m, now) {
|
|
99
|
+
if (!m.expiresOn)
|
|
100
|
+
return false;
|
|
101
|
+
const t = Date.parse(`${m.expiresOn}T23:59:59Z`);
|
|
102
|
+
return Number.isFinite(t) && t < now;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Resolve one chain against what its vendors currently offer.
|
|
106
|
+
*
|
|
107
|
+
* Costs one GET /models per provider and ZERO tokens — the same property that
|
|
108
|
+
* made the catalogue check schedulable makes this callable on a warm path, and
|
|
109
|
+
* consumers are expected to cache the result rather than resolve per request.
|
|
110
|
+
*/
|
|
111
|
+
export async function resolveChain(chain, opts = {}) {
|
|
112
|
+
const env = opts.env ?? process.env;
|
|
113
|
+
const discover = opts.discover ?? true;
|
|
114
|
+
const req = opts.require ?? {};
|
|
115
|
+
const maxDiscovered = opts.maxDiscovered ?? 10;
|
|
116
|
+
const now = (opts.now ?? Date.now)();
|
|
117
|
+
const out = [];
|
|
118
|
+
for (const provider of chain) {
|
|
119
|
+
const declared = providerModels(provider, env);
|
|
120
|
+
const records = await fetchCatalog(provider.baseUrl, env[provider.keyEnv], {
|
|
121
|
+
fetchImpl: opts.fetchImpl,
|
|
122
|
+
timeoutMs: opts.timeoutMs,
|
|
123
|
+
});
|
|
124
|
+
// Rule 1. Could not look ⇒ change nothing, and say so.
|
|
125
|
+
if (!records) {
|
|
126
|
+
out.push({
|
|
127
|
+
provider: provider.id,
|
|
128
|
+
models: declared,
|
|
129
|
+
kept: [],
|
|
130
|
+
dropped: [],
|
|
131
|
+
discovered: [],
|
|
132
|
+
unverified: true,
|
|
133
|
+
expiring: [],
|
|
134
|
+
});
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
const byId = new Map(records.map((r) => [r.id, r]));
|
|
138
|
+
const kept = declared.filter((m) => {
|
|
139
|
+
const rec = byId.get(m);
|
|
140
|
+
return rec !== undefined && !expired(rec, now);
|
|
141
|
+
});
|
|
142
|
+
const dropped = declared.filter((m) => !kept.includes(m));
|
|
143
|
+
// Rule 2. Discovered ids go after the curated ones, never before.
|
|
144
|
+
const discovered = discover
|
|
145
|
+
? records
|
|
146
|
+
.filter((r) => !declared.includes(r.id) && !expired(r, now) && meets(r, req))
|
|
147
|
+
.map((r) => r.id)
|
|
148
|
+
.sort()
|
|
149
|
+
.slice(0, maxDiscovered)
|
|
150
|
+
: [];
|
|
151
|
+
const models = [...kept, ...discovered];
|
|
152
|
+
const expiring = models
|
|
153
|
+
.map((m) => ({ model: m, on: byId.get(m)?.expiresOn ?? null }))
|
|
154
|
+
.filter((e) => e.on !== null);
|
|
155
|
+
out.push({
|
|
156
|
+
provider: provider.id,
|
|
157
|
+
models,
|
|
158
|
+
kept,
|
|
159
|
+
dropped,
|
|
160
|
+
discovered,
|
|
161
|
+
unverified: false,
|
|
162
|
+
expiring,
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
return out;
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Fold a resolution back into providers, ready for `usableChain`.
|
|
169
|
+
*
|
|
170
|
+
* A provider whose every model resolved away is left with an EMPTY model list
|
|
171
|
+
* rather than being dropped from the chain. `usableChain` already skips links
|
|
172
|
+
* with nothing to try, and keeping the row means the vendor still appears in
|
|
173
|
+
* reports — a silently vanished provider is how a chain quietly becomes a
|
|
174
|
+
* single point of failure without anyone noticing.
|
|
175
|
+
*/
|
|
176
|
+
export function applyResolution(chain, resolved) {
|
|
177
|
+
const byId = new Map(resolved.map((r) => [r.provider, r]));
|
|
178
|
+
return chain.map((p) => {
|
|
179
|
+
const r = byId.get(p.id);
|
|
180
|
+
return r ? { ...p, models: r.models } : p;
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
/** True when any declared id was confirmed gone and routed around. */
|
|
184
|
+
export function routedAroundRot(resolved) {
|
|
185
|
+
return resolved.some((r) => r.dropped.length > 0);
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Providers left with nothing to try, despite a readable catalogue.
|
|
189
|
+
*
|
|
190
|
+
* Kept separate from "could not look" on purpose: a vendor whose catalogue
|
|
191
|
+
* answered and contained none of our models is a real lost link, while an
|
|
192
|
+
* unreadable one is an unknown. Collapsing them produces either a false alarm
|
|
193
|
+
* every time a key expires, or silence when a vendor actually goes away.
|
|
194
|
+
*/
|
|
195
|
+
export function emptyProviders(resolved) {
|
|
196
|
+
return resolved.filter((r) => !r.unverified && r.models.length === 0).map((r) => r.provider);
|
|
197
|
+
}
|
|
198
|
+
/** Human-readable report. Could-not-look stays visibly distinct from a pass. */
|
|
199
|
+
export function resolutionReport(resolved) {
|
|
200
|
+
const lines = [];
|
|
201
|
+
for (const r of resolved) {
|
|
202
|
+
if (r.unverified) {
|
|
203
|
+
lines.push(`? ${r.provider}: catalogue unreadable — using the declared ${r.models.length} id(s) UNVERIFIED`);
|
|
204
|
+
continue;
|
|
205
|
+
}
|
|
206
|
+
lines.push(` ${r.provider}: ${r.models.length} model(s) — ${r.kept.length} declared, ${r.discovered.length} discovered`);
|
|
207
|
+
for (const m of r.dropped)
|
|
208
|
+
lines.push(` GONE, routed around: ${m}`);
|
|
209
|
+
for (const m of r.discovered)
|
|
210
|
+
lines.push(` + ${m}`);
|
|
211
|
+
for (const e of r.expiring)
|
|
212
|
+
lines.push(` ! ${e.model} — vendor says it ends ${e.on}`);
|
|
213
|
+
}
|
|
214
|
+
const empty = emptyProviders(resolved);
|
|
215
|
+
if (empty.length)
|
|
216
|
+
lines.push(`\nNothing left to try at: ${empty.join(", ")} — that vendor is gone from the chain.`);
|
|
217
|
+
return lines.join("\n");
|
|
218
|
+
}
|
package/package.json
CHANGED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One GET /models, parsed once, for everything that needs to know what a vendor
|
|
3
|
+
* currently offers.
|
|
4
|
+
*
|
|
5
|
+
* `catalog.ts` already fetched this list to answer "has a pinned id rotted".
|
|
6
|
+
* `resolve.ts` needs the SAME request to answer "what else is on the shelf, and
|
|
7
|
+
* what does the vendor say about it" — and the ids alone cannot answer that.
|
|
8
|
+
*
|
|
9
|
+
* Rather than issue a second, slightly different GET (the exact failure
|
|
10
|
+
* catalog.ts's own header complains about — "the app that wrote its own first
|
|
11
|
+
* wrote it slightly differently"), both read this.
|
|
12
|
+
*
|
|
13
|
+
* ── Two vendors, two schemas, one shape ──────────────────────────────────────
|
|
14
|
+
* Verified against live responses on 2026-09-13, and they do NOT agree:
|
|
15
|
+
*
|
|
16
|
+
* OpenRouter architecture.output_modalities supported_parameters: ["tools"]
|
|
17
|
+
* Groq output_modalities (top level) supported_features: ["json_mode"]
|
|
18
|
+
*
|
|
19
|
+
* So a reader that knows only OpenRouter's shape reports every Groq model as
|
|
20
|
+
* having no text output and no tools — which, in a filter, silently removes a
|
|
21
|
+
* working vendor. Normalising here means each consumer states its requirement
|
|
22
|
+
* once instead of learning both schemas.
|
|
23
|
+
*
|
|
24
|
+
* ── Unknown is a value, not a default ────────────────────────────────────────
|
|
25
|
+
* Every normalised field is nullable, and null means "the vendor did not say".
|
|
26
|
+
* That is deliberately distinct from false. A filter that treats "did not say"
|
|
27
|
+
* as "does not support" narrows silently as vendors change their schemas; one
|
|
28
|
+
* that treats it as "supports" invents capability. Callers must choose, and the
|
|
29
|
+
* choice is visible at the call site because the type forces it.
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
/** What a vendor's catalogue says about one model, in one shape. */
|
|
33
|
+
export type ModelRecord = {
|
|
34
|
+
id: string;
|
|
35
|
+
/** Output modalities, or null when the vendor does not publish them. */
|
|
36
|
+
outputModalities: string[] | null;
|
|
37
|
+
/**
|
|
38
|
+
* True only when the vendor publishes a price and every component is zero.
|
|
39
|
+
* Null when it publishes no price at all.
|
|
40
|
+
*
|
|
41
|
+
* Not the same question as "does the `:free` suffix appear". Checked live on
|
|
42
|
+
* 2026-09-13: OpenRouter lists 19 ids ending `:free` and 22 priced at zero —
|
|
43
|
+
* and the three zero-priced ids WITHOUT the suffix include `openrouter/free`,
|
|
44
|
+
* the auto-router that is the most rot-resistant entry in the whole chain.
|
|
45
|
+
* A suffix check misses it. Price is the vendor telling you who pays.
|
|
46
|
+
*/
|
|
47
|
+
costsNothing: boolean | null;
|
|
48
|
+
/** True when the vendor declares tool/function calling. Null when unstated. */
|
|
49
|
+
tools: boolean | null;
|
|
50
|
+
/** Vendor-declared context window, when published. */
|
|
51
|
+
contextLength: number | null;
|
|
52
|
+
/**
|
|
53
|
+
* Date the vendor says this id stops working (ISO yyyy-mm-dd), when it says so.
|
|
54
|
+
*
|
|
55
|
+
* Rare but real: of 445 OpenRouter models on 2026-09-13, five carried one, and
|
|
56
|
+
* one of those was a FREE model expiring in 17 days. Where present this turns
|
|
57
|
+
* rot from something detected afterwards into something known in advance, so
|
|
58
|
+
* it is worth surfacing even though most models omit it.
|
|
59
|
+
*/
|
|
60
|
+
expiresOn: string | null;
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
type RawModel = Record<string, unknown>;
|
|
64
|
+
|
|
65
|
+
function asStringArray(v: unknown): string[] | null {
|
|
66
|
+
if (!Array.isArray(v)) return null;
|
|
67
|
+
const out = v.filter((x): x is string => typeof x === "string");
|
|
68
|
+
return out.length > 0 ? out : null;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Zero only when a price is published AND every numeric component is zero. */
|
|
72
|
+
function priceIsZero(pricing: unknown): boolean | null {
|
|
73
|
+
if (!pricing || typeof pricing !== "object") return null;
|
|
74
|
+
const entries = Object.entries(pricing as Record<string, unknown>);
|
|
75
|
+
const numbers = entries
|
|
76
|
+
.map(([, v]) => (typeof v === "string" || typeof v === "number" ? Number(v) : NaN))
|
|
77
|
+
.filter((n) => Number.isFinite(n));
|
|
78
|
+
if (numbers.length === 0) return null;
|
|
79
|
+
return numbers.every((n) => n === 0);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Tool support as DECLARED by the vendor, across both known schemas. */
|
|
83
|
+
function declaresTools(m: RawModel): boolean | null {
|
|
84
|
+
const params = asStringArray(m.supported_parameters);
|
|
85
|
+
if (params) return params.includes("tools");
|
|
86
|
+
const features = asStringArray(m.supported_features);
|
|
87
|
+
if (features) return features.some((f) => f === "tools" || f === "tool_use");
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function normalise(m: RawModel): ModelRecord | null {
|
|
92
|
+
const id = typeof m.id === "string" ? m.id : "";
|
|
93
|
+
if (!id) return null;
|
|
94
|
+
const arch = (m.architecture ?? {}) as RawModel;
|
|
95
|
+
const expires = m.expiration_date;
|
|
96
|
+
return {
|
|
97
|
+
id,
|
|
98
|
+
outputModalities: asStringArray(arch.output_modalities) ?? asStringArray(m.output_modalities),
|
|
99
|
+
costsNothing: priceIsZero(m.pricing),
|
|
100
|
+
tools: declaresTools(m),
|
|
101
|
+
contextLength: typeof m.context_length === "number" ? m.context_length : null,
|
|
102
|
+
expiresOn: typeof expires === "string" && expires.trim() ? expires.trim() : null,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export type FetchCatalogOptions = {
|
|
107
|
+
fetchImpl?: typeof fetch;
|
|
108
|
+
timeoutMs?: number;
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Every model one vendor lists, normalised — or NULL when the catalogue could
|
|
113
|
+
* not be read.
|
|
114
|
+
*
|
|
115
|
+
* Null covers no key, a network failure, a non-200, an unparseable body, and a
|
|
116
|
+
* body that parses but lists nothing. It never means "this vendor has no
|
|
117
|
+
* models", and the distinction is not academic: while building this, a local
|
|
118
|
+
* checkout missing GROQ_API_KEY answered 401, and a reader that collapsed that
|
|
119
|
+
* into an empty list reported BOTH live Groq models as retired. Had resolution
|
|
120
|
+
* trusted it, an expired key would have emptied the chain rather than failing a
|
|
121
|
+
* single call.
|
|
122
|
+
*/
|
|
123
|
+
export async function fetchCatalog(
|
|
124
|
+
baseUrl: string,
|
|
125
|
+
key: string | undefined,
|
|
126
|
+
opts: FetchCatalogOptions = {},
|
|
127
|
+
): Promise<ModelRecord[] | null> {
|
|
128
|
+
if (!key?.trim()) return null;
|
|
129
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
130
|
+
const timeoutMs = opts.timeoutMs ?? 20_000;
|
|
131
|
+
try {
|
|
132
|
+
const res = await fetchImpl(`${baseUrl.replace(/\/$/, "")}/models`, {
|
|
133
|
+
headers: { Authorization: `Bearer ${key.trim()}` },
|
|
134
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
135
|
+
});
|
|
136
|
+
if (!res.ok) return null;
|
|
137
|
+
const body = (await res.json()) as { data?: unknown };
|
|
138
|
+
if (!Array.isArray(body?.data)) return null;
|
|
139
|
+
const records = body.data
|
|
140
|
+
.map((m) => normalise((m ?? {}) as RawModel))
|
|
141
|
+
.filter((m): m is ModelRecord => m !== null);
|
|
142
|
+
return records.length > 0 ? records : null;
|
|
143
|
+
} catch {
|
|
144
|
+
return null;
|
|
145
|
+
}
|
|
146
|
+
}
|
package/src/catalog.ts
CHANGED
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
*/
|
|
24
24
|
|
|
25
25
|
import { providerModels, type Env, type Provider } from "./chain.js";
|
|
26
|
+
import { fetchCatalog } from "./catalog-fetch.js";
|
|
26
27
|
|
|
27
28
|
export type CatalogVerdict = {
|
|
28
29
|
provider: string;
|
|
@@ -50,31 +51,27 @@ export type CheckCatalogOptions = {
|
|
|
50
51
|
timeoutMs?: number;
|
|
51
52
|
};
|
|
52
53
|
|
|
53
|
-
/**
|
|
54
|
+
/**
|
|
55
|
+
* Ids listed by one provider, or null when the catalogue could not be read.
|
|
56
|
+
*
|
|
57
|
+
* The request itself now lives in catalog-fetch.ts, because `resolve.ts` needs
|
|
58
|
+
* the SAME GET and the ids alone cannot tell it what a model costs or can do.
|
|
59
|
+
* Two near-identical fetches drifting apart is the failure this module's own
|
|
60
|
+
* header objects to; one fetch, two readers.
|
|
61
|
+
*
|
|
62
|
+
* Every null case is preserved exactly — no key, network failure, non-200,
|
|
63
|
+
* unparseable body, and a body that parses but lists nothing (a malformed
|
|
64
|
+
* answer, not a vendor with no models: refusing it keeps a bad response from
|
|
65
|
+
* reading as total rot).
|
|
66
|
+
*/
|
|
54
67
|
async function liveIds(
|
|
55
68
|
provider: Provider,
|
|
56
69
|
key: string,
|
|
57
70
|
fetchImpl: typeof fetch,
|
|
58
71
|
timeoutMs: number,
|
|
59
72
|
): Promise<string[] | null> {
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
headers: { Authorization: `Bearer ${key}` },
|
|
63
|
-
signal: AbortSignal.timeout(timeoutMs),
|
|
64
|
-
});
|
|
65
|
-
if (!res.ok) return null;
|
|
66
|
-
const body = (await res.json()) as { data?: Array<{ id?: unknown }> };
|
|
67
|
-
if (!Array.isArray(body?.data)) return null;
|
|
68
|
-
const ids = body.data
|
|
69
|
-
.map((m) => (typeof m?.id === "string" ? m.id : ""))
|
|
70
|
-
.filter((id): id is string => id.length > 0);
|
|
71
|
-
// A catalogue that parses but lists nothing is a malformed answer, not a
|
|
72
|
-
// vendor with no models. Refusing it keeps a bad response from reading as
|
|
73
|
-
// total rot.
|
|
74
|
-
return ids.length > 0 ? ids : null;
|
|
75
|
-
} catch {
|
|
76
|
-
return null;
|
|
77
|
-
}
|
|
73
|
+
const records = await fetchCatalog(provider.baseUrl, key, { fetchImpl, timeoutMs });
|
|
74
|
+
return records ? records.map((r) => r.id) : null;
|
|
78
75
|
}
|
|
79
76
|
|
|
80
77
|
/**
|
package/src/index.ts
CHANGED
|
@@ -75,6 +75,22 @@ export {
|
|
|
75
75
|
catalogReport,
|
|
76
76
|
} from "./catalog.js";
|
|
77
77
|
|
|
78
|
+
export { type ModelRecord, type FetchCatalogOptions, fetchCatalog } from "./catalog-fetch.js";
|
|
79
|
+
|
|
80
|
+
// Resolution is the answer to the question `catalog` only asks. checkCatalog
|
|
81
|
+
// REPORTS that a pinned id is gone; resolveChain drops it and calls the next
|
|
82
|
+
// one, so a vendor retirement stops needing a pull request to survive.
|
|
83
|
+
export {
|
|
84
|
+
type Requirements,
|
|
85
|
+
type ResolveOptions,
|
|
86
|
+
type ProviderResolution,
|
|
87
|
+
resolveChain,
|
|
88
|
+
applyResolution,
|
|
89
|
+
routedAroundRot,
|
|
90
|
+
emptyProviders,
|
|
91
|
+
resolutionReport,
|
|
92
|
+
} from "./resolve.js";
|
|
93
|
+
|
|
78
94
|
export {
|
|
79
95
|
type ChainAttemptFailure,
|
|
80
96
|
type TryChainOptions,
|
package/src/resolve.ts
ADDED
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* resolve — stop writing model ids down.
|
|
3
|
+
*
|
|
4
|
+
* ── The problem the chain did not solve ──────────────────────────────────────
|
|
5
|
+
* `chain.ts` replaced one pinned model with a list of pinned models, and says so
|
|
6
|
+
* itself: "a single pinned free model is not a configuration, it is a scheduled
|
|
7
|
+
* outage." `catalog.ts` then noticed the list rots too, and added a check. But a
|
|
8
|
+
* check only produces a SENTENCE. Somebody still has to read it, edit a file,
|
|
9
|
+
* open a pull request, wait for CI, and deploy — and until they do, the app is
|
|
10
|
+
* handing users a model that answers 404.
|
|
11
|
+
*
|
|
12
|
+
* That round trip is not hypothetical, it is this module's origin. On
|
|
13
|
+
* 2026-09-13 a freshly scheduled health check alerted within minutes of its
|
|
14
|
+
* first run: two ids in a consumer's registry were gone from OpenRouter. The
|
|
15
|
+
* alert was correct, fast, and completely dependent on a human being available
|
|
16
|
+
* to act on it. The vendor had published the truth the whole time.
|
|
17
|
+
*
|
|
18
|
+
* So: keep the declared list, but treat it as a PREFERENCE, and let the vendor's
|
|
19
|
+
* own catalogue decide what is actually callable. A retirement stops being an
|
|
20
|
+
* incident and becomes a no-op.
|
|
21
|
+
*
|
|
22
|
+
* ── Three rules, each paid for by a real failure ─────────────────────────────
|
|
23
|
+
*
|
|
24
|
+
* 1. NEVER SHRINK ON IGNORANCE. An unreadable catalogue leaves the declared
|
|
25
|
+
* list exactly as written and sets `unverified`. While building this, a
|
|
26
|
+
* checkout without GROQ_API_KEY got a 401, and the first draft of the probe
|
|
27
|
+
* printed "GONE" for both Groq models — both of which were live. Resolution
|
|
28
|
+
* that trusted that would let an expired key empty the chain.
|
|
29
|
+
*
|
|
30
|
+
* 2. DISCOVERY EXTENDS THE TAIL, NEVER THE HEAD. Declared ids were curated —
|
|
31
|
+
* in this package's case probed with a real tool call, because five of nine
|
|
32
|
+
* free models turned out to emit tool calls only as text. A discovered id
|
|
33
|
+
* has no such evidence behind it, so it may be a last resort and never a
|
|
34
|
+
* first choice.
|
|
35
|
+
*
|
|
36
|
+
* 3. DISCOVER ONLY WHAT THE VENDOR CALLS FREE, AND ONLY WHAT IT CALLS USABLE.
|
|
37
|
+
* Both halves matter, and both are read from the vendor rather than guessed:
|
|
38
|
+
*
|
|
39
|
+
* - Price. Not the `:free` suffix — the published number. Checked live on
|
|
40
|
+
* 2026-09-13, OpenRouter listed 19 ids ending `:free` and 22 priced at
|
|
41
|
+
* zero. Groq publishes prices too, and NONE of its models are zero, so
|
|
42
|
+
* discovery there correctly finds nothing and that vendor stays curated.
|
|
43
|
+
* A suffix heuristic would have gone looking for `:free` at a vendor
|
|
44
|
+
* that does not use the convention; a price check just declines.
|
|
45
|
+
*
|
|
46
|
+
* - Suitability. Of the 19 free OpenRouter ids, one was
|
|
47
|
+
* `nemotron-3.5-content-safety` — a classifier, and the only one of the
|
|
48
|
+
* 19 declaring `tools: false`. Two more zero-priced ids were
|
|
49
|
+
* `lyria-3-*`, which emit AUDIO. Adding all zero-priced models to a chat
|
|
50
|
+
* chain would have put a safety classifier and a music generator in
|
|
51
|
+
* front of a user asking a question. Requiring text output and declared
|
|
52
|
+
* tool support removes exactly those three and nothing else.
|
|
53
|
+
*
|
|
54
|
+
* ── What this is worth ───────────────────────────────────────────────────────
|
|
55
|
+
* Measured against the live catalogue on 2026-09-13: `freeChain()` names five
|
|
56
|
+
* OpenRouter ids by hand; nineteen zero-priced, text-out, tool-capable ids were
|
|
57
|
+
* available. Same key, same account, no human — a free pool nearly four times
|
|
58
|
+
* larger, which grows when the vendor adds models and shrinks when it removes
|
|
59
|
+
* them, without anyone being told.
|
|
60
|
+
*
|
|
61
|
+
* ── What this is NOT ─────────────────────────────────────────────────────────
|
|
62
|
+
* Presence in a catalogue is not proof a model works. This package already
|
|
63
|
+
* records two counter-examples: an id that answered "Provider returned error"
|
|
64
|
+
* on probe, and one that returned HTTP 200 with EMPTY content. Resolution fixes
|
|
65
|
+
* "the id does not exist"; it cannot fix "the id exists and misbehaves." That
|
|
66
|
+
* is what the chain's own fallback, and observed health, are for. Declared
|
|
67
|
+
* capability is a starting hypothesis — traffic is the evidence.
|
|
68
|
+
*/
|
|
69
|
+
|
|
70
|
+
import { providerModels, type Env, type Provider } from "./chain.js";
|
|
71
|
+
import { fetchCatalog, type ModelRecord } from "./catalog-fetch.js";
|
|
72
|
+
|
|
73
|
+
/** What a discovered model must prove about itself before it joins the tail. */
|
|
74
|
+
export type Requirements = {
|
|
75
|
+
/** Vendor must publish a price and it must be zero. Default true. */
|
|
76
|
+
free?: boolean;
|
|
77
|
+
/** Vendor must declare text as its only output modality. Default true. */
|
|
78
|
+
textOnly?: boolean;
|
|
79
|
+
/** Vendor must declare tool/function calling. Default true. */
|
|
80
|
+
tools?: boolean;
|
|
81
|
+
/** Minimum published context window, when the vendor publishes one. */
|
|
82
|
+
minContext?: number;
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
export type ResolveOptions = {
|
|
86
|
+
env?: Env;
|
|
87
|
+
fetchImpl?: typeof fetch;
|
|
88
|
+
timeoutMs?: number;
|
|
89
|
+
/**
|
|
90
|
+
* Append live models the declaration never named. Default true — that is the
|
|
91
|
+
* half that makes the pool GROW without a human. Set false to verify the
|
|
92
|
+
* declared list against the catalogue and nothing more.
|
|
93
|
+
*/
|
|
94
|
+
discover?: boolean;
|
|
95
|
+
/** What a discovered model must declare. See Requirements. */
|
|
96
|
+
require?: Requirements;
|
|
97
|
+
/**
|
|
98
|
+
* Cap on discovered ids appended per provider. Default 10.
|
|
99
|
+
*
|
|
100
|
+
* A cap, not a preference: the chain is walked in order on failure, so an
|
|
101
|
+
* unbounded tail turns one bad minute at a vendor into dozens of sequential
|
|
102
|
+
* requests before the caller sees an error.
|
|
103
|
+
*/
|
|
104
|
+
maxDiscovered?: number;
|
|
105
|
+
/** Clock injection for tests. Default `Date.now`. */
|
|
106
|
+
now?: () => number;
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
export type ProviderResolution = {
|
|
110
|
+
provider: string;
|
|
111
|
+
/** The ids to actually try, in order: kept declarations, then discovered. */
|
|
112
|
+
models: string[];
|
|
113
|
+
/** Declared ids the vendor still lists. */
|
|
114
|
+
kept: string[];
|
|
115
|
+
/**
|
|
116
|
+
* Declared ids the vendor no longer lists. Dropped from `models` — this is
|
|
117
|
+
* the rot, already routed around rather than merely reported.
|
|
118
|
+
*/
|
|
119
|
+
dropped: string[];
|
|
120
|
+
/** Live ids the declaration never named, appended after the kept ones. */
|
|
121
|
+
discovered: string[];
|
|
122
|
+
/**
|
|
123
|
+
* True when the catalogue could not be read, so `models` is the declaration
|
|
124
|
+
* verbatim and NOTHING here was verified. `dropped` is empty because no id
|
|
125
|
+
* was confirmed gone — not because none is.
|
|
126
|
+
*/
|
|
127
|
+
unverified: boolean;
|
|
128
|
+
/** Ids in `models` the vendor says will stop working, with the date. */
|
|
129
|
+
expiring: Array<{ model: string; on: string }>;
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
const DEFAULTS: Required<Omit<Requirements, "minContext">> = {
|
|
133
|
+
free: true,
|
|
134
|
+
textOnly: true,
|
|
135
|
+
tools: true,
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Does this record satisfy the requirement, reading "the vendor did not say" as
|
|
140
|
+
* a failure?
|
|
141
|
+
*
|
|
142
|
+
* Silence is refused rather than assumed. A model that does not declare a price
|
|
143
|
+
* might be free; if it is not, discovery has quietly started spending money,
|
|
144
|
+
* which is the one failure here with a bill attached. The cost of being wrong is
|
|
145
|
+
* asymmetric, so the default leans to declining.
|
|
146
|
+
*/
|
|
147
|
+
function meets(m: ModelRecord, req: Requirements): boolean {
|
|
148
|
+
const want = { ...DEFAULTS, ...req };
|
|
149
|
+
if (want.free && m.costsNothing !== true) return false;
|
|
150
|
+
if (want.textOnly && !(m.outputModalities?.length === 1 && m.outputModalities[0] === "text"))
|
|
151
|
+
return false;
|
|
152
|
+
if (want.tools && m.tools !== true) return false;
|
|
153
|
+
if (req.minContext !== undefined && (m.contextLength ?? 0) < req.minContext) return false;
|
|
154
|
+
return true;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** Has the vendor's own stated end-date already passed? */
|
|
158
|
+
function expired(m: ModelRecord, now: number): boolean {
|
|
159
|
+
if (!m.expiresOn) return false;
|
|
160
|
+
const t = Date.parse(`${m.expiresOn}T23:59:59Z`);
|
|
161
|
+
return Number.isFinite(t) && t < now;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Resolve one chain against what its vendors currently offer.
|
|
166
|
+
*
|
|
167
|
+
* Costs one GET /models per provider and ZERO tokens — the same property that
|
|
168
|
+
* made the catalogue check schedulable makes this callable on a warm path, and
|
|
169
|
+
* consumers are expected to cache the result rather than resolve per request.
|
|
170
|
+
*/
|
|
171
|
+
export async function resolveChain(
|
|
172
|
+
chain: Provider[],
|
|
173
|
+
opts: ResolveOptions = {},
|
|
174
|
+
): Promise<ProviderResolution[]> {
|
|
175
|
+
const env = opts.env ?? process.env;
|
|
176
|
+
const discover = opts.discover ?? true;
|
|
177
|
+
const req = opts.require ?? {};
|
|
178
|
+
const maxDiscovered = opts.maxDiscovered ?? 10;
|
|
179
|
+
const now = (opts.now ?? Date.now)();
|
|
180
|
+
|
|
181
|
+
const out: ProviderResolution[] = [];
|
|
182
|
+
for (const provider of chain) {
|
|
183
|
+
const declared = providerModels(provider, env);
|
|
184
|
+
const records = await fetchCatalog(provider.baseUrl, env[provider.keyEnv], {
|
|
185
|
+
fetchImpl: opts.fetchImpl,
|
|
186
|
+
timeoutMs: opts.timeoutMs,
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
// Rule 1. Could not look ⇒ change nothing, and say so.
|
|
190
|
+
if (!records) {
|
|
191
|
+
out.push({
|
|
192
|
+
provider: provider.id,
|
|
193
|
+
models: declared,
|
|
194
|
+
kept: [],
|
|
195
|
+
dropped: [],
|
|
196
|
+
discovered: [],
|
|
197
|
+
unverified: true,
|
|
198
|
+
expiring: [],
|
|
199
|
+
});
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const byId = new Map(records.map((r) => [r.id, r]));
|
|
204
|
+
const kept = declared.filter((m) => {
|
|
205
|
+
const rec = byId.get(m);
|
|
206
|
+
return rec !== undefined && !expired(rec, now);
|
|
207
|
+
});
|
|
208
|
+
const dropped = declared.filter((m) => !kept.includes(m));
|
|
209
|
+
|
|
210
|
+
// Rule 2. Discovered ids go after the curated ones, never before.
|
|
211
|
+
const discovered = discover
|
|
212
|
+
? records
|
|
213
|
+
.filter((r) => !declared.includes(r.id) && !expired(r, now) && meets(r, req))
|
|
214
|
+
.map((r) => r.id)
|
|
215
|
+
.sort()
|
|
216
|
+
.slice(0, maxDiscovered)
|
|
217
|
+
: [];
|
|
218
|
+
|
|
219
|
+
const models = [...kept, ...discovered];
|
|
220
|
+
const expiring = models
|
|
221
|
+
.map((m) => ({ model: m, on: byId.get(m)?.expiresOn ?? null }))
|
|
222
|
+
.filter((e): e is { model: string; on: string } => e.on !== null);
|
|
223
|
+
|
|
224
|
+
out.push({
|
|
225
|
+
provider: provider.id,
|
|
226
|
+
models,
|
|
227
|
+
kept,
|
|
228
|
+
dropped,
|
|
229
|
+
discovered,
|
|
230
|
+
unverified: false,
|
|
231
|
+
expiring,
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
return out;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Fold a resolution back into providers, ready for `usableChain`.
|
|
239
|
+
*
|
|
240
|
+
* A provider whose every model resolved away is left with an EMPTY model list
|
|
241
|
+
* rather than being dropped from the chain. `usableChain` already skips links
|
|
242
|
+
* with nothing to try, and keeping the row means the vendor still appears in
|
|
243
|
+
* reports — a silently vanished provider is how a chain quietly becomes a
|
|
244
|
+
* single point of failure without anyone noticing.
|
|
245
|
+
*/
|
|
246
|
+
export function applyResolution(chain: Provider[], resolved: ProviderResolution[]): Provider[] {
|
|
247
|
+
const byId = new Map(resolved.map((r) => [r.provider, r]));
|
|
248
|
+
return chain.map((p) => {
|
|
249
|
+
const r = byId.get(p.id);
|
|
250
|
+
return r ? { ...p, models: r.models } : p;
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** True when any declared id was confirmed gone and routed around. */
|
|
255
|
+
export function routedAroundRot(resolved: ProviderResolution[]): boolean {
|
|
256
|
+
return resolved.some((r) => r.dropped.length > 0);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Providers left with nothing to try, despite a readable catalogue.
|
|
261
|
+
*
|
|
262
|
+
* Kept separate from "could not look" on purpose: a vendor whose catalogue
|
|
263
|
+
* answered and contained none of our models is a real lost link, while an
|
|
264
|
+
* unreadable one is an unknown. Collapsing them produces either a false alarm
|
|
265
|
+
* every time a key expires, or silence when a vendor actually goes away.
|
|
266
|
+
*/
|
|
267
|
+
export function emptyProviders(resolved: ProviderResolution[]): string[] {
|
|
268
|
+
return resolved.filter((r) => !r.unverified && r.models.length === 0).map((r) => r.provider);
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** Human-readable report. Could-not-look stays visibly distinct from a pass. */
|
|
272
|
+
export function resolutionReport(resolved: ProviderResolution[]): string {
|
|
273
|
+
const lines: string[] = [];
|
|
274
|
+
for (const r of resolved) {
|
|
275
|
+
if (r.unverified) {
|
|
276
|
+
lines.push(
|
|
277
|
+
`? ${r.provider}: catalogue unreadable — using the declared ${r.models.length} id(s) UNVERIFIED`,
|
|
278
|
+
);
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
lines.push(
|
|
282
|
+
` ${r.provider}: ${r.models.length} model(s) — ${r.kept.length} declared, ${r.discovered.length} discovered`,
|
|
283
|
+
);
|
|
284
|
+
for (const m of r.dropped) lines.push(` GONE, routed around: ${m}`);
|
|
285
|
+
for (const m of r.discovered) lines.push(` + ${m}`);
|
|
286
|
+
for (const e of r.expiring) lines.push(` ! ${e.model} — vendor says it ends ${e.on}`);
|
|
287
|
+
}
|
|
288
|
+
const empty = emptyProviders(resolved);
|
|
289
|
+
if (empty.length)
|
|
290
|
+
lines.push(
|
|
291
|
+
`\nNothing left to try at: ${empty.join(", ")} — that vendor is gone from the chain.`,
|
|
292
|
+
);
|
|
293
|
+
return lines.join("\n");
|
|
294
|
+
}
|