@extrafemi-ai/lodestone 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/README.md +46 -0
- package/SECURITY.md +35 -0
- package/dist/catalog.d.ts +41 -5
- package/dist/catalog.js +53 -25
- package/dist/discover.d.ts +24 -0
- package/dist/discover.js +20 -0
- package/dist/errors.d.ts +2 -0
- package/dist/explain.js +2 -0
- package/dist/index.d.ts +8 -1
- package/dist/index.js +3 -0
- package/dist/lodestone.d.ts +34 -7
- package/dist/lodestone.js +96 -20
- package/dist/providers/openaiCompatible.d.ts +47 -0
- package/dist/providers/openaiCompatible.js +92 -0
- package/dist/providers/openrouter.js +2 -0
- package/dist/trust.d.ts +43 -0
- package/dist/trust.js +42 -0
- package/dist/types.d.ts +5 -0
- package/examples/multi-provider.ts +26 -0
- package/examples/netlify-daily-check.ts +13 -0
- package/examples/netlify-function.ts +35 -0
- package/examples/stubs.d.ts +7 -0
- package/package.json +16 -7
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
Versions before 1.0 may change between minor releases; each change that could break a caller is marked **breaking**.
|
|
4
|
+
|
|
5
|
+
## 0.5.0
|
|
6
|
+
|
|
7
|
+
Redundancy across providers, and model lists that keep themselves alive and safe.
|
|
8
|
+
|
|
9
|
+
- **Other providers.** `openaiCompatible()` and presets for [Requesty](https://www.requesty.ai) (`requesty()`, with an EU region), [LLM API](https://llmapi.ai) (`llmapi()`), Groq, Cerebras, OpenAI, Mistral, Together and Ollama. Each preset reads that service's model list, so a model it no longer offers is dropped. Use them as `fallbacks` so one provider's outage does not stop the tool.
|
|
10
|
+
- **Discovery that stays short.** A provider's list can be topped up from its own catalogue (`discover: { fill }`) with trusted models within a price limit, and never above `max`. For OpenRouter, a longer pool is kept and the live models are chosen from it, so a model that is down is replaced by the next best while a request still tries at most three.
|
|
11
|
+
- **Trust policy** (`trust`). Discovered models must come from a trusted lab (a default list, yours to change); a block list; refuse models a provider says are trained on, or that keep prompts too long (Requesty publishes both); drop models retiring within a week (OpenRouter publishes the date). **Breaking:** a free model from a lab that is not on the default list is no longer added by itself. Name it in your own list, or set `trust: { publishers: 'any' }`.
|
|
12
|
+
- **Private calls** (`private: true`). OpenRouter is told to use only providers that do not keep or train on the prompt; other models are used only if their provider's list says so.
|
|
13
|
+
- `checkModels()` now checks every provider that can list its models and reports models that are retiring (`expiring`).
|
|
14
|
+
- **Cancelling.** `signal` stops a call, and every model after it, when the caller no longer wants the answer.
|
|
15
|
+
- `peek(cacheKey)` reads a saved answer without calling a model.
|
|
16
|
+
- `free: true` on a fallback: a free tier or local server is not counted against, or stopped by, the spend limit.
|
|
17
|
+
- Fixed: an OpenRouter outage could use the whole time budget on free models before another provider got a turn.
|
|
18
|
+
- Fixed: a model's catalogue is fetched in the background the first time it is needed, not only by the daily check.
|
|
19
|
+
|
|
20
|
+
## 0.4.0
|
|
21
|
+
|
|
22
|
+
- **Circuit breaker** (`createHealth`): a model whose last three attempts failed is skipped by every function until it recovers; one attempt is let through every two minutes to see.
|
|
23
|
+
- **Spend control** (`spend`): asked before the platform's money is used, told what each call cost (OpenRouter reports it; otherwise estimated from catalogue price). A caller's own key never goes through it.
|
|
24
|
+
- `openrouter.paidFirst`: try the paid models first on the platform's key and use the free ones as backup, for when free models are not worth waiting for.
|
|
25
|
+
- `openrouter({ reasoning })`: set how much each model thinks. Measured: turning it off for `claude-haiku-5.5` halved scoring time with the same answers; Gemini refuses to run with it off.
|
|
26
|
+
- New failure kind `budget`.
|
|
27
|
+
|
|
28
|
+
## 0.3.1
|
|
29
|
+
|
|
30
|
+
- A call on the caller's own key no longer reaches the platform's fallback providers unless a fallback sets `allowWithCallerKey`.
|
|
31
|
+
- Attempts that fail with 401, 402 or 403 are not counted against a model in live stats.
|
|
32
|
+
|
|
33
|
+
## 0.3.0
|
|
34
|
+
|
|
35
|
+
- **Scorecards**: a `model-scorecard/1` file of measured results (for example from [Naija Eval](https://naija-eval.extrafemi.com)) can put the best-measured model first, per task. `parseScorecard`, `fetchScorecard`, `rankModels`.
|
|
36
|
+
- `createStats()` publishes live success rate and speed in the same format. `ai.candidates()` lists the models in use for an evaluator. See `docs/scorecards.md`.
|
|
37
|
+
|
|
38
|
+
## 0.2.0
|
|
39
|
+
|
|
40
|
+
- `route`: try exactly these providers, models and keys, in order, for callers that choose their own.
|
|
41
|
+
- Errors carry the provider's own message (`detail.providerMessage`). Temperature is sent only when set.
|
|
42
|
+
|
|
43
|
+
## 0.1.0
|
|
44
|
+
|
|
45
|
+
- First release: `complete()` and `json()` with time-boxed attempts across OpenRouter's free models, a model list that replaces withdrawn models from the live catalogue, four kinds of rate limit told apart, a paid fallback, plain-language `explain()`, bring-your-own-key, and Anthropic, Google and mock adapters.
|
package/README.md
CHANGED
|
@@ -117,6 +117,52 @@ const ai = createLodestone({
|
|
|
117
117
|
|
|
118
118
|
`allow` is asked before each attempt that spends the platform's money (the paid models and fallback providers on the platform's keys), and a `false` means that model is not used. `record` is told what each such call cost: the provider's own figure when it gives one (OpenRouter does), otherwise an estimate from the model's catalogue price. Free models and a caller's own key (their money, not yours) never go through either. When the limit stops every paid option the error is `kind: 'budget'`. A `record` that throws never fails the answer.
|
|
119
119
|
|
|
120
|
+
### Providers, and not depending on one
|
|
121
|
+
|
|
122
|
+
OpenRouter is the default, but one provider's outage should not stop your tool. Give Lodestone others as `fallbacks`, each with its own key. Any service that speaks OpenAI's chat API works through `openaiCompatible()`, and presets exist for the ones Lodestone has been checked against:
|
|
123
|
+
|
|
124
|
+
| Preset | Service | Notes |
|
|
125
|
+
|---|---|---|
|
|
126
|
+
| `requesty()` | [Requesty](https://www.requesty.ai) | Gateway to 800+ models with failover. Publishes price, lab, whether a model is trained on and how long prompts are kept. `requesty({ region: 'eu' })` keeps prompts in the EU. No free models; a 5% markup. |
|
|
127
|
+
| `llmapi()` | [LLM API](https://llmapi.ai) | Gateway to 150+ chat models. Lists price; ignores its image, video and speech models. |
|
|
128
|
+
| `groq()`, `cerebras()` | Groq, Cerebras | Fast inference with free tiers. Use `free: true` so they are not counted against a spend limit. |
|
|
129
|
+
| `openai()`, `mistral()`, `together()` | the providers themselves | |
|
|
130
|
+
| `ollama()` | a local server | Free and private. |
|
|
131
|
+
| `anthropic()`, `google()` | direct | Not OpenAI-style, so they have their own adapters. |
|
|
132
|
+
|
|
133
|
+
```ts
|
|
134
|
+
fallbacks: [
|
|
135
|
+
{ provider: requesty(), apiKey: process.env.REQUESTY_API_KEY!, models: ['bedrock/claude-haiku-5-5'], discover: { fill: { maxPromptUsdPerMTok: 1, maxCompletionUsdPerMTok: 5 }, max: 3 } },
|
|
136
|
+
{ provider: groq(), apiKey: process.env.GROQ_API_KEY!, models: ['llama-3.3-70b-versatile'], free: true },
|
|
137
|
+
]
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Model ids are each service's own. See [examples/multi-provider.ts](examples/multi-provider.ts).
|
|
141
|
+
|
|
142
|
+
### Model lists that keep themselves alive, and short
|
|
143
|
+
|
|
144
|
+
- **Outdated or shut down.** The daily `checkModels()` saves each provider's current catalogue. A model the provider no longer lists is dropped; one with an expiry date within a week (OpenRouter publishes it) is left out; one that is failing right now is skipped by the circuit breaker. Nobody edits a list.
|
|
145
|
+
- **New models.** `discover: { fill }` tops a short list up from the provider's own catalogue with models within a price limit ("free" is a limit of zero) from trusted labs. For OpenRouter's free models this happens by itself.
|
|
146
|
+
- **Not a longer wait.** Discovery never lengthens what a request walks. A background pool (up to 8 per tier) is ranked, the models that are down are filtered out, and a request tries at most three. When one fails, the next best takes its place for later requests, so adding models adds choice, not delay.
|
|
147
|
+
- **Safe by default.** A discovered model is used only if its publisher is on the trust list; `trust` also takes a block list, refuses models a provider says are trained on or that keep prompts too long, and `private: true` applies the same to one call. There is no vulnerability database for hosted models, so none is claimed: see [SECURITY.md](SECURITY.md) for exactly what is and is not checked.
|
|
148
|
+
|
|
149
|
+
### Cancelling
|
|
150
|
+
|
|
151
|
+
```ts
|
|
152
|
+
const controller = new AbortController();
|
|
153
|
+
req.on('close', () => controller.abort()); // the person left the page
|
|
154
|
+
await ai.complete({ prompt, signal: controller.signal }); // throws a LodestoneError of kind 'aborted'
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
### Where it runs
|
|
158
|
+
|
|
159
|
+
Lodestone uses only `fetch`, `AbortSignal` and `URL`, with no Node-specific APIs, so it should run on Cloudflare Workers, Deno and edge runtimes too. **It is tested on Node 20 and 22 only.** Reports from other runtimes are welcome.
|
|
160
|
+
|
|
161
|
+
### Not yet
|
|
162
|
+
|
|
163
|
+
- **Streaming.** `complete()` and `json()` wait for the whole answer. Streaming needs a decision about a model that fails halfway; the likely rule is to fail over only before the first token.
|
|
164
|
+
- **Tool calling** (functions the model can call), and **embeddings or speech**. Lodestone is for one-shot text and JSON calls today.
|
|
165
|
+
|
|
120
166
|
### Letting measurements decide the order
|
|
121
167
|
|
|
122
168
|
A scorecard (a small JSON file of measured results) can put the models that did best on your kind of task first, and Lodestone can publish its own live success rates and speeds in the same format. [Naija Eval](https://naija-eval.extrafemi.com) publishes translation and speech quality for Hausa, Igbo and Yoruba this way. It is opt-in, per task, and only changes the order models are tried in. See [docs/scorecards.md](docs/scorecards.md).
|
package/SECURITY.md
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Security
|
|
2
|
+
|
|
3
|
+
## Reporting a problem
|
|
4
|
+
|
|
5
|
+
Please report a vulnerability privately, not in a public issue: use GitHub's **Report a vulnerability** button on the
|
|
6
|
+
Security tab of this repository. Say what you found, how to reproduce it, and which version. You will get a reply
|
|
7
|
+
within a few days. Please give a reasonable time to fix it before you publish details.
|
|
8
|
+
|
|
9
|
+
## What this package is, for supply-chain purposes
|
|
10
|
+
|
|
11
|
+
- **No runtime dependencies.** Lodestone uses only `fetch`, `AbortSignal` and `URL`. Installing it brings in nothing else,
|
|
12
|
+
so there is nothing underneath it to be taken over.
|
|
13
|
+
- **Published with provenance.** Each release is built by a GitHub Actions workflow from a tagged commit and published to
|
|
14
|
+
npm with a signed attestation. `npm audit signatures` verifies that a package came from this repository.
|
|
15
|
+
- **Small.** About 1,600 lines of source, with tests beside it. Read it before you depend on it.
|
|
16
|
+
- **Your keys stay yours.** Keys are passed in by your code and sent only to the provider you name. They are never stored,
|
|
17
|
+
logged or sent anywhere else, and prompts and answers are never logged. `onAttempt` reports the provider, model, outcome
|
|
18
|
+
and timing only.
|
|
19
|
+
|
|
20
|
+
## What Lodestone checks about models, and what it does not
|
|
21
|
+
|
|
22
|
+
There is no vulnerability database for hosted AI models, so Lodestone does not claim to scan for one. It checks what can
|
|
23
|
+
really be checked, and says where it cannot:
|
|
24
|
+
|
|
25
|
+
- **Who made it.** A model found by discovery is used only if its publisher is on a list of well-known labs (yours to
|
|
26
|
+
change). A model you name yourself is your decision.
|
|
27
|
+
- **What happens to the prompt.** Where a provider publishes it (Requesty does, for every model), a model that is trained on,
|
|
28
|
+
or that keeps prompts longer than you allow, is refused. For OpenRouter, a call marked `private` asks for providers that do
|
|
29
|
+
not keep or train on prompts. For other providers Lodestone cannot enforce this: it relies on the model's list entry, and
|
|
30
|
+
refuses a model whose list says nothing.
|
|
31
|
+
- **That it is not about to disappear.** A model with an expiry date within a week is left out.
|
|
32
|
+
- **That the answer is the shape you expect.** `json()` skips a model whose answer is not valid JSON or fails your check,
|
|
33
|
+
which also limits what a misbehaving model can hand to the rest of your code. Treat every model answer as untrusted input.
|
|
34
|
+
|
|
35
|
+
These are self-reported by providers and are only as good as what they report.
|
package/dist/catalog.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type TrustPolicy } from './trust.js';
|
|
1
2
|
import type { KeyValueStore, Tier } from './types.js';
|
|
2
3
|
/** Works out which models to ask for from OpenRouter's own catalogue, so a model
|
|
3
4
|
* that has been withdrawn is dropped and a current one takes its place without
|
|
@@ -14,17 +15,52 @@ export interface CatalogModel {
|
|
|
14
15
|
prompt?: string | number;
|
|
15
16
|
completion?: string | number;
|
|
16
17
|
};
|
|
18
|
+
/** The lab that made it, when the provider says (Requesty's model_lab); otherwise the first part of the id. */
|
|
19
|
+
publisher?: string;
|
|
20
|
+
/** The day the provider stops offering it (OpenRouter publishes this for models being retired). */
|
|
21
|
+
expiration_date?: string;
|
|
22
|
+
/** What the provider says happens to prompts (Requesty publishes these for every model). */
|
|
23
|
+
data_used_for_training?: boolean;
|
|
24
|
+
data_retention_days?: number;
|
|
25
|
+
/** Zero price, as the provider says it. */
|
|
26
|
+
free?: boolean;
|
|
17
27
|
}
|
|
18
28
|
export declare const ROUTER = "openrouter/free";
|
|
19
|
-
|
|
29
|
+
/** US dollars per million tokens, or undefined if the provider gives no price. */
|
|
30
|
+
export declare const promptPerMTok: (m: CatalogModel) => number | undefined;
|
|
31
|
+
export declare const completionPerMTok: (m: CatalogModel) => number | undefined;
|
|
32
|
+
/** Text models fit for the tier. Free ones, unless `price` allows more (then at most that, per million tokens). */
|
|
33
|
+
export declare function eligible(catalog: CatalogModel[], tier: Tier, price?: {
|
|
34
|
+
promptUsdPerMTok: number;
|
|
35
|
+
completionUsdPerMTok: number;
|
|
36
|
+
}): CatalogModel[];
|
|
37
|
+
/** How the candidates for a tier are ranked: bigger is stronger, lighter is faster, then longer context. */
|
|
38
|
+
export declare function rank(tier: Tier, a: CatalogModel, b: CatalogModel): number;
|
|
39
|
+
export interface PickOptions {
|
|
40
|
+
policy?: TrustPolicy;
|
|
41
|
+
/** Treat unknown privacy facts as a refusal even for a preferred model (a call marked private). */
|
|
42
|
+
strict?: boolean;
|
|
43
|
+
/** How many to return, the router included. Default 3. A larger number gives a pool to choose live models from. */
|
|
44
|
+
depth?: number;
|
|
45
|
+
now?: number;
|
|
46
|
+
}
|
|
20
47
|
/** The ids to send, best first: preferred models that still exist, then the best
|
|
21
|
-
* others by tier, then the router; at most three. With no catalogue at all, the
|
|
22
|
-
* preferences are returned as they are.
|
|
23
|
-
|
|
48
|
+
* others by tier, then the router; at most `depth` (three by default). With no catalogue at all, the
|
|
49
|
+
* preferences are returned as they are. A model the trust policy refuses, or that is about to be
|
|
50
|
+
* retired, is left out. */
|
|
51
|
+
export declare function pickModels(tier: Tier, catalog: CatalogModel[] | null, preferences: string[], opts?: PickOptions): string[];
|
|
24
52
|
export interface Catalog {
|
|
25
53
|
/** Fetches OpenRouter's catalogue and saves it. Returns null if it can't be read. */
|
|
26
54
|
refresh(): Promise<CatalogModel[] | null>;
|
|
27
55
|
/** The saved catalogue (six-hour copy in memory; refreshed in the background if over a day old). Never waits on the network unless nothing is saved. */
|
|
28
56
|
load(): Promise<CatalogModel[] | null>;
|
|
29
57
|
}
|
|
30
|
-
|
|
58
|
+
/** OpenRouter's catalogue, as a list of CatalogModel. */
|
|
59
|
+
export declare function listOpenRouter(fetchImpl: typeof fetch): Promise<CatalogModel[] | null>;
|
|
60
|
+
export interface CatalogSource {
|
|
61
|
+
/** Where it is saved. Default: the OpenRouter catalogue's own key. */
|
|
62
|
+
key?: string;
|
|
63
|
+
/** How to read the provider's list. Default: OpenRouter's. */
|
|
64
|
+
list?: () => Promise<CatalogModel[] | null>;
|
|
65
|
+
}
|
|
66
|
+
export declare function createCatalog(store: KeyValueStore, fetchImpl: typeof fetch, now?: () => number, source?: CatalogSource): Catalog;
|
package/dist/catalog.js
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
|
+
import { vet } from './trust.js';
|
|
1
2
|
export const ROUTER = 'openrouter/free';
|
|
2
3
|
const MAX_ENTRIES = 3;
|
|
3
4
|
const isFree = (m) => m.id.endsWith(':free') || (Number(m.pricing?.prompt) === 0 && Number(m.pricing?.completion) === 0 && m.id !== ROUTER);
|
|
5
|
+
/** US dollars per million tokens, or undefined if the provider gives no price. */
|
|
6
|
+
export const promptPerMTok = (m) => { const n = Number(m.pricing?.prompt); return m.pricing?.prompt !== undefined && m.pricing?.prompt !== '' && Number.isFinite(n) ? n * 1e6 : undefined; };
|
|
7
|
+
export const completionPerMTok = (m) => { const n = Number(m.pricing?.completion); return m.pricing?.completion !== undefined && m.pricing?.completion !== '' && Number.isFinite(n) ? n * 1e6 : undefined; };
|
|
4
8
|
const textOut = (m) => (m.architecture?.output_modalities ?? ['text']).join() === 'text';
|
|
5
9
|
const takesImages = (m) => (m.architecture?.input_modalities ?? []).includes('image');
|
|
6
10
|
// Safety classifiers, audio and music models, code-only models and embedders are not for writing or reading text.
|
|
@@ -8,45 +12,64 @@ const EXCLUDE = /safety|guard|moderation|lyria|embed|rerank|tts|whisper|code|cod
|
|
|
8
12
|
/** "550b" or "26b" in an id: bigger is the stronger pick. */
|
|
9
13
|
const size = (id) => Number(id.match(/(\d{2,4})b\b/i)?.[1] ?? 0);
|
|
10
14
|
const LIGHT = /lightning|flash|mini|small|nano|lite|fast/i;
|
|
11
|
-
|
|
12
|
-
|
|
15
|
+
/** Text models fit for the tier. Free ones, unless `price` allows more (then at most that, per million tokens). */
|
|
16
|
+
export function eligible(catalog, tier, price) {
|
|
17
|
+
return catalog.filter((m) => {
|
|
18
|
+
const affordable = price
|
|
19
|
+
? (promptPerMTok(m) ?? Infinity) <= price.promptUsdPerMTok && (completionPerMTok(m) ?? Infinity) <= price.completionUsdPerMTok
|
|
20
|
+
: isFree(m);
|
|
21
|
+
return affordable && textOut(m) && (m.context_length ?? 0) >= 32_000 && !EXCLUDE.test(m.id) && (tier !== 'vision' || takesImages(m));
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
/** How the candidates for a tier are ranked: bigger is stronger, lighter is faster, then longer context. */
|
|
25
|
+
export function rank(tier, a, b) {
|
|
26
|
+
return (tier === 'strong' ? size(b.id) - size(a.id) : tier === 'fast' ? Number(LIGHT.test(b.id)) - Number(LIGHT.test(a.id)) || size(a.id) - size(b.id) : size(b.id) - size(a.id)) || (b.context_length ?? 0) - (a.context_length ?? 0);
|
|
13
27
|
}
|
|
14
28
|
/** The ids to send, best first: preferred models that still exist, then the best
|
|
15
|
-
* others by tier, then the router; at most three. With no catalogue at all, the
|
|
16
|
-
* preferences are returned as they are.
|
|
17
|
-
|
|
29
|
+
* others by tier, then the router; at most `depth` (three by default). With no catalogue at all, the
|
|
30
|
+
* preferences are returned as they are. A model the trust policy refuses, or that is about to be
|
|
31
|
+
* retired, is left out. */
|
|
32
|
+
export function pickModels(tier, catalog, preferences, opts = {}) {
|
|
33
|
+
const depth = opts.depth ?? MAX_ENTRIES;
|
|
18
34
|
if (!catalog || catalog.length === 0)
|
|
19
|
-
return preferences.slice(0,
|
|
35
|
+
return preferences.slice(0, depth);
|
|
36
|
+
const now = opts.now ?? Date.now();
|
|
20
37
|
const ok = new Map(eligible(catalog, tier).map((m) => [m.id, m]));
|
|
21
|
-
const out = preferences.filter((id) => id !== ROUTER && ok.has(id));
|
|
22
|
-
const rest = [...ok.values()]
|
|
23
|
-
.filter((m) => !out.includes(m.id))
|
|
24
|
-
.sort((a, b) => (tier === 'strong' ? size(b.id) - size(a.id) : tier === 'fast' ? Number(LIGHT.test(b.id)) - Number(LIGHT.test(a.id)) || size(a.id) - size(b.id) : size(b.id) - size(a.id)) || (b.context_length ?? 0) - (a.context_length ?? 0));
|
|
38
|
+
const out = preferences.filter((id) => id !== ROUTER && ok.has(id) && vet(ok.get(id), opts.policy, { discovered: false, strict: opts.strict, now }).ok);
|
|
39
|
+
const rest = [...ok.values()].filter((m) => !out.includes(m.id) && vet(m, opts.policy, { discovered: true, strict: opts.strict, now }).ok).sort((a, b) => rank(tier, a, b));
|
|
25
40
|
for (const m of rest) {
|
|
26
|
-
if (out.length >=
|
|
41
|
+
if (out.length >= depth - 1)
|
|
27
42
|
break;
|
|
28
43
|
out.push(m.id);
|
|
29
44
|
}
|
|
30
|
-
return [...out.slice(0,
|
|
45
|
+
return [...out.slice(0, depth - 1), ROUTER];
|
|
31
46
|
}
|
|
32
47
|
const KEY = 'lodestone:model-catalog';
|
|
33
48
|
const MEMORY_MS = 6 * 3_600_000;
|
|
34
49
|
const STALE_MS = 24 * 3_600_000;
|
|
35
|
-
|
|
50
|
+
/** OpenRouter's catalogue, as a list of CatalogModel. */
|
|
51
|
+
export async function listOpenRouter(fetchImpl) {
|
|
52
|
+
const res = await fetchImpl('https://openrouter.ai/api/v1/models', { signal: AbortSignal.timeout(8_000) });
|
|
53
|
+
if (!res.ok)
|
|
54
|
+
return null;
|
|
55
|
+
const models = ((await res.json()).data ?? []).map((m) => ({
|
|
56
|
+
id: m.id,
|
|
57
|
+
context_length: m.context_length,
|
|
58
|
+
architecture: m.architecture && { input_modalities: m.architecture.input_modalities, output_modalities: m.architecture.output_modalities },
|
|
59
|
+
pricing: m.pricing && { prompt: m.pricing.prompt, completion: m.pricing.completion },
|
|
60
|
+
...(m.expiration_date ? { expiration_date: m.expiration_date } : {}),
|
|
61
|
+
}));
|
|
62
|
+
return models.length ? models : null;
|
|
63
|
+
}
|
|
64
|
+
export function createCatalog(store, fetchImpl, now = Date.now, source = {}) {
|
|
36
65
|
let memory = null;
|
|
66
|
+
let lastKick = -Infinity;
|
|
67
|
+
const KEY_ = source.key ?? KEY;
|
|
37
68
|
const refresh = async () => {
|
|
38
|
-
const
|
|
39
|
-
if (!
|
|
69
|
+
const models = await (source.list ? source.list() : listOpenRouter(fetchImpl));
|
|
70
|
+
if (!models || models.length === 0)
|
|
40
71
|
return null;
|
|
41
|
-
|
|
42
|
-
id: m.id,
|
|
43
|
-
context_length: m.context_length,
|
|
44
|
-
architecture: m.architecture && { input_modalities: m.architecture.input_modalities, output_modalities: m.architecture.output_modalities },
|
|
45
|
-
pricing: m.pricing && { prompt: m.pricing.prompt, completion: m.pricing.completion },
|
|
46
|
-
}));
|
|
47
|
-
if (models.length === 0)
|
|
48
|
-
return null;
|
|
49
|
-
await store.set(KEY, { at: new Date(now()).toISOString(), models }).catch(() => { });
|
|
72
|
+
await store.set(KEY_, { at: new Date(now()).toISOString(), models }).catch(() => { });
|
|
50
73
|
memory = { at: now(), models };
|
|
51
74
|
return models;
|
|
52
75
|
};
|
|
@@ -54,13 +77,18 @@ export function createCatalog(store, fetchImpl, now = Date.now) {
|
|
|
54
77
|
if (memory && now() - memory.at < MEMORY_MS)
|
|
55
78
|
return memory.models;
|
|
56
79
|
try {
|
|
57
|
-
const saved = (await store.get(
|
|
80
|
+
const saved = (await store.get(KEY_));
|
|
58
81
|
if (saved?.models?.length) {
|
|
59
82
|
memory = { at: now(), models: saved.models };
|
|
60
83
|
if (now() - Date.parse(saved.at) > STALE_MS)
|
|
61
84
|
refresh().catch(() => { });
|
|
62
85
|
return saved.models;
|
|
63
86
|
}
|
|
87
|
+
// Nothing saved yet: use the preferences this time, and fetch it for the next request.
|
|
88
|
+
if (now() - lastKick > 5 * 60_000) {
|
|
89
|
+
lastKick = now();
|
|
90
|
+
refresh().catch(() => { });
|
|
91
|
+
}
|
|
64
92
|
}
|
|
65
93
|
catch { /* an unreadable store is the same as an empty one */ }
|
|
66
94
|
return null;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { type CatalogModel } from './catalog.js';
|
|
2
|
+
import { type TrustPolicy } from './trust.js';
|
|
3
|
+
import type { Tier } from './types.js';
|
|
4
|
+
/** Keeps a provider's list of models alive without anyone editing it: a model the provider no
|
|
5
|
+
* longer lists is dropped, one that is retiring soon or that the trust policy refuses is dropped,
|
|
6
|
+
* and if the list is then short, it is filled from the provider's own catalogue with models that are
|
|
7
|
+
* within a price limit and made by a trusted publisher. Filling only ever adds models after the
|
|
8
|
+
* app's own, and never above `max`, so the list a request walks stays short however many models
|
|
9
|
+
* a provider offers. */
|
|
10
|
+
export interface FillOptions {
|
|
11
|
+
/** The most a filled-in model may cost, in US dollars per million tokens. { prompt: 0, completion: 0 } is "free". */
|
|
12
|
+
maxPromptUsdPerMTok: number;
|
|
13
|
+
maxCompletionUsdPerMTok: number;
|
|
14
|
+
}
|
|
15
|
+
export interface ResolveOptions {
|
|
16
|
+
tier: Tier;
|
|
17
|
+
policy?: TrustPolicy;
|
|
18
|
+
strict?: boolean;
|
|
19
|
+
fill?: FillOptions;
|
|
20
|
+
/** The most models to return. Default 3. */
|
|
21
|
+
max?: number;
|
|
22
|
+
now: number;
|
|
23
|
+
}
|
|
24
|
+
export declare function resolveProviderModels(curated: string[], catalog: CatalogModel[] | null, o: ResolveOptions): string[];
|
package/dist/discover.js
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { eligible, rank } from './catalog.js';
|
|
2
|
+
import { vet } from './trust.js';
|
|
3
|
+
export function resolveProviderModels(curated, catalog, o) {
|
|
4
|
+
if (!catalog || catalog.length === 0)
|
|
5
|
+
return curated; // nothing known about what exists: use what the app named
|
|
6
|
+
const max = o.max ?? 3;
|
|
7
|
+
const byId = new Map(catalog.map((m) => [m.id, m]));
|
|
8
|
+
const kept = curated.filter((id) => { const m = byId.get(id); return m !== undefined && vet(m, o.policy, { discovered: false, strict: o.strict, now: o.now }).ok; });
|
|
9
|
+
if (!o.fill || kept.length >= max)
|
|
10
|
+
return kept;
|
|
11
|
+
const fill = eligible(catalog, o.tier, { promptUsdPerMTok: o.fill.maxPromptUsdPerMTok, completionUsdPerMTok: o.fill.maxCompletionUsdPerMTok })
|
|
12
|
+
.filter((m) => !kept.includes(m.id) && vet(m, o.policy, { discovered: true, strict: o.strict, now: o.now }).ok)
|
|
13
|
+
.sort((a, b) => rank(o.tier, a, b));
|
|
14
|
+
for (const m of fill) {
|
|
15
|
+
if (kept.length >= max)
|
|
16
|
+
break;
|
|
17
|
+
kept.push(m.id);
|
|
18
|
+
}
|
|
19
|
+
return kept;
|
|
20
|
+
}
|
package/dist/errors.d.ts
CHANGED
|
@@ -22,6 +22,8 @@ export type FailureKind =
|
|
|
22
22
|
| 'unavailable'
|
|
23
23
|
/** A model answered but the answer was not usable (empty, not valid JSON, failed validation). */
|
|
24
24
|
| 'bad_output'
|
|
25
|
+
/** The caller stopped the call (it passed a signal that was aborted). */
|
|
26
|
+
| 'aborted'
|
|
25
27
|
/** The platform's spend limit has been reached, so no paid model was used. */
|
|
26
28
|
| 'budget'
|
|
27
29
|
/** No provider has a key. */
|
package/dist/explain.js
CHANGED
|
@@ -42,6 +42,8 @@ export function explain(err, opts = {}) {
|
|
|
42
42
|
return "The AI gave back something that couldn't be read. Try again; it usually works on a second attempt.";
|
|
43
43
|
case 'credit':
|
|
44
44
|
return 'The AI account has no credit for the backup model. Add a little credit and try again.';
|
|
45
|
+
case 'aborted':
|
|
46
|
+
return 'Cancelled.';
|
|
45
47
|
case 'budget':
|
|
46
48
|
return `${product}'s AI budget for now is used up, and the free models are busy. Try again later${opts.settingsHint ? `, or add your own OpenRouter key ${opts.settingsHint.replace(/^in /, 'in ')} to keep going` : ''}.`;
|
|
47
49
|
case 'not_configured':
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { createLodestone } from './lodestone.js';
|
|
2
|
-
export type { Lodestone, LodestoneConfig, CompleteOptions, CompleteResult, Profile, FallbackConfig, AttemptEvent, RouteStep, SpendControl } from './lodestone.js';
|
|
2
|
+
export type { Lodestone, LodestoneConfig, CompleteOptions, CompleteResult, Profile, FallbackConfig, AttemptEvent, RouteStep, SpendControl, CheckResult } from './lodestone.js';
|
|
3
3
|
export { LodestoneError, ProviderError, messageFrom } from './errors.js';
|
|
4
4
|
export type { FailureKind } from './errors.js';
|
|
5
5
|
export { explain } from './explain.js';
|
|
@@ -24,3 +24,10 @@ export { CANDIDATES_SCHEMA } from './candidates.js';
|
|
|
24
24
|
export type { Candidates, Candidate } from './candidates.js';
|
|
25
25
|
export { createHealth } from './health.js';
|
|
26
26
|
export type { Health, HealthOptions, ModelHealth } from './health.js';
|
|
27
|
+
export { openaiCompatible, requesty, llmapi, groq, cerebras, openai, mistral, together, ollama, normalizeRequesty, normalizeLlmapi } from './providers/openaiCompatible.js';
|
|
28
|
+
export type { OpenAICompatibleOptions, PresetOptions } from './providers/openaiCompatible.js';
|
|
29
|
+
export { vet, publisherOf, DEFAULT_PUBLISHERS } from './trust.js';
|
|
30
|
+
export type { TrustPolicy, Verdict } from './trust.js';
|
|
31
|
+
export { resolveProviderModels } from './discover.js';
|
|
32
|
+
export type { FillOptions } from './discover.js';
|
|
33
|
+
export type { CatalogModel as ModelInfo } from './catalog.js';
|
package/dist/index.js
CHANGED
|
@@ -14,3 +14,6 @@ export { SCORECARD_SCHEMA, ScorecardError, parseScorecard, fetchScorecard, rankM
|
|
|
14
14
|
export { createStats } from './stats.js';
|
|
15
15
|
export { CANDIDATES_SCHEMA } from './candidates.js';
|
|
16
16
|
export { createHealth } from './health.js';
|
|
17
|
+
export { openaiCompatible, requesty, llmapi, groq, cerebras, openai, mistral, together, ollama, normalizeRequesty, normalizeLlmapi } from './providers/openaiCompatible.js';
|
|
18
|
+
export { vet, publisherOf, DEFAULT_PUBLISHERS } from './trust.js';
|
|
19
|
+
export { resolveProviderModels } from './discover.js';
|
package/dist/lodestone.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { type CatalogModel } from './catalog.js';
|
|
2
|
+
import { type FillOptions } from './discover.js';
|
|
3
|
+
import type { TrustPolicy } from './trust.js';
|
|
2
4
|
import { type Candidates } from './candidates.js';
|
|
3
5
|
import type { Health } from './health.js';
|
|
4
6
|
import { type QualityQuery, type Scorecard } from './scorecard.js';
|
|
@@ -16,6 +18,13 @@ export interface FallbackConfig {
|
|
|
16
18
|
apiKey: string;
|
|
17
19
|
/** Tried in order. */
|
|
18
20
|
models: string[];
|
|
21
|
+
/** This provider costs nothing (a free tier, a local server): its calls are not counted against, or stopped by, the spend limit. */
|
|
22
|
+
free?: boolean;
|
|
23
|
+
/** Keep this provider's list alive from its own catalogue (needs a provider that can list its models): models it no longer offers, or that are retiring or refused by `trust`, are dropped, and with `fill` a short list is topped up with trusted models within a price limit, never above `max` (default 3). */
|
|
24
|
+
discover?: {
|
|
25
|
+
fill?: FillOptions;
|
|
26
|
+
max?: number;
|
|
27
|
+
};
|
|
19
28
|
/** Also use this provider when the caller supplied their own OpenRouter key. Default false: a fallback has the platform's key, and a caller's own key must never lead to the platform's money being spent. */
|
|
20
29
|
allowWithCallerKey?: boolean;
|
|
21
30
|
}
|
|
@@ -68,6 +77,8 @@ export interface LodestoneConfig {
|
|
|
68
77
|
* nobody has measured keep their place. A function is called per request, so it can fetch
|
|
69
78
|
* and cache; if it fails or takes over 1.5 s the order is left alone. */
|
|
70
79
|
scorecards?: Scorecard[] | (() => Promise<Scorecard[]>);
|
|
80
|
+
/** What may be used, and what discovery may add by itself: trusted publishers, a block list, what providers say they do with prompts, retiring models. See trust.ts. */
|
|
81
|
+
trust?: TrustPolicy;
|
|
71
82
|
/** Skips models that have been failing, so a request does not spend its time on them (see health.ts). */
|
|
72
83
|
health?: Health;
|
|
73
84
|
/** Counts and limits the platform's spend. Without it nothing is counted or limited. */
|
|
@@ -120,6 +131,10 @@ export interface CompleteOptions {
|
|
|
120
131
|
apiKey?: string;
|
|
121
132
|
/** An answer that fails this check counts as a failed attempt, and the next model is tried. */
|
|
122
133
|
accept?: (text: string) => boolean;
|
|
134
|
+
/** The prompt must not be kept or trained on (a CV, say). OpenRouter is told to use only providers that do not; other models are used only if their provider's list says so. Many free models cannot meet this, and the paid ones are used instead. */
|
|
135
|
+
private?: boolean;
|
|
136
|
+
/** Stops the call when the caller no longer wants the answer (the person left the page). */
|
|
137
|
+
signal?: AbortSignal;
|
|
123
138
|
/** Same key, same answer: the answer is saved and reused. Only for pure, repeatable tasks. */
|
|
124
139
|
cacheKey?: string;
|
|
125
140
|
cacheTtlMs?: number;
|
|
@@ -131,6 +146,23 @@ export interface CompleteResult {
|
|
|
131
146
|
usage?: Usage;
|
|
132
147
|
cached?: boolean;
|
|
133
148
|
}
|
|
149
|
+
export interface CheckResult {
|
|
150
|
+
ok: boolean;
|
|
151
|
+
error?: string;
|
|
152
|
+
catalogSize?: number;
|
|
153
|
+
/** Preferred models the catalogue no longer lists (already replaced). */
|
|
154
|
+
withdrawn?: string[];
|
|
155
|
+
/** Models that stop being offered within a week (already left out). */
|
|
156
|
+
expiring?: string[];
|
|
157
|
+
shortTiers?: Tier[];
|
|
158
|
+
/** The same check for each other provider that can list its models. */
|
|
159
|
+
providers?: Record<string, {
|
|
160
|
+
models: number;
|
|
161
|
+
withdrawn: string[];
|
|
162
|
+
expiring: string[];
|
|
163
|
+
error?: string;
|
|
164
|
+
}>;
|
|
165
|
+
}
|
|
134
166
|
export declare function createLodestone(config?: LodestoneConfig): {
|
|
135
167
|
complete: (opts: CompleteOptions) => Promise<CompleteResult>;
|
|
136
168
|
json: <T = unknown>(opts: CompleteOptions & {
|
|
@@ -140,13 +172,8 @@ export declare function createLodestone(config?: LodestoneConfig): {
|
|
|
140
172
|
}>;
|
|
141
173
|
resolveModels: (tier: Tier) => Promise<string[]>;
|
|
142
174
|
refreshCatalog: () => Promise<CatalogModel[] | null>;
|
|
143
|
-
checkModels: () => Promise<
|
|
144
|
-
ok: boolean;
|
|
145
|
-
error?: string;
|
|
146
|
-
catalogSize?: number;
|
|
147
|
-
withdrawn?: string[];
|
|
148
|
-
shortTiers?: Tier[];
|
|
149
|
-
}>;
|
|
175
|
+
checkModels: () => Promise<CheckResult>;
|
|
150
176
|
candidates: () => Promise<Candidates>;
|
|
177
|
+
peek: (cacheKey: string, ttlMs?: number) => Promise<CompleteResult | null>;
|
|
151
178
|
};
|
|
152
179
|
export type Lodestone = ReturnType<typeof createLodestone>;
|
package/dist/lodestone.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createCatalog, pickModels, ROUTER } from './catalog.js';
|
|
2
|
+
import { resolveProviderModels } from './discover.js';
|
|
2
3
|
import { CANDIDATES_SCHEMA } from './candidates.js';
|
|
3
4
|
import { LodestoneError, messageFrom, ProviderError } from './errors.js';
|
|
4
5
|
import { tryParseJson } from './json.js';
|
|
@@ -10,6 +11,9 @@ import { openrouter } from './providers/openrouter.js';
|
|
|
10
11
|
import { rankModels } from './scorecard.js';
|
|
11
12
|
import { memoryStore } from './store.js';
|
|
12
13
|
const hasImage = (messages) => messages.some((m) => Array.isArray(m.content) && m.content.some((p) => p.type === 'image'));
|
|
14
|
+
/** How many candidates are kept in the background for a tier. A request still tries at most three. */
|
|
15
|
+
const POOL = 8;
|
|
16
|
+
const ACTIVE = 3;
|
|
13
17
|
export function createLodestone(config = {}) {
|
|
14
18
|
const budget = config.budgetMs ?? 24_000;
|
|
15
19
|
const t = {
|
|
@@ -23,6 +27,13 @@ export function createLodestone(config = {}) {
|
|
|
23
27
|
const store = config.store ?? memoryStore();
|
|
24
28
|
const fetchImpl = config.fetch ?? fetch;
|
|
25
29
|
const catalog = createCatalog(store, fetchImpl, now);
|
|
30
|
+
const policy = config.trust;
|
|
31
|
+
/** One catalogue per fallback provider that can list its models, kept in the same store. */
|
|
32
|
+
const fbCatalogs = new Map();
|
|
33
|
+
for (const fb of config.fallbacks ?? []) {
|
|
34
|
+
if (fb.provider.listModels)
|
|
35
|
+
fbCatalogs.set(fb, createCatalog(store, fetchImpl, now, { key: `lodestone:catalog:${fb.provider.name}`, list: () => fb.provider.listModels(fb.apiKey) }));
|
|
36
|
+
}
|
|
26
37
|
const router = config.openrouter?.provider ?? openrouter({ fetch: fetchImpl });
|
|
27
38
|
const prefs = { ...DEFAULT_MODELS, ...config.openrouter?.models };
|
|
28
39
|
const paidModels = config.openrouter?.paidModels ?? DEFAULT_PAID_MODELS;
|
|
@@ -31,12 +42,33 @@ export function createLodestone(config = {}) {
|
|
|
31
42
|
/** The ids to send for a tier: preferences that still exist, plus current replacements. Never throws, never waits on the network. */
|
|
32
43
|
async function resolveModels(tier) {
|
|
33
44
|
try {
|
|
34
|
-
return pickModels(tier, await catalog.load(), prefs[tier]);
|
|
45
|
+
return pickModels(tier, await catalog.load(), prefs[tier], { policy, now: now() });
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
return prefs[tier];
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/** A longer, ranked list of live candidates for a tier, to choose from when some are down: the best models for the tier that exist, are allowed, and are not retiring. */
|
|
52
|
+
async function resolvePool(tier, strict) {
|
|
53
|
+
try {
|
|
54
|
+
return pickModels(tier, await catalog.load(), prefs[tier], { policy, strict, depth: POOL, now: now() });
|
|
35
55
|
}
|
|
36
56
|
catch {
|
|
37
57
|
return prefs[tier];
|
|
38
58
|
}
|
|
39
59
|
}
|
|
60
|
+
/** A fallback provider's models for a tier: the app's own, pruned and topped up from the provider's catalogue. */
|
|
61
|
+
async function resolveFallback(fb, tier, strict) {
|
|
62
|
+
const cat = fbCatalogs.get(fb);
|
|
63
|
+
if (!cat)
|
|
64
|
+
return fb.models;
|
|
65
|
+
try {
|
|
66
|
+
return resolveProviderModels(fb.models, await cat.load(), { tier, policy, strict, fill: fb.discover?.fill, max: fb.discover?.max, now: now() });
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
return fb.models;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
40
72
|
/** The scorecards, or none: never throws and never holds a request up for long. */
|
|
41
73
|
async function cards() {
|
|
42
74
|
const src = config.scorecards;
|
|
@@ -71,6 +103,9 @@ export function createLodestone(config = {}) {
|
|
|
71
103
|
const tier = opts.tier ?? profile?.tier ?? (images ? 'vision' : 'fast');
|
|
72
104
|
const maxTokens = opts.maxTokens ?? profile?.maxTokens ?? 3500;
|
|
73
105
|
const temperature = opts.temperature ?? profile?.temperature ?? (opts.route ? undefined : 0.2);
|
|
106
|
+
const strict = Boolean(opts.private);
|
|
107
|
+
if (opts.signal?.aborted)
|
|
108
|
+
throw new LodestoneError('aborted');
|
|
74
109
|
const quality = opts.quality ?? profile?.quality;
|
|
75
110
|
const order = async (models) => (quality && models.length > 1 ? rankModels(models, await cards(), quality, { now }) : models);
|
|
76
111
|
const cacheSlot = opts.cacheKey ? `lodestone:cache:${opts.cacheKey}` : null;
|
|
@@ -112,8 +147,8 @@ export function createLodestone(config = {}) {
|
|
|
112
147
|
return keep;
|
|
113
148
|
};
|
|
114
149
|
/** May the platform's money be spent on this attempt? A caller's own key is their money, not ours. */
|
|
115
|
-
const mayUse = async (provider, model, phase) => {
|
|
116
|
-
if (callerKey || !config.spend)
|
|
150
|
+
const mayUse = async (provider, model, phase, free = false) => {
|
|
151
|
+
if (callerKey || free || !config.spend)
|
|
117
152
|
return true;
|
|
118
153
|
if (await config.spend.allow({ provider: provider.name, model, phase }))
|
|
119
154
|
return true;
|
|
@@ -126,7 +161,9 @@ export function createLodestone(config = {}) {
|
|
|
126
161
|
const at = now();
|
|
127
162
|
let out;
|
|
128
163
|
try {
|
|
129
|
-
const
|
|
164
|
+
const timeout = AbortSignal.timeout(ms);
|
|
165
|
+
const signal = opts.signal ? (typeof AbortSignal.any === 'function' ? AbortSignal.any([timeout, opts.signal]) : timeout) : timeout;
|
|
166
|
+
const reply = await provider.call({ model, system: opts.system, messages, maxTokens, temperature, apiKey, signal, ...(strict ? { privacy: 'strict' } : {}) });
|
|
130
167
|
if (!reply.text.trim())
|
|
131
168
|
out = { kind: 'fail', status: 502, body: 'empty answer', unusable: true };
|
|
132
169
|
else if (opts.accept && !opts.accept(reply.text))
|
|
@@ -136,17 +173,22 @@ export function createLodestone(config = {}) {
|
|
|
136
173
|
}
|
|
137
174
|
catch (e) {
|
|
138
175
|
const name = e.name;
|
|
139
|
-
if (
|
|
176
|
+
if (opts.signal?.aborted)
|
|
177
|
+
out = { kind: 'aborted' };
|
|
178
|
+
else if (name === 'TimeoutError' || name === 'AbortError')
|
|
140
179
|
out = { kind: 'timeout' };
|
|
141
180
|
else if (e instanceof ProviderError)
|
|
142
181
|
out = { kind: 'fail', status: e.status, body: e.body, message: messageFrom(e.body) };
|
|
143
182
|
else
|
|
144
183
|
out = { kind: 'fail', status: 599, body: e.message };
|
|
145
184
|
}
|
|
185
|
+
if (out.kind === 'aborted')
|
|
186
|
+
throw new LodestoneError('aborted'); // the person left: nothing more to try
|
|
146
187
|
const event = { provider: provider.name, model, phase, outcome: out.kind === 'ok' ? 'ok' : out.kind === 'timeout' ? 'timeout' : 'failed', status: out.kind === 'fail' ? out.status : undefined, ms: now() - at };
|
|
147
188
|
// A key or credit problem (401, 402, 403) says nothing about the model, so it is not counted against it.
|
|
148
189
|
const credential = out.kind === 'fail' && [401, 402, 403].includes(out.status);
|
|
149
|
-
|
|
190
|
+
// A call marked private can fail because few providers meet the rule: that says nothing about the model.
|
|
191
|
+
if (!credential && !strict) {
|
|
150
192
|
config.stats?.record(event);
|
|
151
193
|
await config.health?.record(provider.name, model, out.kind === 'ok');
|
|
152
194
|
}
|
|
@@ -154,9 +196,9 @@ export function createLodestone(config = {}) {
|
|
|
154
196
|
last = out;
|
|
155
197
|
return out;
|
|
156
198
|
};
|
|
157
|
-
const finish = async (provider, model, reply, phase) => {
|
|
199
|
+
const finish = async (provider, model, reply, phase, free = false) => {
|
|
158
200
|
const result = { text: reply.text, provider: provider.name, model, usage: reply.usage };
|
|
159
|
-
if (phase && !callerKey && config.spend) {
|
|
201
|
+
if (phase && !callerKey && !free && config.spend) {
|
|
160
202
|
const costUsd = reply.usage?.costUsd ?? (await estimateCost(model, reply.usage));
|
|
161
203
|
// A ledger that fails must never fail an answer the person is waiting for.
|
|
162
204
|
try {
|
|
@@ -201,12 +243,18 @@ export function createLodestone(config = {}) {
|
|
|
201
243
|
const paidFirst = Boolean(config.openrouter?.paidFirst) && !callerKey && paidAvailable;
|
|
202
244
|
/** The free models. A withdrawn one has already been replaced by resolveModels. */
|
|
203
245
|
const runFree = async () => {
|
|
204
|
-
const all = await order(await
|
|
205
|
-
|
|
246
|
+
const all = await order(await resolvePool(tier, strict));
|
|
247
|
+
// Choose the live ones from the pool: a model that is down is replaced by the next best, so the
|
|
248
|
+
// list a request walks stays short however long the pool is.
|
|
249
|
+
const live = await healthy(router, all, 'free');
|
|
250
|
+
const routerLive = live.includes(ROUTER);
|
|
251
|
+
let models = [...live.filter((m) => m !== ROUTER).slice(0, ACTIVE - (routerLive ? 1 : 0)), ...(routerLive ? [ROUTER] : [])];
|
|
206
252
|
// With nothing after the free models to fall back on, a model that may be down is still better than no attempt.
|
|
207
253
|
if (models.length === 0 && !later && !paidFirst)
|
|
208
|
-
models = all;
|
|
209
|
-
|
|
254
|
+
models = all.slice(0, ACTIVE);
|
|
255
|
+
// Behind a paid-first attempt the free models are a backup, and other providers may still be waiting their turn: an OpenRouter outage must not use up the whole budget on free models.
|
|
256
|
+
const freeStartedAt = now();
|
|
257
|
+
const leftHere = () => (paidFirst ? (fbs.length ? Math.min(left(), t.freePhaseLater - (now() - freeStartedAt)) : left()) : freeLeft());
|
|
210
258
|
for (let i = 0; i < models.length && leftHere() >= t.minSlice; i++) {
|
|
211
259
|
const slice = i >= models.length - 1 ? leftHere() : Math.min(later && !paidFirst ? t.freeSliceLater : t.freeSlice, leftHere());
|
|
212
260
|
const out = await attempt(router, orKey, models[i], 'free', slice);
|
|
@@ -235,8 +283,8 @@ export function createLodestone(config = {}) {
|
|
|
235
283
|
break;
|
|
236
284
|
if (!(await mayUse(router, m, 'paid')))
|
|
237
285
|
continue;
|
|
238
|
-
// Going first, a paid model may not use all the time: the free models and
|
|
239
|
-
const ms = paidFirst
|
|
286
|
+
// Going first, a paid model may not use all the time: the free models and other providers come after it and need some.
|
|
287
|
+
const ms = paidFirst ? Math.min(left(), t.paidSlice) : left();
|
|
240
288
|
const out = await attempt(router, orKey, m, 'paid', ms);
|
|
241
289
|
if (out.kind === 'ok')
|
|
242
290
|
return finish(router, m, out.reply, 'paid');
|
|
@@ -273,18 +321,18 @@ export function createLodestone(config = {}) {
|
|
|
273
321
|
}
|
|
274
322
|
// Phase 3: other providers, with their own keys.
|
|
275
323
|
for (const fb of fbs) {
|
|
276
|
-
const fbAll = await order(fb
|
|
324
|
+
const fbAll = await order(await resolveFallback(fb, tier, strict));
|
|
277
325
|
let fbList = await healthy(fb.provider, fbAll, 'fallback');
|
|
278
326
|
if (fbList.length === 0)
|
|
279
327
|
fbList = fbAll;
|
|
280
328
|
for (const m of fbList) {
|
|
281
329
|
if (left() < t.minSlice)
|
|
282
330
|
break;
|
|
283
|
-
if (!(await mayUse(fb.provider, m, 'fallback')))
|
|
331
|
+
if (!(await mayUse(fb.provider, m, 'fallback', fb.free)))
|
|
284
332
|
continue;
|
|
285
333
|
const out = await attempt(fb.provider, fb.apiKey, m, 'fallback', left());
|
|
286
334
|
if (out.kind === 'ok')
|
|
287
|
-
return finish(fb.provider, m, out.reply, 'fallback');
|
|
335
|
+
return finish(fb.provider, m, out.reply, 'fallback', fb.free);
|
|
288
336
|
if (out.kind === 'fail' && out.status === 401)
|
|
289
337
|
break; // this provider's key is wrong: skip its other models
|
|
290
338
|
}
|
|
@@ -334,10 +382,38 @@ export function createLodestone(config = {}) {
|
|
|
334
382
|
if (!models)
|
|
335
383
|
return { ok: false, error: 'could not read the OpenRouter catalogue' };
|
|
336
384
|
const ids = new Set(models.map((m) => m.id));
|
|
385
|
+
const byId = new Map(models.map((m) => [m.id, m]));
|
|
337
386
|
const all = [...new Set([...Object.values(prefs).flat(), ...paidModels])].filter((m) => m !== ROUTER);
|
|
338
387
|
const withdrawn = all.filter((m) => !ids.has(m));
|
|
339
|
-
const
|
|
340
|
-
|
|
388
|
+
const horizon = (policy?.expiringWithinDays ?? 7) * 86_400_000;
|
|
389
|
+
const expiring = all.filter((m) => { const d = byId.get(m)?.expiration_date; return d !== undefined && Date.parse(d) - now() < horizon; });
|
|
390
|
+
const shortTiers = Object.keys(prefs).filter((tier) => pickModels(tier, models, prefs[tier], { policy, now: now() }).filter((m) => m !== ROUTER).length < 2);
|
|
391
|
+
// Every other provider that can list its models gets the same check: what has gone, and what is retiring.
|
|
392
|
+
const providers = {};
|
|
393
|
+
for (const [fb, cat] of fbCatalogs) {
|
|
394
|
+
const listed = await cat.refresh().catch(() => null);
|
|
395
|
+
if (!listed) {
|
|
396
|
+
providers[fb.provider.name] = { models: 0, withdrawn: [], expiring: [], error: 'could not read its model list' };
|
|
397
|
+
continue;
|
|
398
|
+
}
|
|
399
|
+
const have = new Map(listed.map((m) => [m.id, m]));
|
|
400
|
+
providers[fb.provider.name] = {
|
|
401
|
+
models: listed.length,
|
|
402
|
+
withdrawn: fb.models.filter((m) => !have.has(m)),
|
|
403
|
+
expiring: fb.models.filter((m) => { const d = have.get(m)?.expiration_date; return d !== undefined && Date.parse(d) - now() < horizon; }),
|
|
404
|
+
};
|
|
405
|
+
}
|
|
406
|
+
return { ok: shortTiers.length === 0, catalogSize: models.length, withdrawn, expiring, shortTiers, ...(Object.keys(providers).length ? { providers } : {}) };
|
|
407
|
+
}
|
|
408
|
+
/** An answer already saved under this key, without calling any model: so a caller can serve it without counting it as a request. */
|
|
409
|
+
async function peek(cacheKey, ttlMs = 24 * 3_600_000) {
|
|
410
|
+
try {
|
|
411
|
+
const hit = (await store.get(`lodestone:cache:${cacheKey}`));
|
|
412
|
+
return hit?.text && now() - hit.at < ttlMs ? { text: hit.text, provider: hit.provider, model: hit.model, cached: true } : null;
|
|
413
|
+
}
|
|
414
|
+
catch {
|
|
415
|
+
return null;
|
|
416
|
+
}
|
|
341
417
|
}
|
|
342
418
|
/** Every model this instance can call, for an evaluator to benchmark (see candidates.ts). */
|
|
343
419
|
async function candidates() {
|
|
@@ -361,5 +437,5 @@ export function createLodestone(config = {}) {
|
|
|
361
437
|
add(id, fb.provider.name, 'fallback');
|
|
362
438
|
return { schema: CANDIDATES_SCHEMA, source: 'lodestone', generatedAt: new Date(now()).toISOString(), models: [...byKey.values()] };
|
|
363
439
|
}
|
|
364
|
-
return { complete, json, resolveModels, refreshCatalog, checkModels, candidates };
|
|
440
|
+
return { complete, json, resolveModels, refreshCatalog, checkModels, candidates, peek };
|
|
365
441
|
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { CatalogModel } from '../catalog.js';
|
|
2
|
+
import type { Provider } from '../types.js';
|
|
3
|
+
/** Any service that speaks OpenAI's chat API: gateways (Requesty, LLM API), providers with free tiers
|
|
4
|
+
* (Groq, Cerebras), OpenAI and Mistral themselves, and local servers (Ollama). One adapter, with a
|
|
5
|
+
* preset for each service Lodestone has been checked against. Each preset also knows how to read that
|
|
6
|
+
* service's list of models, so Lodestone can drop a model that has gone and see its price and what it
|
|
7
|
+
* does with prompts. */
|
|
8
|
+
export interface OpenAICompatibleOptions {
|
|
9
|
+
name: string;
|
|
10
|
+
/** Up to and including /v1, with no trailing slash. */
|
|
11
|
+
baseUrl: string;
|
|
12
|
+
fetch?: typeof fetch;
|
|
13
|
+
vision?: boolean;
|
|
14
|
+
/** Extra headers (a referer, an organisation). */
|
|
15
|
+
headers?: Record<string, string>;
|
|
16
|
+
/** How much each model should think, as the service understands it; null to send nothing. */
|
|
17
|
+
reasoning?: (model: string) => Record<string, unknown> | null;
|
|
18
|
+
/** Reads one entry of the service's model list. Return null to skip it (not a chat model). Default: just the id. */
|
|
19
|
+
normalize?: (raw: Record<string, unknown>) => CatalogModel | null;
|
|
20
|
+
}
|
|
21
|
+
export declare function openaiCompatible(o: OpenAICompatibleOptions): Provider;
|
|
22
|
+
/** Requesty publishes, for every model, its price per token, the lab that made it, whether it is trained on and how long prompts are kept. */
|
|
23
|
+
export declare const normalizeRequesty: (r: Record<string, unknown>) => CatalogModel | null;
|
|
24
|
+
/** LLM API lists chat, image, video, speech and embedding models together: only "chat" is kept. */
|
|
25
|
+
export declare const normalizeLlmapi: (r: Record<string, unknown>) => CatalogModel | null;
|
|
26
|
+
export interface PresetOptions {
|
|
27
|
+
fetch?: typeof fetch;
|
|
28
|
+
headers?: Record<string, string>;
|
|
29
|
+
reasoning?: OpenAICompatibleOptions['reasoning'];
|
|
30
|
+
}
|
|
31
|
+
/** https://www.requesty.ai: a gateway to 800+ models, with failover between providers. `region: 'eu'` keeps prompts in the EU. */
|
|
32
|
+
export declare const requesty: (o?: PresetOptions & {
|
|
33
|
+
region?: "eu";
|
|
34
|
+
}) => Provider;
|
|
35
|
+
/** https://llmapi.ai: a gateway to 150+ chat models. */
|
|
36
|
+
export declare const llmapi: (o?: PresetOptions) => Provider;
|
|
37
|
+
/** https://groq.com: fast inference with a free developer tier. */
|
|
38
|
+
export declare const groq: (o?: PresetOptions) => Provider;
|
|
39
|
+
/** https://cerebras.ai: fast inference with a free tier. */
|
|
40
|
+
export declare const cerebras: (o?: PresetOptions) => Provider;
|
|
41
|
+
export declare const openai: (o?: PresetOptions) => Provider;
|
|
42
|
+
export declare const mistral: (o?: PresetOptions) => Provider;
|
|
43
|
+
export declare const together: (o?: PresetOptions) => Provider;
|
|
44
|
+
/** A local Ollama server: free, private, and nothing to go down. The key is ignored but must be non-empty. */
|
|
45
|
+
export declare const ollama: (o?: PresetOptions & {
|
|
46
|
+
baseUrl?: string;
|
|
47
|
+
}) => Provider;
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { ProviderError } from '../errors.js';
|
|
2
|
+
const part = (p) => (p.type === 'text' ? { type: 'text', text: p.text } : { type: 'image_url', image_url: { url: p.dataUrl } });
|
|
3
|
+
const wire = (m) => ({ role: m.role, content: typeof m.content === 'string' ? m.content : m.content.map(part) });
|
|
4
|
+
export function openaiCompatible(o) {
|
|
5
|
+
const f = o.fetch ?? fetch;
|
|
6
|
+
const norm = o.normalize ?? ((raw) => (typeof raw.id === 'string' ? { id: raw.id } : null));
|
|
7
|
+
return {
|
|
8
|
+
name: o.name,
|
|
9
|
+
vision: o.vision ?? true,
|
|
10
|
+
async call(req) {
|
|
11
|
+
const reasoning = o.reasoning?.(req.model);
|
|
12
|
+
const res = await f(`${o.baseUrl}/chat/completions`, {
|
|
13
|
+
method: 'POST',
|
|
14
|
+
headers: { Authorization: `Bearer ${req.apiKey}`, 'Content-Type': 'application/json', ...o.headers },
|
|
15
|
+
body: JSON.stringify({
|
|
16
|
+
model: req.model,
|
|
17
|
+
max_tokens: req.maxTokens,
|
|
18
|
+
...(req.temperature === undefined ? {} : { temperature: req.temperature }),
|
|
19
|
+
...(reasoning ? { reasoning } : {}),
|
|
20
|
+
messages: [...(req.system ? [{ role: 'system', content: req.system }] : []), ...req.messages.map(wire)],
|
|
21
|
+
}),
|
|
22
|
+
signal: req.signal,
|
|
23
|
+
});
|
|
24
|
+
if (!res.ok)
|
|
25
|
+
throw new ProviderError(res.status, await res.text());
|
|
26
|
+
const data = (await res.json());
|
|
27
|
+
return {
|
|
28
|
+
text: data.choices?.[0]?.message?.content ?? '',
|
|
29
|
+
usage: data.usage ? { inputTokens: data.usage.prompt_tokens ?? 0, outputTokens: data.usage.completion_tokens ?? 0, ...(typeof data.usage.cost === 'number' ? { costUsd: data.usage.cost } : {}) } : undefined,
|
|
30
|
+
};
|
|
31
|
+
},
|
|
32
|
+
async listModels(apiKey, signal) {
|
|
33
|
+
const res = await f(`${o.baseUrl}/models`, { headers: { Authorization: `Bearer ${apiKey}`, ...o.headers }, signal: signal ?? AbortSignal.timeout(10_000) });
|
|
34
|
+
if (!res.ok)
|
|
35
|
+
return null;
|
|
36
|
+
const body = (await res.json());
|
|
37
|
+
const rows = Array.isArray(body) ? body : body.data ?? [];
|
|
38
|
+
const models = rows.map(norm).filter((m) => m !== null);
|
|
39
|
+
return models.length ? models : null;
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : undefined);
|
|
44
|
+
const str = (v) => (typeof v === 'string' && v ? v : undefined);
|
|
45
|
+
/** Requesty publishes, for every model, its price per token, the lab that made it, whether it is trained on and how long prompts are kept. */
|
|
46
|
+
export const normalizeRequesty = (r) => {
|
|
47
|
+
if (typeof r.id !== 'string' || (r.api !== undefined && r.api !== 'chat'))
|
|
48
|
+
return null;
|
|
49
|
+
const inP = num(r.input_price);
|
|
50
|
+
const outP = num(r.output_price);
|
|
51
|
+
return {
|
|
52
|
+
id: r.id,
|
|
53
|
+
context_length: num(r.context_window),
|
|
54
|
+
architecture: { input_modalities: r.supports_vision ? ['text', 'image'] : ['text'], output_modalities: ['text'] },
|
|
55
|
+
...(inP !== undefined && outP !== undefined ? { pricing: { prompt: inP, completion: outP }, free: inP === 0 && outP === 0 } : {}),
|
|
56
|
+
...(str(r.model_lab) ? { publisher: str(r.model_lab).toLowerCase() } : {}),
|
|
57
|
+
...(typeof r.data_used_for_training === 'boolean' ? { data_used_for_training: r.data_used_for_training } : {}),
|
|
58
|
+
...(num(r.data_retention_days) !== undefined ? { data_retention_days: num(r.data_retention_days) } : {}),
|
|
59
|
+
};
|
|
60
|
+
};
|
|
61
|
+
/** LLM API gives a model "family" (anthropic, gemini-flash, glm...): turned into the lab where that is clear. */
|
|
62
|
+
const labOf = (family) => {
|
|
63
|
+
const f = family.toLowerCase();
|
|
64
|
+
return /gemini|gemma/.test(f) ? 'google' : /claude/.test(f) ? 'anthropic' : /^gpt|^o\d|openai/.test(f) ? 'openai' : /mistral|mixtral|codestral/.test(f) ? 'mistralai' : /llama/.test(f) ? 'meta-llama' : /grok/.test(f) ? 'x-ai' : f;
|
|
65
|
+
};
|
|
66
|
+
/** LLM API lists chat, image, video, speech and embedding models together: only "chat" is kept. */
|
|
67
|
+
export const normalizeLlmapi = (r) => {
|
|
68
|
+
if (typeof r.id !== 'string' || r.kind !== 'chat')
|
|
69
|
+
return null;
|
|
70
|
+
const p = (r.pricing ?? {});
|
|
71
|
+
return {
|
|
72
|
+
id: r.id,
|
|
73
|
+
context_length: num(r.context_length),
|
|
74
|
+
architecture: { input_modalities: ['text'], output_modalities: ['text'] },
|
|
75
|
+
...(str(p.prompt) !== undefined && str(p.completion) !== undefined ? { pricing: { prompt: str(p.prompt), completion: str(p.completion) } } : {}),
|
|
76
|
+
...(r.free === true ? { free: true } : {}),
|
|
77
|
+
...(str(r.family) ? { publisher: labOf(str(r.family)) } : {}),
|
|
78
|
+
};
|
|
79
|
+
};
|
|
80
|
+
/** https://www.requesty.ai: a gateway to 800+ models, with failover between providers. `region: 'eu'` keeps prompts in the EU. */
|
|
81
|
+
export const requesty = (o = {}) => openaiCompatible({ name: 'requesty', baseUrl: o.region === 'eu' ? 'https://router.eu.requesty.ai/v1' : 'https://router.requesty.ai/v1', normalize: normalizeRequesty, ...o });
|
|
82
|
+
/** https://llmapi.ai: a gateway to 150+ chat models. */
|
|
83
|
+
export const llmapi = (o = {}) => openaiCompatible({ name: 'llmapi', baseUrl: 'https://api.llmapi.ai/v1', normalize: normalizeLlmapi, ...o });
|
|
84
|
+
/** https://groq.com: fast inference with a free developer tier. */
|
|
85
|
+
export const groq = (o = {}) => openaiCompatible({ name: 'groq', baseUrl: 'https://api.groq.com/openai/v1', ...o });
|
|
86
|
+
/** https://cerebras.ai: fast inference with a free tier. */
|
|
87
|
+
export const cerebras = (o = {}) => openaiCompatible({ name: 'cerebras', baseUrl: 'https://api.cerebras.ai/v1', ...o });
|
|
88
|
+
export const openai = (o = {}) => openaiCompatible({ name: 'openai', baseUrl: 'https://api.openai.com/v1', ...o });
|
|
89
|
+
export const mistral = (o = {}) => openaiCompatible({ name: 'mistral', baseUrl: 'https://api.mistral.ai/v1', ...o });
|
|
90
|
+
export const together = (o = {}) => openaiCompatible({ name: 'together', baseUrl: 'https://api.together.xyz/v1', ...o });
|
|
91
|
+
/** A local Ollama server: free, private, and nothing to go down. The key is ignored but must be non-empty. */
|
|
92
|
+
export const ollama = (o = {}) => openaiCompatible({ name: 'ollama', baseUrl: o.baseUrl ?? 'http://localhost:11434/v1', vision: false, ...o });
|
|
@@ -28,6 +28,8 @@ export function openrouter(opts = {}) {
|
|
|
28
28
|
...(req.temperature === undefined ? {} : { temperature: req.temperature }),
|
|
29
29
|
max_tokens: req.maxTokens,
|
|
30
30
|
...reasoningFor(req.model),
|
|
31
|
+
// Only providers that do not keep or train on prompts. Many free models cannot meet this, so the request fails and the next model is tried.
|
|
32
|
+
...(req.privacy === 'strict' ? { provider: { data_collection: 'deny' } } : {}),
|
|
31
33
|
messages: [...(req.system ? [{ role: 'system', content: req.system }] : []), ...req.messages.map(wire)],
|
|
32
34
|
}),
|
|
33
35
|
signal: req.signal,
|
package/dist/trust.d.ts
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { CatalogModel } from './catalog.js';
|
|
2
|
+
/** What may be used, and what may be added by itself. There is no vulnerability feed for hosted models,
|
|
3
|
+
* so this does not claim one. It checks what can really be checked, from what providers publish in
|
|
4
|
+
* their model lists and what the app decides:
|
|
5
|
+
*
|
|
6
|
+
* - who made it: a model found by discovery is only used if its publisher is on a list of well-known
|
|
7
|
+
* labs (the app's own curated lists are its own decision and are not held to this);
|
|
8
|
+
* - what happens to the prompt: a model its provider says is trained on, or keeps prompts longer than
|
|
9
|
+
* allowed, is refused (Requesty publishes both for every model);
|
|
10
|
+
* - that it is not about to disappear: a model with an expiry date within a week is dropped;
|
|
11
|
+
* - an explicit block list. */
|
|
12
|
+
/** Well-known labs. A discovered model from anyone else is not added by itself. */
|
|
13
|
+
export declare const DEFAULT_PUBLISHERS: string[];
|
|
14
|
+
export interface TrustPolicy {
|
|
15
|
+
/** Who a discovered model may come from. Default: DEFAULT_PUBLISHERS. 'any' turns the check off. */
|
|
16
|
+
publishers?: string[] | 'any';
|
|
17
|
+
/** Never use these: exact model ids, or a publisher followed by a slash ("shady-lab/"). Applies to every model, curated or discovered. */
|
|
18
|
+
block?: string[];
|
|
19
|
+
/** Refuse a model its provider says is used for training. Where the provider does not say, a discovered model is refused and a curated one is allowed. */
|
|
20
|
+
requireNoTraining?: boolean;
|
|
21
|
+
/** Refuse a model that keeps prompts longer than this. Unknown is treated like requireNoTraining treats it. */
|
|
22
|
+
maxRetentionDays?: number;
|
|
23
|
+
/** Drop a model that stops being offered within this many days. Default 7. */
|
|
24
|
+
expiringWithinDays?: number;
|
|
25
|
+
}
|
|
26
|
+
export interface VetContext {
|
|
27
|
+
/** Found by discovery, not named by the app. */
|
|
28
|
+
discovered: boolean;
|
|
29
|
+
/** Treat unknown privacy facts as a refusal even for a curated model (a call marked private). */
|
|
30
|
+
strict?: boolean;
|
|
31
|
+
now: number;
|
|
32
|
+
}
|
|
33
|
+
export type Verdict = {
|
|
34
|
+
ok: true;
|
|
35
|
+
} | {
|
|
36
|
+
ok: false;
|
|
37
|
+
reason: 'blocked' | 'untrusted-publisher' | 'trains-on-data' | 'unknown-data-policy' | 'retention' | 'expiring';
|
|
38
|
+
};
|
|
39
|
+
export declare const publisherOf: (m: {
|
|
40
|
+
id: string;
|
|
41
|
+
publisher?: string;
|
|
42
|
+
}) => string;
|
|
43
|
+
export declare function vet(m: CatalogModel, policy: TrustPolicy | undefined, ctx: VetContext): Verdict;
|
package/dist/trust.js
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/** What may be used, and what may be added by itself. There is no vulnerability feed for hosted models,
|
|
2
|
+
* so this does not claim one. It checks what can really be checked, from what providers publish in
|
|
3
|
+
* their model lists and what the app decides:
|
|
4
|
+
*
|
|
5
|
+
* - who made it: a model found by discovery is only used if its publisher is on a list of well-known
|
|
6
|
+
* labs (the app's own curated lists are its own decision and are not held to this);
|
|
7
|
+
* - what happens to the prompt: a model its provider says is trained on, or keeps prompts longer than
|
|
8
|
+
* allowed, is refused (Requesty publishes both for every model);
|
|
9
|
+
* - that it is not about to disappear: a model with an expiry date within a week is dropped;
|
|
10
|
+
* - an explicit block list. */
|
|
11
|
+
/** Well-known labs. A discovered model from anyone else is not added by itself. */
|
|
12
|
+
export const DEFAULT_PUBLISHERS = ['anthropic', 'google', 'openai', 'meta-llama', 'mistralai', 'nvidia', 'qwen', 'deepseek', 'microsoft', 'cohere', 'x-ai', 'amazon'];
|
|
13
|
+
/** The same lab goes by different names in different lists (Requesty says "mistral", OpenRouter "mistralai"; LLM API says "gemini-flash"). */
|
|
14
|
+
const ALIASES = { mistral: 'mistralai', meta: 'meta-llama', xai: 'x-ai', 'x.ai': 'x-ai', claude: 'anthropic', gemini: 'google', 'gemini-flash': 'google', 'gemini-pro': 'google', gemma: 'google', gpt: 'openai' };
|
|
15
|
+
export const publisherOf = (m) => { const raw = (m.publisher ?? m.id.split('/')[0]).toLowerCase(); return ALIASES[raw] ?? raw; };
|
|
16
|
+
export function vet(m, policy, ctx) {
|
|
17
|
+
const p = policy ?? {};
|
|
18
|
+
const pub = publisherOf(m);
|
|
19
|
+
if ((p.block ?? []).some((b) => (b.endsWith('/') ? m.id.toLowerCase().startsWith(b.toLowerCase()) || pub === b.slice(0, -1).toLowerCase() : m.id === b)))
|
|
20
|
+
return { ok: false, reason: 'blocked' };
|
|
21
|
+
if (m.expiration_date) {
|
|
22
|
+
const at = Date.parse(m.expiration_date);
|
|
23
|
+
if (Number.isFinite(at) && at - ctx.now < (p.expiringWithinDays ?? 7) * 86_400_000)
|
|
24
|
+
return { ok: false, reason: 'expiring' };
|
|
25
|
+
}
|
|
26
|
+
const unknownRefused = ctx.discovered || ctx.strict;
|
|
27
|
+
if (p.requireNoTraining) {
|
|
28
|
+
if (m.data_used_for_training === true)
|
|
29
|
+
return { ok: false, reason: 'trains-on-data' };
|
|
30
|
+
if (m.data_used_for_training === undefined && unknownRefused)
|
|
31
|
+
return { ok: false, reason: 'unknown-data-policy' };
|
|
32
|
+
}
|
|
33
|
+
if (p.maxRetentionDays !== undefined) {
|
|
34
|
+
if (m.data_retention_days !== undefined && m.data_retention_days > p.maxRetentionDays)
|
|
35
|
+
return { ok: false, reason: 'retention' };
|
|
36
|
+
if (m.data_retention_days === undefined && unknownRefused)
|
|
37
|
+
return { ok: false, reason: 'unknown-data-policy' };
|
|
38
|
+
}
|
|
39
|
+
if (ctx.discovered && p.publishers !== 'any' && !(p.publishers ?? DEFAULT_PUBLISHERS).map((x) => x.toLowerCase()).includes(pub))
|
|
40
|
+
return { ok: false, reason: 'untrusted-publisher' };
|
|
41
|
+
return { ok: true };
|
|
42
|
+
}
|
package/dist/types.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { CatalogModel } from './catalog.js';
|
|
1
2
|
export type Tier = 'fast' | 'strong' | 'vision';
|
|
2
3
|
export type ContentPart = {
|
|
3
4
|
type: 'text';
|
|
@@ -25,6 +26,8 @@ export interface ProviderRequest {
|
|
|
25
26
|
/** Left out when the caller did not set one, so the provider's own default applies. */
|
|
26
27
|
temperature?: number;
|
|
27
28
|
apiKey: string;
|
|
29
|
+
/** 'strict': the prompt must not be kept or used for training. A provider that can enforce it does (OpenRouter); the rest rely on Lodestone choosing only models whose list says so. */
|
|
30
|
+
privacy?: 'strict';
|
|
28
31
|
signal: AbortSignal;
|
|
29
32
|
}
|
|
30
33
|
export interface ProviderReply {
|
|
@@ -36,6 +39,8 @@ export interface Provider {
|
|
|
36
39
|
/** Can this provider read images? */
|
|
37
40
|
vision: boolean;
|
|
38
41
|
call(req: ProviderRequest): Promise<ProviderReply>;
|
|
42
|
+
/** The models the provider offers right now, with price and what it does with prompts where it says. Lets Lodestone drop a model that has gone and find others. */
|
|
43
|
+
listModels?(apiKey: string, signal?: AbortSignal): Promise<CatalogModel[] | null>;
|
|
39
44
|
}
|
|
40
45
|
/** Where Lodestone keeps the model catalogue and cached answers. Anything that can
|
|
41
46
|
* get and set JSON by key will do: Netlify Blobs, Redis, a SQLite table, a Map. */
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// Redundancy: OpenRouter first, then two other gateways and a free tier, each with its own key, so one
|
|
2
|
+
// provider's outage does not take the tool down. Discovery keeps each list alive and keeps it short.
|
|
3
|
+
import { createLodestone, groq, llmapi, requesty } from '@extrafemi-ai/lodestone';
|
|
4
|
+
|
|
5
|
+
const CHEAP = { maxPromptUsdPerMTok: 1, maxCompletionUsdPerMTok: 5 };
|
|
6
|
+
|
|
7
|
+
export const ai = createLodestone({
|
|
8
|
+
openrouter: { apiKey: process.env.OPENROUTER_API_KEY, allowPaid: true, paidFirst: true, paidModels: ['anthropic/claude-haiku-5.5'] },
|
|
9
|
+
fallbacks: [
|
|
10
|
+
// Model ids are each gateway's own. Models a gateway stops listing are dropped, and if the list is short it is
|
|
11
|
+
// topped up (never above 3) with trusted-lab models under the price limit.
|
|
12
|
+
{ provider: requesty(), apiKey: process.env.REQUESTY_API_KEY ?? '', models: ['bedrock/claude-haiku-5-5', 'vertex/gemini-3.8-flash'], discover: { fill: CHEAP, max: 3 } },
|
|
13
|
+
{ provider: llmapi(), apiKey: process.env.LLMAPI_API_KEY ?? '', models: ['claude-haiku-5-5', 'gemini-3.8-flash'], discover: { fill: CHEAP, max: 3 } },
|
|
14
|
+
// A free tier costs nothing, so it is never counted against a spend limit.
|
|
15
|
+
{ provider: groq(), apiKey: process.env.GROQ_API_KEY ?? '', models: ['llama-3.3-70b-versatile'], free: true },
|
|
16
|
+
], // a fallback with no key is skipped, so only the ones you have keys for are used
|
|
17
|
+
trust: {
|
|
18
|
+
requireNoTraining: true, // refuse a model its provider says is trained on (Requesty publishes this for every model)
|
|
19
|
+
maxRetentionDays: 30,
|
|
20
|
+
block: [], // exact model ids, or a lab followed by a slash
|
|
21
|
+
},
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
// For something sensitive, like a CV: OpenRouter is told to use only providers that do not keep or train on it,
|
|
25
|
+
// and other models are used only if their provider's list says the same.
|
|
26
|
+
export const summariseCv = (cv: string) => ai.json({ prompt: `Summarise:\n${cv}`, private: true });
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// Run once a day. Saves the current model catalogues, so a withdrawn or retiring model is replaced
|
|
2
|
+
// without anyone editing a list, and says what it found. Reported as a failure (so your monitor
|
|
3
|
+
// notices) only when a catalogue cannot be read or a tier is left with fewer than two models.
|
|
4
|
+
import { ai } from './netlify-function.js';
|
|
5
|
+
|
|
6
|
+
export const config = { schedule: '0 4 * * *' };
|
|
7
|
+
|
|
8
|
+
export const handler = async () => {
|
|
9
|
+
const r = await ai.checkModels();
|
|
10
|
+
console.log(`models: ${r.catalogSize ?? 0} in the catalogue; withdrawn: ${r.withdrawn?.join(', ') || 'none'}; retiring soon: ${r.expiring?.join(', ') || 'none'}`);
|
|
11
|
+
for (const [name, p] of Object.entries(r.providers ?? {})) console.log(`${name}: ${p.error ?? `${p.models} models; withdrawn: ${p.withdrawn.join(', ') || 'none'}; retiring: ${p.expiring.join(', ') || 'none'}`}`);
|
|
12
|
+
return { statusCode: 200, body: JSON.stringify(r) }; // body.ok === false is what a monitor should alert on
|
|
13
|
+
};
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// A Netlify function that asks a model for JSON, written so it keeps working when a model is
|
|
2
|
+
// withdrawn, slow or rate-limited. Needs OPENROUTER_API_KEY (and, for a second route, GROQ_API_KEY).
|
|
3
|
+
import { createHealth, createLodestone, explain, groq, LodestoneError } from '@extrafemi-ai/lodestone';
|
|
4
|
+
import { getStore } from '@netlify/blobs';
|
|
5
|
+
import type { Handler } from '@netlify/functions';
|
|
6
|
+
|
|
7
|
+
// Where Lodestone keeps what it learns: the model catalogue, which models are failing, saved answers.
|
|
8
|
+
// Blob keys cannot contain colons, so they are made simple.
|
|
9
|
+
const blobs = getStore('lodestone');
|
|
10
|
+
const simple = (k: string) => k.replace(/[^a-z0-9._-]/gi, '-');
|
|
11
|
+
export const store = { get: (k: string) => blobs.get(simple(k), { type: 'json' }), set: (k: string, v: unknown) => blobs.setJSON(simple(k), v) };
|
|
12
|
+
|
|
13
|
+
export const ai = createLodestone({
|
|
14
|
+
openrouter: { apiKey: process.env.OPENROUTER_API_KEY },
|
|
15
|
+
fallbacks: process.env.GROQ_API_KEY ? [{ provider: groq(), apiKey: process.env.GROQ_API_KEY, models: ['llama-3.3-70b-versatile'], free: true }] : [],
|
|
16
|
+
store,
|
|
17
|
+
health: createHealth({ store }), // a model that keeps failing is skipped by every function until it recovers
|
|
18
|
+
budgetMs: 24_000, // Netlify cuts a function off at 30 seconds
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
interface Summary { title: string; points: string[] }
|
|
22
|
+
const isSummary = (v: unknown): v is Summary => typeof v === 'object' && v !== null && typeof (v as Summary).title === 'string' && Array.isArray((v as Summary).points);
|
|
23
|
+
|
|
24
|
+
export const handler: Handler = async (event) => {
|
|
25
|
+
try {
|
|
26
|
+
const { text } = JSON.parse(event.body ?? '{}') as { text?: string };
|
|
27
|
+
if (!text) return { statusCode: 400, body: JSON.stringify({ error: 'text is required' }) };
|
|
28
|
+
// A model whose answer is not valid JSON of this shape is skipped for the next one.
|
|
29
|
+
const r = await ai.json<Summary>({ prompt: `Summarise as JSON {"title": string, "points": string[]}:\n\n${text}`, tier: 'fast', validate: isSummary });
|
|
30
|
+
return { statusCode: 200, body: JSON.stringify(r.data) };
|
|
31
|
+
} catch (err) {
|
|
32
|
+
// A sentence a person can be shown, not a stack trace.
|
|
33
|
+
return { statusCode: err instanceof LodestoneError && err.kind === 'rate_limit' ? 429 : 502, body: JSON.stringify({ error: explain(err, { product: 'MyApp' }) }) };
|
|
34
|
+
}
|
|
35
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// Just enough of the Netlify packages for the examples to be type-checked here without installing them.
|
|
2
|
+
declare module '@netlify/blobs' {
|
|
3
|
+
export function getStore(name: string): { get(key: string, opts: { type: 'json' }): Promise<unknown>; setJSON(key: string, value: unknown): Promise<void> };
|
|
4
|
+
}
|
|
5
|
+
declare module '@netlify/functions' {
|
|
6
|
+
export type Handler = (event: { httpMethod: string; headers: Record<string, string | undefined>; body: string | null }) => Promise<{ statusCode: number; headers?: Record<string, string>; body: string }>;
|
|
7
|
+
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@extrafemi-ai/lodestone",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Calls AI models so your tool keeps working when a model is withdrawn, slow or
|
|
3
|
+
"version": "0.5.0",
|
|
4
|
+
"description": "Calls AI models so your tool keeps working when a model is withdrawn, slow, rate-limited or down: failover across providers, self-updating model lists, spend limits.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -24,7 +24,10 @@
|
|
|
24
24
|
"files": [
|
|
25
25
|
"dist",
|
|
26
26
|
"docs",
|
|
27
|
+
"examples",
|
|
27
28
|
"README.md",
|
|
29
|
+
"CHANGELOG.md",
|
|
30
|
+
"SECURITY.md",
|
|
28
31
|
"LICENSE"
|
|
29
32
|
],
|
|
30
33
|
"sideEffects": false,
|
|
@@ -35,7 +38,8 @@
|
|
|
35
38
|
"build": "tsc -p tsconfig.build.json",
|
|
36
39
|
"typecheck": "tsc --noEmit",
|
|
37
40
|
"test": "vitest run",
|
|
38
|
-
"prepublishOnly": "npm run typecheck && npm test && npm run build"
|
|
41
|
+
"prepublishOnly": "npm run typecheck && npm run typecheck:examples && npm test && npm run build",
|
|
42
|
+
"typecheck:examples": "tsc -p tsconfig.examples.json"
|
|
39
43
|
},
|
|
40
44
|
"devDependencies": {
|
|
41
45
|
"@types/node": "^22.10.0",
|
|
@@ -44,12 +48,17 @@
|
|
|
44
48
|
},
|
|
45
49
|
"keywords": [
|
|
46
50
|
"ai",
|
|
47
|
-
"llm",
|
|
48
|
-
"openrouter",
|
|
49
51
|
"anthropic",
|
|
50
|
-
"
|
|
52
|
+
"circuit-breaker",
|
|
53
|
+
"failover",
|
|
51
54
|
"fallback",
|
|
52
|
-
"
|
|
55
|
+
"gemini",
|
|
56
|
+
"groq",
|
|
57
|
+
"llm",
|
|
58
|
+
"llm-gateway",
|
|
59
|
+
"model-routing",
|
|
60
|
+
"openrouter",
|
|
61
|
+
"requesty"
|
|
53
62
|
],
|
|
54
63
|
"publishConfig": {
|
|
55
64
|
"access": "public",
|