@extrafemi-ai/lodestone 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/README.md +76 -0
- package/SECURITY.md +35 -0
- package/dist/catalog.d.ts +41 -5
- package/dist/catalog.js +53 -25
- package/dist/discover.d.ts +24 -0
- package/dist/discover.js +20 -0
- package/dist/errors.d.ts +4 -0
- package/dist/explain.js +4 -0
- package/dist/health.d.ts +39 -0
- package/dist/health.js +73 -0
- package/dist/index.d.ts +10 -1
- package/dist/index.js +4 -0
- package/dist/lodestone.d.ts +62 -8
- package/dist/lodestone.js +198 -29
- package/dist/providers/openaiCompatible.d.ts +47 -0
- package/dist/providers/openaiCompatible.js +92 -0
- package/dist/providers/openrouter.d.ts +2 -0
- package/dist/providers/openrouter.js +12 -2
- package/dist/stats.js +2 -0
- package/dist/trust.d.ts +43 -0
- package/dist/trust.js +42 -0
- package/dist/types.d.ts +7 -0
- package/examples/multi-provider.ts +26 -0
- package/examples/netlify-daily-check.ts +13 -0
- package/examples/netlify-function.ts +35 -0
- package/examples/stubs.d.ts +7 -0
- package/package.json +16 -7
package/dist/lodestone.d.ts
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { type CatalogModel } from './catalog.js';
|
|
2
|
+
import { type FillOptions } from './discover.js';
|
|
3
|
+
import type { TrustPolicy } from './trust.js';
|
|
2
4
|
import { type Candidates } from './candidates.js';
|
|
5
|
+
import type { Health } from './health.js';
|
|
3
6
|
import { type QualityQuery, type Scorecard } from './scorecard.js';
|
|
4
7
|
import type { Stats } from './stats.js';
|
|
5
8
|
import type { KeyValueStore, Message, Provider, Tier, Usage } from './types.js';
|
|
@@ -15,6 +18,13 @@ export interface FallbackConfig {
|
|
|
15
18
|
apiKey: string;
|
|
16
19
|
/** Tried in order. */
|
|
17
20
|
models: string[];
|
|
21
|
+
/** This provider costs nothing (a free tier, a local server): its calls are not counted against, or stopped by, the spend limit. */
|
|
22
|
+
free?: boolean;
|
|
23
|
+
/** Keep this provider's list alive from its own catalogue (needs a provider that can list its models): models it no longer offers, or that are retiring or refused by `trust`, are dropped, and with `fill` a short list is topped up with trusted models within a price limit, never above `max` (default 3). */
|
|
24
|
+
discover?: {
|
|
25
|
+
fill?: FillOptions;
|
|
26
|
+
max?: number;
|
|
27
|
+
};
|
|
18
28
|
/** Also use this provider when the caller supplied their own OpenRouter key. Default false: a fallback has the platform's key, and a caller's own key must never lead to the platform's money being spent. */
|
|
19
29
|
allowWithCallerKey?: boolean;
|
|
20
30
|
}
|
|
@@ -22,16 +32,37 @@ export interface AttemptEvent {
|
|
|
22
32
|
provider: string;
|
|
23
33
|
model: string;
|
|
24
34
|
phase: 'free' | 'paid' | 'fallback' | 'route';
|
|
25
|
-
|
|
35
|
+
/** 'skipped': not tried, because the breaker had it down or the spend limit was reached. */
|
|
36
|
+
outcome: 'ok' | 'timeout' | 'failed' | 'skipped';
|
|
26
37
|
status?: number;
|
|
27
38
|
ms: number;
|
|
28
39
|
}
|
|
40
|
+
/** Where the platform's money is counted. Only attempts that spend it go through: the paid models and the
|
|
41
|
+
* fallback providers on the platform's keys. A call on the caller's own key never does. */
|
|
42
|
+
export interface SpendControl {
|
|
43
|
+
/** Asked before each such attempt. Return false once the limit is reached and that model is not used. */
|
|
44
|
+
allow(info: {
|
|
45
|
+
provider: string;
|
|
46
|
+
model: string;
|
|
47
|
+
phase: 'paid' | 'fallback';
|
|
48
|
+
}): boolean | Promise<boolean>;
|
|
49
|
+
/** Told after each such attempt that answered, with what it cost: the provider's own figure when it gives one (OpenRouter does), otherwise an estimate from the model's catalogue price, otherwise nothing. */
|
|
50
|
+
record(info: {
|
|
51
|
+
provider: string;
|
|
52
|
+
model: string;
|
|
53
|
+
phase: 'paid' | 'fallback';
|
|
54
|
+
usage?: Usage;
|
|
55
|
+
costUsd?: number;
|
|
56
|
+
}): void | Promise<void>;
|
|
57
|
+
}
|
|
29
58
|
export interface LodestoneConfig {
|
|
30
59
|
openrouter?: {
|
|
31
60
|
/** The platform's key. A caller's own key (`apiKey` on a call) takes priority. */
|
|
32
61
|
apiKey?: string;
|
|
33
62
|
/** Use the paid models after the free ones fail, on the platform's key. A caller's own key always allows it. */
|
|
34
63
|
allowPaid?: boolean;
|
|
64
|
+
/** On the platform's key, try the paid models first and the free ones only if they fail or the spend limit is reached. For when the free models are too unreliable to be worth waiting for. Not applied to a caller's own key: it is their money, so their free models are still tried first. */
|
|
65
|
+
paidFirst?: boolean;
|
|
35
66
|
models?: Partial<Record<Tier, string[]>>;
|
|
36
67
|
paidModels?: string[];
|
|
37
68
|
/** Replace the OpenRouter adapter (tests, proxies). */
|
|
@@ -46,6 +77,12 @@ export interface LodestoneConfig {
|
|
|
46
77
|
* nobody has measured keep their place. A function is called per request, so it can fetch
|
|
47
78
|
* and cache; if it fails or takes over 1.5 s the order is left alone. */
|
|
48
79
|
scorecards?: Scorecard[] | (() => Promise<Scorecard[]>);
|
|
80
|
+
/** What may be used, and what discovery may add by itself: trusted publishers, a block list, what providers say they do with prompts, retiring models. See trust.ts. */
|
|
81
|
+
trust?: TrustPolicy;
|
|
82
|
+
/** Skips models that have been failing, so a request does not spend its time on them (see health.ts). */
|
|
83
|
+
health?: Health;
|
|
84
|
+
/** Counts and limits the platform's spend. Without it nothing is counted or limited. */
|
|
85
|
+
spend?: SpendControl;
|
|
49
86
|
/** Records every attempt so you can publish how each model does (stats.scorecard()). */
|
|
50
87
|
stats?: Stats;
|
|
51
88
|
/** Named tasks, so calls say what they are doing and not which model to use. */
|
|
@@ -60,6 +97,7 @@ export interface LodestoneConfig {
|
|
|
60
97
|
freeSliceWithLaterMs?: number;
|
|
61
98
|
freePhaseWithLaterMs?: number;
|
|
62
99
|
minSliceMs?: number;
|
|
100
|
+
paidSliceMs?: number;
|
|
63
101
|
};
|
|
64
102
|
fetch?: typeof fetch;
|
|
65
103
|
now?: () => number;
|
|
@@ -93,6 +131,10 @@ export interface CompleteOptions {
|
|
|
93
131
|
apiKey?: string;
|
|
94
132
|
/** An answer that fails this check counts as a failed attempt, and the next model is tried. */
|
|
95
133
|
accept?: (text: string) => boolean;
|
|
134
|
+
/** The prompt must not be kept or trained on (a CV, say). OpenRouter is told to use only providers that do not; other models are used only if their provider's list says so. Many free models cannot meet this, and the paid ones are used instead. */
|
|
135
|
+
private?: boolean;
|
|
136
|
+
/** Stops the call when the caller no longer wants the answer (the person left the page). */
|
|
137
|
+
signal?: AbortSignal;
|
|
96
138
|
/** Same key, same answer: the answer is saved and reused. Only for pure, repeatable tasks. */
|
|
97
139
|
cacheKey?: string;
|
|
98
140
|
cacheTtlMs?: number;
|
|
@@ -104,6 +146,23 @@ export interface CompleteResult {
|
|
|
104
146
|
usage?: Usage;
|
|
105
147
|
cached?: boolean;
|
|
106
148
|
}
|
|
149
|
+
export interface CheckResult {
|
|
150
|
+
ok: boolean;
|
|
151
|
+
error?: string;
|
|
152
|
+
catalogSize?: number;
|
|
153
|
+
/** Preferred models the catalogue no longer lists (already replaced). */
|
|
154
|
+
withdrawn?: string[];
|
|
155
|
+
/** Models that stop being offered within a week (already left out). */
|
|
156
|
+
expiring?: string[];
|
|
157
|
+
shortTiers?: Tier[];
|
|
158
|
+
/** The same check for each other provider that can list its models. */
|
|
159
|
+
providers?: Record<string, {
|
|
160
|
+
models: number;
|
|
161
|
+
withdrawn: string[];
|
|
162
|
+
expiring: string[];
|
|
163
|
+
error?: string;
|
|
164
|
+
}>;
|
|
165
|
+
}
|
|
107
166
|
export declare function createLodestone(config?: LodestoneConfig): {
|
|
108
167
|
complete: (opts: CompleteOptions) => Promise<CompleteResult>;
|
|
109
168
|
json: <T = unknown>(opts: CompleteOptions & {
|
|
@@ -113,13 +172,8 @@ export declare function createLodestone(config?: LodestoneConfig): {
|
|
|
113
172
|
}>;
|
|
114
173
|
resolveModels: (tier: Tier) => Promise<string[]>;
|
|
115
174
|
refreshCatalog: () => Promise<CatalogModel[] | null>;
|
|
116
|
-
checkModels: () => Promise<
|
|
117
|
-
ok: boolean;
|
|
118
|
-
error?: string;
|
|
119
|
-
catalogSize?: number;
|
|
120
|
-
withdrawn?: string[];
|
|
121
|
-
shortTiers?: Tier[];
|
|
122
|
-
}>;
|
|
175
|
+
checkModels: () => Promise<CheckResult>;
|
|
123
176
|
candidates: () => Promise<Candidates>;
|
|
177
|
+
peek: (cacheKey: string, ttlMs?: number) => Promise<CompleteResult | null>;
|
|
124
178
|
};
|
|
125
179
|
export type Lodestone = ReturnType<typeof createLodestone>;
|
package/dist/lodestone.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createCatalog, pickModels, ROUTER } from './catalog.js';
|
|
2
|
+
import { resolveProviderModels } from './discover.js';
|
|
2
3
|
import { CANDIDATES_SCHEMA } from './candidates.js';
|
|
3
4
|
import { LodestoneError, messageFrom, ProviderError } from './errors.js';
|
|
4
5
|
import { tryParseJson } from './json.js';
|
|
@@ -10,6 +11,9 @@ import { openrouter } from './providers/openrouter.js';
|
|
|
10
11
|
import { rankModels } from './scorecard.js';
|
|
11
12
|
import { memoryStore } from './store.js';
|
|
12
13
|
const hasImage = (messages) => messages.some((m) => Array.isArray(m.content) && m.content.some((p) => p.type === 'image'));
|
|
14
|
+
/** How many candidates are kept in the background for a tier. A request still tries at most three. */
|
|
15
|
+
const POOL = 8;
|
|
16
|
+
const ACTIVE = 3;
|
|
13
17
|
export function createLodestone(config = {}) {
|
|
14
18
|
const budget = config.budgetMs ?? 24_000;
|
|
15
19
|
const t = {
|
|
@@ -17,11 +21,19 @@ export function createLodestone(config = {}) {
|
|
|
17
21
|
freeSliceLater: config.timing?.freeSliceWithLaterMs ?? 5_000,
|
|
18
22
|
freePhaseLater: config.timing?.freePhaseWithLaterMs ?? 9_000,
|
|
19
23
|
minSlice: config.timing?.minSliceMs ?? 3_000,
|
|
24
|
+
paidSlice: config.timing?.paidSliceMs ?? 12_000,
|
|
20
25
|
};
|
|
21
26
|
const now = config.now ?? Date.now;
|
|
22
27
|
const store = config.store ?? memoryStore();
|
|
23
28
|
const fetchImpl = config.fetch ?? fetch;
|
|
24
29
|
const catalog = createCatalog(store, fetchImpl, now);
|
|
30
|
+
const policy = config.trust;
|
|
31
|
+
/** One catalogue per fallback provider that can list its models, kept in the same store. */
|
|
32
|
+
const fbCatalogs = new Map();
|
|
33
|
+
for (const fb of config.fallbacks ?? []) {
|
|
34
|
+
if (fb.provider.listModels)
|
|
35
|
+
fbCatalogs.set(fb, createCatalog(store, fetchImpl, now, { key: `lodestone:catalog:${fb.provider.name}`, list: () => fb.provider.listModels(fb.apiKey) }));
|
|
36
|
+
}
|
|
25
37
|
const router = config.openrouter?.provider ?? openrouter({ fetch: fetchImpl });
|
|
26
38
|
const prefs = { ...DEFAULT_MODELS, ...config.openrouter?.models };
|
|
27
39
|
const paidModels = config.openrouter?.paidModels ?? DEFAULT_PAID_MODELS;
|
|
@@ -30,12 +42,33 @@ export function createLodestone(config = {}) {
|
|
|
30
42
|
/** The ids to send for a tier: preferences that still exist, plus current replacements. Never throws, never waits on the network. */
|
|
31
43
|
async function resolveModels(tier) {
|
|
32
44
|
try {
|
|
33
|
-
return pickModels(tier, await catalog.load(), prefs[tier]);
|
|
45
|
+
return pickModels(tier, await catalog.load(), prefs[tier], { policy, now: now() });
|
|
34
46
|
}
|
|
35
47
|
catch {
|
|
36
48
|
return prefs[tier];
|
|
37
49
|
}
|
|
38
50
|
}
|
|
51
|
+
/** A longer, ranked list of live candidates for a tier, to choose from when some are down: the best models for the tier that exist, are allowed, and are not retiring. */
|
|
52
|
+
async function resolvePool(tier, strict) {
|
|
53
|
+
try {
|
|
54
|
+
return pickModels(tier, await catalog.load(), prefs[tier], { policy, strict, depth: POOL, now: now() });
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return prefs[tier];
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/** A fallback provider's models for a tier: the app's own, pruned and topped up from the provider's catalogue. */
|
|
61
|
+
async function resolveFallback(fb, tier, strict) {
|
|
62
|
+
const cat = fbCatalogs.get(fb);
|
|
63
|
+
if (!cat)
|
|
64
|
+
return fb.models;
|
|
65
|
+
try {
|
|
66
|
+
return resolveProviderModels(fb.models, await cat.load(), { tier, policy, strict, fill: fb.discover?.fill, max: fb.discover?.max, now: now() });
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
return fb.models;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
39
72
|
/** The scorecards, or none: never throws and never holds a request up for long. */
|
|
40
73
|
async function cards() {
|
|
41
74
|
const src = config.scorecards;
|
|
@@ -50,6 +83,15 @@ export function createLodestone(config = {}) {
|
|
|
50
83
|
return [];
|
|
51
84
|
}
|
|
52
85
|
}
|
|
86
|
+
/** What a call cost, estimated from the model's catalogue price, for providers that do not report it. */
|
|
87
|
+
async function estimateCost(model, usage) {
|
|
88
|
+
if (!usage)
|
|
89
|
+
return undefined;
|
|
90
|
+
const m = (await catalog.load().catch(() => null))?.find((x) => x.id === model);
|
|
91
|
+
const inP = Number(m?.pricing?.prompt);
|
|
92
|
+
const outP = Number(m?.pricing?.completion);
|
|
93
|
+
return Number.isFinite(inP) && Number.isFinite(outP) ? usage.inputTokens * inP + usage.outputTokens * outP : undefined;
|
|
94
|
+
}
|
|
53
95
|
async function complete(opts) {
|
|
54
96
|
const messages = opts.messages ?? (opts.prompt !== undefined ? [{ role: 'user', content: opts.prompt }] : []);
|
|
55
97
|
if (messages.length === 0)
|
|
@@ -61,6 +103,9 @@ export function createLodestone(config = {}) {
|
|
|
61
103
|
const tier = opts.tier ?? profile?.tier ?? (images ? 'vision' : 'fast');
|
|
62
104
|
const maxTokens = opts.maxTokens ?? profile?.maxTokens ?? 3500;
|
|
63
105
|
const temperature = opts.temperature ?? profile?.temperature ?? (opts.route ? undefined : 0.2);
|
|
106
|
+
const strict = Boolean(opts.private);
|
|
107
|
+
if (opts.signal?.aborted)
|
|
108
|
+
throw new LodestoneError('aborted');
|
|
64
109
|
const quality = opts.quality ?? profile?.quality;
|
|
65
110
|
const order = async (models) => (quality && models.length > 1 ? rankModels(models, await cards(), quality, { now }) : models);
|
|
66
111
|
const cacheSlot = opts.cacheKey ? `lodestone:cache:${opts.cacheKey}` : null;
|
|
@@ -86,12 +131,39 @@ export function createLodestone(config = {}) {
|
|
|
86
131
|
let rateLimited = null;
|
|
87
132
|
let authFailed = false;
|
|
88
133
|
let needsCredit = false;
|
|
134
|
+
let spendBlocked = false;
|
|
135
|
+
const callerKey = Boolean(opts.apiKey);
|
|
136
|
+
/** Drops the models the breaker has down, saying so. */
|
|
137
|
+
const healthy = async (provider, list, phase) => {
|
|
138
|
+
if (!config.health)
|
|
139
|
+
return list;
|
|
140
|
+
const keep = [];
|
|
141
|
+
for (const m of list) {
|
|
142
|
+
if (await config.health.shouldSkip(provider.name, m))
|
|
143
|
+
config.onAttempt?.({ provider: provider.name, model: m, phase, outcome: 'skipped', ms: 0 });
|
|
144
|
+
else
|
|
145
|
+
keep.push(m);
|
|
146
|
+
}
|
|
147
|
+
return keep;
|
|
148
|
+
};
|
|
149
|
+
/** May the platform's money be spent on this attempt? A caller's own key is their money, not ours. */
|
|
150
|
+
const mayUse = async (provider, model, phase, free = false) => {
|
|
151
|
+
if (callerKey || free || !config.spend)
|
|
152
|
+
return true;
|
|
153
|
+
if (await config.spend.allow({ provider: provider.name, model, phase }))
|
|
154
|
+
return true;
|
|
155
|
+
spendBlocked = true;
|
|
156
|
+
config.onAttempt?.({ provider: provider.name, model, phase, outcome: 'skipped', ms: 0 });
|
|
157
|
+
return false;
|
|
158
|
+
};
|
|
89
159
|
const attempt = async (provider, apiKey, model, phase, ms) => {
|
|
90
160
|
attempted = true;
|
|
91
161
|
const at = now();
|
|
92
162
|
let out;
|
|
93
163
|
try {
|
|
94
|
-
const
|
|
164
|
+
const timeout = AbortSignal.timeout(ms);
|
|
165
|
+
const signal = opts.signal ? (typeof AbortSignal.any === 'function' ? AbortSignal.any([timeout, opts.signal]) : timeout) : timeout;
|
|
166
|
+
const reply = await provider.call({ model, system: opts.system, messages, maxTokens, temperature, apiKey, signal, ...(strict ? { privacy: 'strict' } : {}) });
|
|
95
167
|
if (!reply.text.trim())
|
|
96
168
|
out = { kind: 'fail', status: 502, body: 'empty answer', unusable: true };
|
|
97
169
|
else if (opts.accept && !opts.accept(reply.text))
|
|
@@ -101,23 +173,39 @@ export function createLodestone(config = {}) {
|
|
|
101
173
|
}
|
|
102
174
|
catch (e) {
|
|
103
175
|
const name = e.name;
|
|
104
|
-
if (
|
|
176
|
+
if (opts.signal?.aborted)
|
|
177
|
+
out = { kind: 'aborted' };
|
|
178
|
+
else if (name === 'TimeoutError' || name === 'AbortError')
|
|
105
179
|
out = { kind: 'timeout' };
|
|
106
180
|
else if (e instanceof ProviderError)
|
|
107
181
|
out = { kind: 'fail', status: e.status, body: e.body, message: messageFrom(e.body) };
|
|
108
182
|
else
|
|
109
183
|
out = { kind: 'fail', status: 599, body: e.message };
|
|
110
184
|
}
|
|
185
|
+
if (out.kind === 'aborted')
|
|
186
|
+
throw new LodestoneError('aborted'); // the person left: nothing more to try
|
|
111
187
|
const event = { provider: provider.name, model, phase, outcome: out.kind === 'ok' ? 'ok' : out.kind === 'timeout' ? 'timeout' : 'failed', status: out.kind === 'fail' ? out.status : undefined, ms: now() - at };
|
|
112
188
|
// A key or credit problem (401, 402, 403) says nothing about the model, so it is not counted against it.
|
|
113
|
-
|
|
189
|
+
const credential = out.kind === 'fail' && [401, 402, 403].includes(out.status);
|
|
190
|
+
// A call marked private can fail because few providers meet the rule: that says nothing about the model.
|
|
191
|
+
if (!credential && !strict) {
|
|
114
192
|
config.stats?.record(event);
|
|
193
|
+
await config.health?.record(provider.name, model, out.kind === 'ok');
|
|
194
|
+
}
|
|
115
195
|
config.onAttempt?.(event);
|
|
116
196
|
last = out;
|
|
117
197
|
return out;
|
|
118
198
|
};
|
|
119
|
-
const finish = async (provider, model, reply) => {
|
|
199
|
+
const finish = async (provider, model, reply, phase, free = false) => {
|
|
120
200
|
const result = { text: reply.text, provider: provider.name, model, usage: reply.usage };
|
|
201
|
+
if (phase && !callerKey && !free && config.spend) {
|
|
202
|
+
const costUsd = reply.usage?.costUsd ?? (await estimateCost(model, reply.usage));
|
|
203
|
+
// A ledger that fails must never fail an answer the person is waiting for.
|
|
204
|
+
try {
|
|
205
|
+
await config.spend.record({ provider: provider.name, model, phase, usage: reply.usage, costUsd });
|
|
206
|
+
}
|
|
207
|
+
catch { /* not counted */ }
|
|
208
|
+
}
|
|
121
209
|
if (cacheSlot)
|
|
122
210
|
await store.set(cacheSlot, { ...result, at: now() }).catch(() => { });
|
|
123
211
|
return result;
|
|
@@ -150,11 +238,25 @@ export function createLodestone(config = {}) {
|
|
|
150
238
|
throw new LodestoneError('slow');
|
|
151
239
|
throw classify(firstFail ?? (lastRoute?.kind === 'fail' ? lastRoute : null), Boolean(opts.apiKey), false, false);
|
|
152
240
|
}
|
|
153
|
-
//
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
241
|
+
// OpenRouter: the free models and the low-cost paid ones. Free first by default; paid first when the
|
|
242
|
+
// platform's key is used and the free models are not worth waiting for (see paidFirst).
|
|
243
|
+
const paidFirst = Boolean(config.openrouter?.paidFirst) && !callerKey && paidAvailable;
|
|
244
|
+
/** The free models. A withdrawn one has already been replaced by resolveModels. */
|
|
245
|
+
const runFree = async () => {
|
|
246
|
+
const all = await order(await resolvePool(tier, strict));
|
|
247
|
+
// Choose the live ones from the pool: a model that is down is replaced by the next best, so the
|
|
248
|
+
// list a request walks stays short however long the pool is.
|
|
249
|
+
const live = await healthy(router, all, 'free');
|
|
250
|
+
const routerLive = live.includes(ROUTER);
|
|
251
|
+
let models = [...live.filter((m) => m !== ROUTER).slice(0, ACTIVE - (routerLive ? 1 : 0)), ...(routerLive ? [ROUTER] : [])];
|
|
252
|
+
// With nothing after the free models to fall back on, a model that may be down is still better than no attempt.
|
|
253
|
+
if (models.length === 0 && !later && !paidFirst)
|
|
254
|
+
models = all.slice(0, ACTIVE);
|
|
255
|
+
// Behind a paid-first attempt the free models are a backup, and other providers may still be waiting their turn: an OpenRouter outage must not use up the whole budget on free models.
|
|
256
|
+
const freeStartedAt = now();
|
|
257
|
+
const leftHere = () => (paidFirst ? (fbs.length ? Math.min(left(), t.freePhaseLater - (now() - freeStartedAt)) : left()) : freeLeft());
|
|
258
|
+
for (let i = 0; i < models.length && leftHere() >= t.minSlice; i++) {
|
|
259
|
+
const slice = i >= models.length - 1 ? leftHere() : Math.min(later && !paidFirst ? t.freeSliceLater : t.freeSlice, leftHere());
|
|
158
260
|
const out = await attempt(router, orKey, models[i], 'free', slice);
|
|
159
261
|
if (out.kind === 'ok')
|
|
160
262
|
return finish(router, models[i], out.reply);
|
|
@@ -167,39 +269,78 @@ export function createLodestone(config = {}) {
|
|
|
167
269
|
rateLimited = out;
|
|
168
270
|
}
|
|
169
271
|
}
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
272
|
+
return null;
|
|
273
|
+
};
|
|
274
|
+
/** A low-cost paid model, which is not under the free-model limits. */
|
|
275
|
+
const runPaid = async () => {
|
|
276
|
+
const paidAll = await order(paidModels);
|
|
277
|
+
let paidList = await healthy(router, paidAll, 'paid');
|
|
278
|
+
if (paidList.length === 0)
|
|
279
|
+
paidList = paidAll; // never leave no paid attempt because of the breaker
|
|
280
|
+
for (let i = 0; i < paidList.length; i++) {
|
|
281
|
+
const m = paidList[i];
|
|
282
|
+
if (left() < t.minSlice)
|
|
283
|
+
break;
|
|
284
|
+
if (!(await mayUse(router, m, 'paid')))
|
|
285
|
+
continue;
|
|
286
|
+
// Going first, a paid model may not use all the time: the free models and other providers come after it and need some.
|
|
287
|
+
const ms = paidFirst ? Math.min(left(), t.paidSlice) : left();
|
|
288
|
+
const out = await attempt(router, orKey, m, 'paid', ms);
|
|
289
|
+
if (out.kind === 'ok')
|
|
290
|
+
return finish(router, m, out.reply, 'paid');
|
|
291
|
+
if (out.kind === 'fail') {
|
|
292
|
+
if (out.status === 402) {
|
|
293
|
+
needsCredit = true;
|
|
174
294
|
break;
|
|
175
|
-
const out = await attempt(router, orKey, m, 'paid', left());
|
|
176
|
-
if (out.kind === 'ok')
|
|
177
|
-
return finish(router, m, out.reply);
|
|
178
|
-
if (out.kind === 'fail') {
|
|
179
|
-
if (out.status === 402) {
|
|
180
|
-
needsCredit = true;
|
|
181
|
-
break;
|
|
182
|
-
}
|
|
183
|
-
if (out.status === 401)
|
|
184
|
-
break;
|
|
185
295
|
}
|
|
296
|
+
if (out.status === 401)
|
|
297
|
+
break;
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
return null;
|
|
301
|
+
};
|
|
302
|
+
if (orKey) {
|
|
303
|
+
if (paidFirst) {
|
|
304
|
+
const a = await runPaid();
|
|
305
|
+
if (a)
|
|
306
|
+
return a;
|
|
307
|
+
const b = authFailed ? null : await runFree();
|
|
308
|
+
if (b)
|
|
309
|
+
return b;
|
|
310
|
+
}
|
|
311
|
+
else {
|
|
312
|
+
const b = await runFree();
|
|
313
|
+
if (b)
|
|
314
|
+
return b;
|
|
315
|
+
if (paidAvailable && !authFailed) {
|
|
316
|
+
const a = await runPaid();
|
|
317
|
+
if (a)
|
|
318
|
+
return a;
|
|
186
319
|
}
|
|
187
320
|
}
|
|
188
321
|
}
|
|
189
322
|
// Phase 3: other providers, with their own keys.
|
|
190
323
|
for (const fb of fbs) {
|
|
191
|
-
|
|
324
|
+
const fbAll = await order(await resolveFallback(fb, tier, strict));
|
|
325
|
+
let fbList = await healthy(fb.provider, fbAll, 'fallback');
|
|
326
|
+
if (fbList.length === 0)
|
|
327
|
+
fbList = fbAll;
|
|
328
|
+
for (const m of fbList) {
|
|
192
329
|
if (left() < t.minSlice)
|
|
193
330
|
break;
|
|
331
|
+
if (!(await mayUse(fb.provider, m, 'fallback', fb.free)))
|
|
332
|
+
continue;
|
|
194
333
|
const out = await attempt(fb.provider, fb.apiKey, m, 'fallback', left());
|
|
195
334
|
if (out.kind === 'ok')
|
|
196
|
-
return finish(fb.provider, m, out.reply);
|
|
335
|
+
return finish(fb.provider, m, out.reply, 'fallback', fb.free);
|
|
197
336
|
if (out.kind === 'fail' && out.status === 401)
|
|
198
337
|
break; // this provider's key is wrong: skip its other models
|
|
199
338
|
}
|
|
200
339
|
}
|
|
201
340
|
// Nothing answered. Say why, using what the first useful failure told us.
|
|
202
341
|
const ownKey = Boolean(opts.apiKey);
|
|
342
|
+
if (spendBlocked)
|
|
343
|
+
throw new LodestoneError('budget');
|
|
203
344
|
if (!attempted)
|
|
204
345
|
throw new LodestoneError('not_configured');
|
|
205
346
|
const lastFail = last;
|
|
@@ -241,10 +382,38 @@ export function createLodestone(config = {}) {
|
|
|
241
382
|
if (!models)
|
|
242
383
|
return { ok: false, error: 'could not read the OpenRouter catalogue' };
|
|
243
384
|
const ids = new Set(models.map((m) => m.id));
|
|
385
|
+
const byId = new Map(models.map((m) => [m.id, m]));
|
|
244
386
|
const all = [...new Set([...Object.values(prefs).flat(), ...paidModels])].filter((m) => m !== ROUTER);
|
|
245
387
|
const withdrawn = all.filter((m) => !ids.has(m));
|
|
246
|
-
const
|
|
247
|
-
|
|
388
|
+
const horizon = (policy?.expiringWithinDays ?? 7) * 86_400_000;
|
|
389
|
+
const expiring = all.filter((m) => { const d = byId.get(m)?.expiration_date; return d !== undefined && Date.parse(d) - now() < horizon; });
|
|
390
|
+
const shortTiers = Object.keys(prefs).filter((tier) => pickModels(tier, models, prefs[tier], { policy, now: now() }).filter((m) => m !== ROUTER).length < 2);
|
|
391
|
+
// Every other provider that can list its models gets the same check: what has gone, and what is retiring.
|
|
392
|
+
const providers = {};
|
|
393
|
+
for (const [fb, cat] of fbCatalogs) {
|
|
394
|
+
const listed = await cat.refresh().catch(() => null);
|
|
395
|
+
if (!listed) {
|
|
396
|
+
providers[fb.provider.name] = { models: 0, withdrawn: [], expiring: [], error: 'could not read its model list' };
|
|
397
|
+
continue;
|
|
398
|
+
}
|
|
399
|
+
const have = new Map(listed.map((m) => [m.id, m]));
|
|
400
|
+
providers[fb.provider.name] = {
|
|
401
|
+
models: listed.length,
|
|
402
|
+
withdrawn: fb.models.filter((m) => !have.has(m)),
|
|
403
|
+
expiring: fb.models.filter((m) => { const d = have.get(m)?.expiration_date; return d !== undefined && Date.parse(d) - now() < horizon; }),
|
|
404
|
+
};
|
|
405
|
+
}
|
|
406
|
+
return { ok: shortTiers.length === 0, catalogSize: models.length, withdrawn, expiring, shortTiers, ...(Object.keys(providers).length ? { providers } : {}) };
|
|
407
|
+
}
|
|
408
|
+
/** An answer already saved under this key, without calling any model: so a caller can serve it without counting it as a request. */
|
|
409
|
+
async function peek(cacheKey, ttlMs = 24 * 3_600_000) {
|
|
410
|
+
try {
|
|
411
|
+
const hit = (await store.get(`lodestone:cache:${cacheKey}`));
|
|
412
|
+
return hit?.text && now() - hit.at < ttlMs ? { text: hit.text, provider: hit.provider, model: hit.model, cached: true } : null;
|
|
413
|
+
}
|
|
414
|
+
catch {
|
|
415
|
+
return null;
|
|
416
|
+
}
|
|
248
417
|
}
|
|
249
418
|
/** Every model this instance can call, for an evaluator to benchmark (see candidates.ts). */
|
|
250
419
|
async function candidates() {
|
|
@@ -268,5 +437,5 @@ export function createLodestone(config = {}) {
|
|
|
268
437
|
add(id, fb.provider.name, 'fallback');
|
|
269
438
|
return { schema: CANDIDATES_SCHEMA, source: 'lodestone', generatedAt: new Date(now()).toISOString(), models: [...byKey.values()] };
|
|
270
439
|
}
|
|
271
|
-
return { complete, json, resolveModels, refreshCatalog, checkModels, candidates };
|
|
440
|
+
return { complete, json, resolveModels, refreshCatalog, checkModels, candidates, peek };
|
|
272
441
|
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { CatalogModel } from '../catalog.js';
|
|
2
|
+
import type { Provider } from '../types.js';
|
|
3
|
+
/** Any service that speaks OpenAI's chat API: gateways (Requesty, LLM API), providers with free tiers
|
|
4
|
+
* (Groq, Cerebras), OpenAI and Mistral themselves, and local servers (Ollama). One adapter, with a
|
|
5
|
+
* preset for each service Lodestone has been checked against. Each preset also knows how to read that
|
|
6
|
+
* service's list of models, so Lodestone can drop a model that has gone and see its price and what it
|
|
7
|
+
* does with prompts. */
|
|
8
|
+
export interface OpenAICompatibleOptions {
|
|
9
|
+
name: string;
|
|
10
|
+
/** Up to and including /v1, with no trailing slash. */
|
|
11
|
+
baseUrl: string;
|
|
12
|
+
fetch?: typeof fetch;
|
|
13
|
+
vision?: boolean;
|
|
14
|
+
/** Extra headers (a referer, an organisation). */
|
|
15
|
+
headers?: Record<string, string>;
|
|
16
|
+
/** How much each model should think, as the service understands it; null to send nothing. */
|
|
17
|
+
reasoning?: (model: string) => Record<string, unknown> | null;
|
|
18
|
+
/** Reads one entry of the service's model list. Return null to skip it (not a chat model). Default: just the id. */
|
|
19
|
+
normalize?: (raw: Record<string, unknown>) => CatalogModel | null;
|
|
20
|
+
}
|
|
21
|
+
export declare function openaiCompatible(o: OpenAICompatibleOptions): Provider;
|
|
22
|
+
/** Requesty publishes, for every model, its price per token, the lab that made it, whether it is trained on and how long prompts are kept. */
|
|
23
|
+
export declare const normalizeRequesty: (r: Record<string, unknown>) => CatalogModel | null;
|
|
24
|
+
/** LLM API lists chat, image, video, speech and embedding models together: only "chat" is kept. */
|
|
25
|
+
export declare const normalizeLlmapi: (r: Record<string, unknown>) => CatalogModel | null;
|
|
26
|
+
export interface PresetOptions {
|
|
27
|
+
fetch?: typeof fetch;
|
|
28
|
+
headers?: Record<string, string>;
|
|
29
|
+
reasoning?: OpenAICompatibleOptions['reasoning'];
|
|
30
|
+
}
|
|
31
|
+
/** https://www.requesty.ai: a gateway to 800+ models, with failover between providers. `region: 'eu'` keeps prompts in the EU. */
|
|
32
|
+
export declare const requesty: (o?: PresetOptions & {
|
|
33
|
+
region?: "eu";
|
|
34
|
+
}) => Provider;
|
|
35
|
+
/** https://llmapi.ai: a gateway to 150+ chat models. */
|
|
36
|
+
export declare const llmapi: (o?: PresetOptions) => Provider;
|
|
37
|
+
/** https://groq.com: fast inference with a free developer tier. */
|
|
38
|
+
export declare const groq: (o?: PresetOptions) => Provider;
|
|
39
|
+
/** https://cerebras.ai: fast inference with a free tier. */
|
|
40
|
+
export declare const cerebras: (o?: PresetOptions) => Provider;
|
|
41
|
+
export declare const openai: (o?: PresetOptions) => Provider;
|
|
42
|
+
export declare const mistral: (o?: PresetOptions) => Provider;
|
|
43
|
+
export declare const together: (o?: PresetOptions) => Provider;
|
|
44
|
+
/** A local Ollama server: free, private, and nothing to go down. The key is ignored but must be non-empty. */
|
|
45
|
+
export declare const ollama: (o?: PresetOptions & {
|
|
46
|
+
baseUrl?: string;
|
|
47
|
+
}) => Provider;
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { ProviderError } from '../errors.js';
|
|
2
|
+
const part = (p) => (p.type === 'text' ? { type: 'text', text: p.text } : { type: 'image_url', image_url: { url: p.dataUrl } });
|
|
3
|
+
const wire = (m) => ({ role: m.role, content: typeof m.content === 'string' ? m.content : m.content.map(part) });
|
|
4
|
+
export function openaiCompatible(o) {
|
|
5
|
+
const f = o.fetch ?? fetch;
|
|
6
|
+
const norm = o.normalize ?? ((raw) => (typeof raw.id === 'string' ? { id: raw.id } : null));
|
|
7
|
+
return {
|
|
8
|
+
name: o.name,
|
|
9
|
+
vision: o.vision ?? true,
|
|
10
|
+
async call(req) {
|
|
11
|
+
const reasoning = o.reasoning?.(req.model);
|
|
12
|
+
const res = await f(`${o.baseUrl}/chat/completions`, {
|
|
13
|
+
method: 'POST',
|
|
14
|
+
headers: { Authorization: `Bearer ${req.apiKey}`, 'Content-Type': 'application/json', ...o.headers },
|
|
15
|
+
body: JSON.stringify({
|
|
16
|
+
model: req.model,
|
|
17
|
+
max_tokens: req.maxTokens,
|
|
18
|
+
...(req.temperature === undefined ? {} : { temperature: req.temperature }),
|
|
19
|
+
...(reasoning ? { reasoning } : {}),
|
|
20
|
+
messages: [...(req.system ? [{ role: 'system', content: req.system }] : []), ...req.messages.map(wire)],
|
|
21
|
+
}),
|
|
22
|
+
signal: req.signal,
|
|
23
|
+
});
|
|
24
|
+
if (!res.ok)
|
|
25
|
+
throw new ProviderError(res.status, await res.text());
|
|
26
|
+
const data = (await res.json());
|
|
27
|
+
return {
|
|
28
|
+
text: data.choices?.[0]?.message?.content ?? '',
|
|
29
|
+
usage: data.usage ? { inputTokens: data.usage.prompt_tokens ?? 0, outputTokens: data.usage.completion_tokens ?? 0, ...(typeof data.usage.cost === 'number' ? { costUsd: data.usage.cost } : {}) } : undefined,
|
|
30
|
+
};
|
|
31
|
+
},
|
|
32
|
+
async listModels(apiKey, signal) {
|
|
33
|
+
const res = await f(`${o.baseUrl}/models`, { headers: { Authorization: `Bearer ${apiKey}`, ...o.headers }, signal: signal ?? AbortSignal.timeout(10_000) });
|
|
34
|
+
if (!res.ok)
|
|
35
|
+
return null;
|
|
36
|
+
const body = (await res.json());
|
|
37
|
+
const rows = Array.isArray(body) ? body : body.data ?? [];
|
|
38
|
+
const models = rows.map(norm).filter((m) => m !== null);
|
|
39
|
+
return models.length ? models : null;
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : undefined);
|
|
44
|
+
const str = (v) => (typeof v === 'string' && v ? v : undefined);
|
|
45
|
+
/** Requesty publishes, for every model, its price per token, the lab that made it, whether it is trained on and how long prompts are kept. */
|
|
46
|
+
export const normalizeRequesty = (r) => {
|
|
47
|
+
if (typeof r.id !== 'string' || (r.api !== undefined && r.api !== 'chat'))
|
|
48
|
+
return null;
|
|
49
|
+
const inP = num(r.input_price);
|
|
50
|
+
const outP = num(r.output_price);
|
|
51
|
+
return {
|
|
52
|
+
id: r.id,
|
|
53
|
+
context_length: num(r.context_window),
|
|
54
|
+
architecture: { input_modalities: r.supports_vision ? ['text', 'image'] : ['text'], output_modalities: ['text'] },
|
|
55
|
+
...(inP !== undefined && outP !== undefined ? { pricing: { prompt: inP, completion: outP }, free: inP === 0 && outP === 0 } : {}),
|
|
56
|
+
...(str(r.model_lab) ? { publisher: str(r.model_lab).toLowerCase() } : {}),
|
|
57
|
+
...(typeof r.data_used_for_training === 'boolean' ? { data_used_for_training: r.data_used_for_training } : {}),
|
|
58
|
+
...(num(r.data_retention_days) !== undefined ? { data_retention_days: num(r.data_retention_days) } : {}),
|
|
59
|
+
};
|
|
60
|
+
};
|
|
61
|
+
/** LLM API gives a model "family" (anthropic, gemini-flash, glm...): turned into the lab where that is clear. */
|
|
62
|
+
const labOf = (family) => {
|
|
63
|
+
const f = family.toLowerCase();
|
|
64
|
+
return /gemini|gemma/.test(f) ? 'google' : /claude/.test(f) ? 'anthropic' : /^gpt|^o\d|openai/.test(f) ? 'openai' : /mistral|mixtral|codestral/.test(f) ? 'mistralai' : /llama/.test(f) ? 'meta-llama' : /grok/.test(f) ? 'x-ai' : f;
|
|
65
|
+
};
|
|
66
|
+
/** LLM API lists chat, image, video, speech and embedding models together: only "chat" is kept. */
|
|
67
|
+
export const normalizeLlmapi = (r) => {
|
|
68
|
+
if (typeof r.id !== 'string' || r.kind !== 'chat')
|
|
69
|
+
return null;
|
|
70
|
+
const p = (r.pricing ?? {});
|
|
71
|
+
return {
|
|
72
|
+
id: r.id,
|
|
73
|
+
context_length: num(r.context_length),
|
|
74
|
+
architecture: { input_modalities: ['text'], output_modalities: ['text'] },
|
|
75
|
+
...(str(p.prompt) !== undefined && str(p.completion) !== undefined ? { pricing: { prompt: str(p.prompt), completion: str(p.completion) } } : {}),
|
|
76
|
+
...(r.free === true ? { free: true } : {}),
|
|
77
|
+
...(str(r.family) ? { publisher: labOf(str(r.family)) } : {}),
|
|
78
|
+
};
|
|
79
|
+
};
|
|
80
|
+
/** https://www.requesty.ai: a gateway to 800+ models, with failover between providers. `region: 'eu'` keeps prompts in the EU. */
|
|
81
|
+
export const requesty = (o = {}) => openaiCompatible({ name: 'requesty', baseUrl: o.region === 'eu' ? 'https://router.eu.requesty.ai/v1' : 'https://router.requesty.ai/v1', normalize: normalizeRequesty, ...o });
|
|
82
|
+
/** https://llmapi.ai: a gateway to 150+ chat models. */
|
|
83
|
+
export const llmapi = (o = {}) => openaiCompatible({ name: 'llmapi', baseUrl: 'https://api.llmapi.ai/v1', normalize: normalizeLlmapi, ...o });
|
|
84
|
+
/** https://groq.com: fast inference with a free developer tier. */
|
|
85
|
+
export const groq = (o = {}) => openaiCompatible({ name: 'groq', baseUrl: 'https://api.groq.com/openai/v1', ...o });
|
|
86
|
+
/** https://cerebras.ai: fast inference with a free tier. */
|
|
87
|
+
export const cerebras = (o = {}) => openaiCompatible({ name: 'cerebras', baseUrl: 'https://api.cerebras.ai/v1', ...o });
|
|
88
|
+
export const openai = (o = {}) => openaiCompatible({ name: 'openai', baseUrl: 'https://api.openai.com/v1', ...o });
|
|
89
|
+
export const mistral = (o = {}) => openaiCompatible({ name: 'mistral', baseUrl: 'https://api.mistral.ai/v1', ...o });
|
|
90
|
+
export const together = (o = {}) => openaiCompatible({ name: 'together', baseUrl: 'https://api.together.xyz/v1', ...o });
|
|
91
|
+
/** A local Ollama server: free, private, and nothing to go down. The key is ignored but must be non-empty. */
|
|
92
|
+
export const ollama = (o = {}) => openaiCompatible({ name: 'ollama', baseUrl: o.baseUrl ?? 'http://localhost:11434/v1', vision: false, ...o });
|
|
@@ -3,6 +3,8 @@ export interface OpenRouterOptions {
|
|
|
3
3
|
fetch?: typeof fetch;
|
|
4
4
|
/** Ask the model not to think at length. Default true: most tasks need an answer, not a long chain of thought, and thinking is what made some free models slower than the time allowed. */
|
|
5
5
|
lowReasoning?: boolean;
|
|
6
|
+
/** How much each model should think, by model id: an OpenRouter `reasoning` setting, or null to send none. Overrides `lowReasoning`. Models differ: some will not run with reasoning off ("mandatory for this endpoint"), some think for thousands of tokens unless told not to, so one setting for every model is wrong. For example `(m) => m.startsWith('anthropic/claude-haiku') ? { enabled: false } : { effort: 'low', exclude: true }`. */
|
|
7
|
+
reasoning?: (model: string) => Record<string, unknown> | null;
|
|
6
8
|
/** Sent as HTTP-Referer and X-Title, which OpenRouter uses to credit apps. */
|
|
7
9
|
referer?: string;
|
|
8
10
|
title?: string;
|