@extrafemi-ai/lodestone 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/dist/candidates.d.ts +16 -0
- package/dist/candidates.js +4 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +3 -0
- package/dist/lodestone.d.ts +17 -0
- package/dist/lodestone.js +50 -6
- package/dist/scorecard.d.ts +68 -0
- package/dist/scorecard.js +96 -0
- package/dist/stats.d.ts +28 -0
- package/dist/stats.js +37 -0
- package/docs/scorecards.md +119 -0
- package/package.json +3 -2
package/README.md
CHANGED
|
@@ -60,6 +60,10 @@ A step with no key is skipped. A route skips the model lists and the catalogue,
|
|
|
60
60
|
|
|
61
61
|
Keep a customer's own key out of any fallback step: build the route from their key only.
|
|
62
62
|
|
|
63
|
+
### Your own key
|
|
64
|
+
|
|
65
|
+
Pass the customer's OpenRouter key as `apiKey` on a call. It replaces the platform's key, allows the paid fallback, and is never stored. The platform's `fallbacks` are **not** used on a call with a customer's own key (they carry the platform's money), unless a fallback sets `allowWithCallerKey`. Attempts that fail with 401, 402 or 403 are not counted in live stats, because they say nothing about the model.
|
|
66
|
+
|
|
63
67
|
### Phases
|
|
64
68
|
|
|
65
69
|
For each call, in order, until one answers:
|
|
@@ -83,6 +87,10 @@ const { ok, withdrawn, shortTiers } = await ai.checkModels();
|
|
|
83
87
|
|
|
84
88
|
`withdrawn` lists preferred models that have gone. That is harmless, they are already replaced. `ok` is false only when the catalogue can't be read or a tier is left with fewer than two real models. Without a daily run, a request that finds the saved catalogue over a day old refreshes it in the background.
|
|
85
89
|
|
|
90
|
+
### Letting measurements decide the order
|
|
91
|
+
|
|
92
|
+
A scorecard (a small JSON file of measured results) can put the models that did best on your kind of task first, and Lodestone can publish its own live success rates and speeds in the same format. [Naija Eval](https://naija-eval.extrafemi.com) publishes translation and speech quality for Hausa, Igbo and Yoruba this way. It is opt-in, per task, and only changes the order models are tried in. See [docs/scorecards.md](docs/scorecards.md).
|
|
93
|
+
|
|
86
94
|
### Caching
|
|
87
95
|
|
|
88
96
|
Pass `cacheKey` for a pure, repeatable task (for example a hash of a job page) and the answer is saved in the store and reused for `cacheTtlMs` (a day by default). Failures are never cached.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/** The models a tool would actually call, in a format an evaluator can read, so it
|
|
2
|
+
* benchmarks what is in use and not a guess. Naija Eval reads this to say which of
|
|
3
|
+
* them it has not measured yet. See docs/scorecards.md. */
|
|
4
|
+
export declare const CANDIDATES_SCHEMA = "model-candidates/1";
|
|
5
|
+
export interface Candidate {
|
|
6
|
+
id: string;
|
|
7
|
+
provider: string;
|
|
8
|
+
/** Where it is used: free:fast, free:strong, free:vision, paid, fallback. */
|
|
9
|
+
roles: string[];
|
|
10
|
+
}
|
|
11
|
+
export interface Candidates {
|
|
12
|
+
schema: typeof CANDIDATES_SCHEMA;
|
|
13
|
+
source: string;
|
|
14
|
+
generatedAt: string;
|
|
15
|
+
models: Candidate[];
|
|
16
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/** The models a tool would actually call, in a format an evaluator can read, so it
|
|
2
|
+
* benchmarks what is in use and not a guess. Naija Eval reads this to say which of
|
|
3
|
+
* them it has not measured yet. See docs/scorecards.md. */
|
|
4
|
+
export const CANDIDATES_SCHEMA = 'model-candidates/1';
|
package/dist/index.d.ts
CHANGED
|
@@ -16,3 +16,9 @@ export { anthropic } from './providers/anthropic.js';
|
|
|
16
16
|
export { google } from './providers/google.js';
|
|
17
17
|
export { mock } from './providers/mock.js';
|
|
18
18
|
export type { Tier, Message, ContentPart, Provider, ProviderRequest, ProviderReply, KeyValueStore, Usage } from './types.js';
|
|
19
|
+
export { SCORECARD_SCHEMA, ScorecardError, parseScorecard, fetchScorecard, rankModels, scoreFor, baseModelId } from './scorecard.js';
|
|
20
|
+
export type { Scorecard, ModelScore, QualityQuery, RankOptions } from './scorecard.js';
|
|
21
|
+
export { createStats } from './stats.js';
|
|
22
|
+
export type { Stats, StatsOptions, StatsState } from './stats.js';
|
|
23
|
+
export { CANDIDATES_SCHEMA } from './candidates.js';
|
|
24
|
+
export type { Candidates, Candidate } from './candidates.js';
|
package/dist/index.js
CHANGED
|
@@ -10,3 +10,6 @@ export { openrouter } from './providers/openrouter.js';
|
|
|
10
10
|
export { anthropic } from './providers/anthropic.js';
|
|
11
11
|
export { google } from './providers/google.js';
|
|
12
12
|
export { mock } from './providers/mock.js';
|
|
13
|
+
export { SCORECARD_SCHEMA, ScorecardError, parseScorecard, fetchScorecard, rankModels, scoreFor, baseModelId } from './scorecard.js';
|
|
14
|
+
export { createStats } from './stats.js';
|
|
15
|
+
export { CANDIDATES_SCHEMA } from './candidates.js';
|
package/dist/lodestone.d.ts
CHANGED
|
@@ -1,15 +1,22 @@
|
|
|
1
1
|
import { type CatalogModel } from './catalog.js';
|
|
2
|
+
import { type Candidates } from './candidates.js';
|
|
3
|
+
import { type QualityQuery, type Scorecard } from './scorecard.js';
|
|
4
|
+
import type { Stats } from './stats.js';
|
|
2
5
|
import type { KeyValueStore, Message, Provider, Tier, Usage } from './types.js';
|
|
3
6
|
export interface Profile {
|
|
4
7
|
tier: Tier;
|
|
5
8
|
maxTokens?: number;
|
|
6
9
|
temperature?: number;
|
|
10
|
+
/** Put the models a scorecard measured best on this first. See `scorecards`. */
|
|
11
|
+
quality?: QualityQuery;
|
|
7
12
|
}
|
|
8
13
|
export interface FallbackConfig {
|
|
9
14
|
provider: Provider;
|
|
10
15
|
apiKey: string;
|
|
11
16
|
/** Tried in order. */
|
|
12
17
|
models: string[];
|
|
18
|
+
/** Also use this provider when the caller supplied their own OpenRouter key. Default false: a fallback has the platform's key, and a caller's own key must never lead to the platform's money being spent. */
|
|
19
|
+
allowWithCallerKey?: boolean;
|
|
13
20
|
}
|
|
14
21
|
export interface AttemptEvent {
|
|
15
22
|
provider: string;
|
|
@@ -34,6 +41,13 @@ export interface LodestoneConfig {
|
|
|
34
41
|
providers?: Record<string, Provider>;
|
|
35
42
|
/** Other providers, each with its own key, tried after OpenRouter has failed. */
|
|
36
43
|
fallbacks?: FallbackConfig[];
|
|
44
|
+
/** Measured results about models (see scorecard.ts). With a `quality` query on the task
|
|
45
|
+
* or the call, the free, paid and fallback models are put in order of their score. Models
|
|
46
|
+
* nobody has measured keep their place. A function is called per request, so it can fetch
|
|
47
|
+
* and cache; if it fails or takes over 1.5 s the order is left alone. */
|
|
48
|
+
scorecards?: Scorecard[] | (() => Promise<Scorecard[]>);
|
|
49
|
+
/** Records every attempt so you can publish how each model does (stats.scorecard()). */
|
|
50
|
+
stats?: Stats;
|
|
37
51
|
/** Named tasks, so calls say what they are doing and not which model to use. */
|
|
38
52
|
profiles?: Record<string, Profile>;
|
|
39
53
|
/** Where the model catalogue and cached answers are kept. Default: memory. */
|
|
@@ -66,6 +80,8 @@ export interface CompleteOptions {
|
|
|
66
80
|
route?: RouteStep[];
|
|
67
81
|
/** A key in `profiles`. */
|
|
68
82
|
task?: string;
|
|
83
|
+
/** Overrides the profile's quality query. */
|
|
84
|
+
quality?: QualityQuery;
|
|
69
85
|
/** Overrides the profile's tier. Default: 'vision' if the messages contain an image, else 'fast'. */
|
|
70
86
|
tier?: Tier;
|
|
71
87
|
prompt?: string;
|
|
@@ -104,5 +120,6 @@ export declare function createLodestone(config?: LodestoneConfig): {
|
|
|
104
120
|
withdrawn?: string[];
|
|
105
121
|
shortTiers?: Tier[];
|
|
106
122
|
}>;
|
|
123
|
+
candidates: () => Promise<Candidates>;
|
|
107
124
|
};
|
|
108
125
|
export type Lodestone = ReturnType<typeof createLodestone>;
|
package/dist/lodestone.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createCatalog, pickModels, ROUTER } from './catalog.js';
|
|
2
|
+
import { CANDIDATES_SCHEMA } from './candidates.js';
|
|
2
3
|
import { LodestoneError, messageFrom, ProviderError } from './errors.js';
|
|
3
4
|
import { tryParseJson } from './json.js';
|
|
4
5
|
import { DEFAULT_MODELS, DEFAULT_PAID_MODELS } from './models.js';
|
|
@@ -6,6 +7,7 @@ import { readRateLimit } from './rateLimit.js';
|
|
|
6
7
|
import { anthropic } from './providers/anthropic.js';
|
|
7
8
|
import { google } from './providers/google.js';
|
|
8
9
|
import { openrouter } from './providers/openrouter.js';
|
|
10
|
+
import { rankModels } from './scorecard.js';
|
|
9
11
|
import { memoryStore } from './store.js';
|
|
10
12
|
const hasImage = (messages) => messages.some((m) => Array.isArray(m.content) && m.content.some((p) => p.type === 'image'));
|
|
11
13
|
export function createLodestone(config = {}) {
|
|
@@ -34,6 +36,20 @@ export function createLodestone(config = {}) {
|
|
|
34
36
|
return prefs[tier];
|
|
35
37
|
}
|
|
36
38
|
}
|
|
39
|
+
/** The scorecards, or none: never throws and never holds a request up for long. */
|
|
40
|
+
async function cards() {
|
|
41
|
+
const src = config.scorecards;
|
|
42
|
+
if (!src)
|
|
43
|
+
return [];
|
|
44
|
+
if (Array.isArray(src))
|
|
45
|
+
return src;
|
|
46
|
+
try {
|
|
47
|
+
return await Promise.race([src(), new Promise((resolve) => setTimeout(() => resolve([]), 1_500).unref?.())]);
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return [];
|
|
51
|
+
}
|
|
52
|
+
}
|
|
37
53
|
async function complete(opts) {
|
|
38
54
|
const messages = opts.messages ?? (opts.prompt !== undefined ? [{ role: 'user', content: opts.prompt }] : []);
|
|
39
55
|
if (messages.length === 0)
|
|
@@ -45,6 +61,8 @@ export function createLodestone(config = {}) {
|
|
|
45
61
|
const tier = opts.tier ?? profile?.tier ?? (images ? 'vision' : 'fast');
|
|
46
62
|
const maxTokens = opts.maxTokens ?? profile?.maxTokens ?? 3500;
|
|
47
63
|
const temperature = opts.temperature ?? profile?.temperature ?? (opts.route ? undefined : 0.2);
|
|
64
|
+
const quality = opts.quality ?? profile?.quality;
|
|
65
|
+
const order = async (models) => (quality && models.length > 1 ? rankModels(models, await cards(), quality, { now }) : models);
|
|
48
66
|
const cacheSlot = opts.cacheKey ? `lodestone:cache:${opts.cacheKey}` : null;
|
|
49
67
|
if (cacheSlot) {
|
|
50
68
|
try {
|
|
@@ -58,7 +76,7 @@ export function createLodestone(config = {}) {
|
|
|
58
76
|
const left = () => budget - (now() - started);
|
|
59
77
|
const orKey = opts.apiKey || config.openrouter?.apiKey;
|
|
60
78
|
const paidAllowed = Boolean(opts.apiKey) || Boolean(config.openrouter?.allowPaid);
|
|
61
|
-
const fbs = fallbacks.filter((f) => f.apiKey && (!images || f.provider.vision));
|
|
79
|
+
const fbs = fallbacks.filter((f) => f.apiKey && (!images || f.provider.vision) && (!opts.apiKey || f.allowWithCallerKey));
|
|
62
80
|
const paidAvailable = Boolean(orKey) && paidAllowed && paidModels.length > 0;
|
|
63
81
|
const later = paidAvailable || fbs.length > 0;
|
|
64
82
|
const freeEnds = later ? t.freePhaseLater : budget;
|
|
@@ -90,7 +108,11 @@ export function createLodestone(config = {}) {
|
|
|
90
108
|
else
|
|
91
109
|
out = { kind: 'fail', status: 599, body: e.message };
|
|
92
110
|
}
|
|
93
|
-
|
|
111
|
+
const event = { provider: provider.name, model, phase, outcome: out.kind === 'ok' ? 'ok' : out.kind === 'timeout' ? 'timeout' : 'failed', status: out.kind === 'fail' ? out.status : undefined, ms: now() - at };
|
|
112
|
+
// A key or credit problem (401, 402, 403) says nothing about the model, so it is not counted against it.
|
|
113
|
+
if (!(out.kind === 'fail' && [401, 402, 403].includes(out.status)))
|
|
114
|
+
config.stats?.record(event);
|
|
115
|
+
config.onAttempt?.(event);
|
|
94
116
|
last = out;
|
|
95
117
|
return out;
|
|
96
118
|
};
|
|
@@ -130,7 +152,7 @@ export function createLodestone(config = {}) {
|
|
|
130
152
|
}
|
|
131
153
|
// Phase 1: OpenRouter's free models. A withdrawn one has already been replaced by resolveModels.
|
|
132
154
|
if (orKey) {
|
|
133
|
-
const models = await resolveModels(tier);
|
|
155
|
+
const models = await order(await resolveModels(tier));
|
|
134
156
|
for (let i = 0; i < models.length && freeLeft() >= t.minSlice; i++) {
|
|
135
157
|
const slice = i >= models.length - 1 ? freeLeft() : Math.min(later ? t.freeSliceLater : t.freeSlice, freeLeft());
|
|
136
158
|
const out = await attempt(router, orKey, models[i], 'free', slice);
|
|
@@ -147,7 +169,7 @@ export function createLodestone(config = {}) {
|
|
|
147
169
|
}
|
|
148
170
|
// Phase 2: a low-cost paid model, which is not under the free-model limits.
|
|
149
171
|
if (paidAvailable && !authFailed) {
|
|
150
|
-
for (const m of paidModels) {
|
|
172
|
+
for (const m of await order(paidModels)) {
|
|
151
173
|
if (left() < t.minSlice)
|
|
152
174
|
break;
|
|
153
175
|
const out = await attempt(router, orKey, m, 'paid', left());
|
|
@@ -166,7 +188,7 @@ export function createLodestone(config = {}) {
|
|
|
166
188
|
}
|
|
167
189
|
// Phase 3: other providers, with their own keys.
|
|
168
190
|
for (const fb of fbs) {
|
|
169
|
-
for (const m of fb.models) {
|
|
191
|
+
for (const m of await order(fb.models)) {
|
|
170
192
|
if (left() < t.minSlice)
|
|
171
193
|
break;
|
|
172
194
|
const out = await attempt(fb.provider, fb.apiKey, m, 'fallback', left());
|
|
@@ -224,5 +246,27 @@ export function createLodestone(config = {}) {
|
|
|
224
246
|
const shortTiers = Object.keys(prefs).filter((tier) => pickModels(tier, models, prefs[tier]).filter((m) => m !== ROUTER).length < 2);
|
|
225
247
|
return { ok: shortTiers.length === 0, catalogSize: models.length, withdrawn, shortTiers };
|
|
226
248
|
}
|
|
227
|
-
|
|
249
|
+
/** Every model this instance can call, for an evaluator to benchmark (see candidates.ts). */
|
|
250
|
+
async function candidates() {
|
|
251
|
+
const byKey = new Map();
|
|
252
|
+
const add = (id, provider, role) => {
|
|
253
|
+
if (id === ROUTER)
|
|
254
|
+
return;
|
|
255
|
+
const k = `${provider}\u0000${id}`;
|
|
256
|
+
const c = byKey.get(k) ?? { id, provider, roles: [] };
|
|
257
|
+
if (!c.roles.includes(role))
|
|
258
|
+
c.roles.push(role);
|
|
259
|
+
byKey.set(k, c);
|
|
260
|
+
};
|
|
261
|
+
for (const tier of Object.keys(prefs))
|
|
262
|
+
for (const id of await resolveModels(tier))
|
|
263
|
+
add(id, 'openrouter', `free:${tier}`);
|
|
264
|
+
for (const id of paidModels)
|
|
265
|
+
add(id, 'openrouter', 'paid');
|
|
266
|
+
for (const fb of fallbacks)
|
|
267
|
+
for (const id of fb.models)
|
|
268
|
+
add(id, fb.provider.name, 'fallback');
|
|
269
|
+
return { schema: CANDIDATES_SCHEMA, source: 'lodestone', generatedAt: new Date(now()).toISOString(), models: [...byKey.values()] };
|
|
270
|
+
}
|
|
271
|
+
return { complete, json, resolveModels, refreshCatalog, checkModels, candidates };
|
|
228
272
|
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/** A model scorecard: measured results about models, in one format that anyone can
|
|
2
|
+
* produce and Lodestone can read. Naija Eval publishes one (translation and speech
|
|
3
|
+
* quality for Hausa, Igbo and Yoruba); Lodestone produces one of its own from live
|
|
4
|
+
* traffic (how often each model answered, and how fast). Lodestone uses a scorecard
|
|
5
|
+
* to put better models first, for tasks the scorecard measured. See docs/scorecards.md. */
|
|
6
|
+
export declare const SCORECARD_SCHEMA = "model-scorecard/1";
|
|
7
|
+
export interface ModelScore {
|
|
8
|
+
/** An OpenRouter model id (for example anthropic/claude-sonnet-5.5), or local/<name> for a model that is not routable. */
|
|
9
|
+
model: string;
|
|
10
|
+
/** What was measured: translate, transcribe, availability, latency... */
|
|
11
|
+
task: string;
|
|
12
|
+
/** How it was scored: chrf++, wer, success_rate, latency_p50_ms... */
|
|
13
|
+
metric: string;
|
|
14
|
+
value: number;
|
|
15
|
+
higherIsBetter: boolean;
|
|
16
|
+
/** How many items the score is based on. */
|
|
17
|
+
n: number;
|
|
18
|
+
/** The day it was measured, ISO (2026-10-08). */
|
|
19
|
+
measuredAt: string;
|
|
20
|
+
/** ISO 639-1 code of the language measured, when the task is about one. */
|
|
21
|
+
language?: string;
|
|
22
|
+
/** A finer cut, for example en->yo, or the provider for live stats. */
|
|
23
|
+
slice?: string;
|
|
24
|
+
}
|
|
25
|
+
export interface Scorecard {
|
|
26
|
+
schema: typeof SCORECARD_SCHEMA;
|
|
27
|
+
/** Who measured it: naija-eval, lodestone-live... */
|
|
28
|
+
source: string;
|
|
29
|
+
generatedAt: string;
|
|
30
|
+
scores: ModelScore[];
|
|
31
|
+
}
|
|
32
|
+
/** Which scores to look at when ordering models for a task. */
|
|
33
|
+
export interface QualityQuery {
|
|
34
|
+
task: string;
|
|
35
|
+
metric: string;
|
|
36
|
+
language?: string;
|
|
37
|
+
slice?: string;
|
|
38
|
+
}
|
|
39
|
+
export interface RankOptions {
|
|
40
|
+
/** Ignore scores based on fewer items than this. Default 30. */
|
|
41
|
+
minN?: number;
|
|
42
|
+
/** Ignore scores older than this. Default 365. */
|
|
43
|
+
maxAgeDays?: number;
|
|
44
|
+
now?: () => number;
|
|
45
|
+
}
|
|
46
|
+
export declare class ScorecardError extends Error {
|
|
47
|
+
problems: string[];
|
|
48
|
+
constructor(problems: string[]);
|
|
49
|
+
}
|
|
50
|
+
/** Checks a scorecard from outside (a file, a URL) and returns it typed. Throws a
|
|
51
|
+
* ScorecardError listing what is wrong. A scorecard with a schema version this
|
|
52
|
+
* library doesn't know is refused, not guessed at. */
|
|
53
|
+
export declare function parseScorecard(raw: unknown): Scorecard;
|
|
54
|
+
/** Fetches and checks a scorecard from a URL. Throws if it can't be read or isn't valid. */
|
|
55
|
+
export declare function fetchScorecard(url: string, opts?: {
|
|
56
|
+
fetch?: typeof fetch;
|
|
57
|
+
timeoutMs?: number;
|
|
58
|
+
}): Promise<Scorecard>;
|
|
59
|
+
/** An id without its variant suffix: anthropic/claude-x:free is the same model as anthropic/claude-x. */
|
|
60
|
+
export declare const baseModelId: (id: string) => string;
|
|
61
|
+
/** The score for a model on a query, or undefined if nobody measured it well enough.
|
|
62
|
+
* Where several scorecards measured the same thing, the one based on most items wins. */
|
|
63
|
+
export declare function scoreFor(cards: Scorecard[], model: string, q: QualityQuery, opts?: RankOptions): ModelScore | undefined;
|
|
64
|
+
/** Puts measured models in order of their score, without moving anything else. The
|
|
65
|
+
* measured models take turns in the slots they already occupy; a model nobody has
|
|
66
|
+
* measured stays exactly where it was, so a good unmeasured model is never pushed
|
|
67
|
+
* down by a weaker measured one. Nothing is added or dropped. */
|
|
68
|
+
export declare function rankModels(models: string[], cards: Scorecard[], q: QualityQuery, opts?: RankOptions): string[];
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/** A model scorecard: measured results about models, in one format that anyone can
|
|
2
|
+
* produce and Lodestone can read. Naija Eval publishes one (translation and speech
|
|
3
|
+
* quality for Hausa, Igbo and Yoruba); Lodestone produces one of its own from live
|
|
4
|
+
* traffic (how often each model answered, and how fast). Lodestone uses a scorecard
|
|
5
|
+
* to put better models first, for tasks the scorecard measured. See docs/scorecards.md. */
|
|
6
|
+
export const SCORECARD_SCHEMA = 'model-scorecard/1';
|
|
7
|
+
export class ScorecardError extends Error {
|
|
8
|
+
problems;
|
|
9
|
+
constructor(problems) {
|
|
10
|
+
super(`not a valid model scorecard: ${problems.slice(0, 3).join('; ')}${problems.length > 3 ? `; and ${problems.length - 3} more` : ''}`);
|
|
11
|
+
this.problems = problems;
|
|
12
|
+
this.name = 'ScorecardError';
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
const isStr = (v) => typeof v === 'string' && v.length > 0;
|
|
16
|
+
const isNum = (v) => typeof v === 'number' && Number.isFinite(v);
|
|
17
|
+
const isDay = (v) => typeof v === 'string' && /^\d{4}-\d{2}-\d{2}/.test(v) && !Number.isNaN(Date.parse(v));
|
|
18
|
+
/** Checks a scorecard from outside (a file, a URL) and returns it typed. Throws a
|
|
19
|
+
* ScorecardError listing what is wrong. A scorecard with a schema version this
|
|
20
|
+
* library doesn't know is refused, not guessed at. */
|
|
21
|
+
export function parseScorecard(raw) {
|
|
22
|
+
const problems = [];
|
|
23
|
+
const c = raw;
|
|
24
|
+
if (!c || typeof c !== 'object')
|
|
25
|
+
throw new ScorecardError(['not an object']);
|
|
26
|
+
if (c.schema !== SCORECARD_SCHEMA)
|
|
27
|
+
problems.push(`schema is ${JSON.stringify(c.schema)}, expected ${SCORECARD_SCHEMA}`);
|
|
28
|
+
if (!isStr(c.source))
|
|
29
|
+
problems.push('source is missing');
|
|
30
|
+
if (!isDay(c.generatedAt))
|
|
31
|
+
problems.push('generatedAt is missing or not a date');
|
|
32
|
+
if (!Array.isArray(c.scores))
|
|
33
|
+
problems.push('scores is not a list');
|
|
34
|
+
else
|
|
35
|
+
c.scores.forEach((s, i) => {
|
|
36
|
+
const at = `scores[${i}]`;
|
|
37
|
+
if (!isStr(s?.model))
|
|
38
|
+
problems.push(`${at}.model is missing`);
|
|
39
|
+
if (!isStr(s?.task))
|
|
40
|
+
problems.push(`${at}.task is missing`);
|
|
41
|
+
if (!isStr(s?.metric))
|
|
42
|
+
problems.push(`${at}.metric is missing`);
|
|
43
|
+
if (!isNum(s?.value))
|
|
44
|
+
problems.push(`${at}.value is not a number`);
|
|
45
|
+
if (typeof s?.higherIsBetter !== 'boolean')
|
|
46
|
+
problems.push(`${at}.higherIsBetter is not true or false`);
|
|
47
|
+
if (!isNum(s?.n) || s.n < 0)
|
|
48
|
+
problems.push(`${at}.n is not a count`);
|
|
49
|
+
if (!isDay(s?.measuredAt))
|
|
50
|
+
problems.push(`${at}.measuredAt is not a date`);
|
|
51
|
+
});
|
|
52
|
+
if (problems.length)
|
|
53
|
+
throw new ScorecardError(problems);
|
|
54
|
+
return c;
|
|
55
|
+
}
|
|
56
|
+
/** Fetches and checks a scorecard from a URL. Throws if it can't be read or isn't valid. */
|
|
57
|
+
export async function fetchScorecard(url, opts = {}) {
|
|
58
|
+
const res = await (opts.fetch ?? fetch)(url, { signal: AbortSignal.timeout(opts.timeoutMs ?? 8_000) });
|
|
59
|
+
if (!res.ok)
|
|
60
|
+
throw new Error(`scorecard ${url}: ${res.status}`);
|
|
61
|
+
return parseScorecard(await res.json());
|
|
62
|
+
}
|
|
63
|
+
/** An id without its variant suffix: anthropic/claude-x:free is the same model as anthropic/claude-x. */
|
|
64
|
+
export const baseModelId = (id) => id.split(':')[0].toLowerCase();
|
|
65
|
+
function matches(s, q) {
|
|
66
|
+
return s.task === q.task && s.metric === q.metric && (q.language === undefined || s.language === q.language) && (q.slice === undefined || s.slice === q.slice);
|
|
67
|
+
}
|
|
68
|
+
/** The score for a model on a query, or undefined if nobody measured it well enough.
|
|
69
|
+
* Where several scorecards measured the same thing, the one based on most items wins. */
|
|
70
|
+
export function scoreFor(cards, model, q, opts = {}) {
|
|
71
|
+
const minN = opts.minN ?? 30;
|
|
72
|
+
const oldest = (opts.now ?? Date.now)() - (opts.maxAgeDays ?? 365) * 86_400_000;
|
|
73
|
+
const id = baseModelId(model);
|
|
74
|
+
let best;
|
|
75
|
+
for (const card of cards)
|
|
76
|
+
for (const s of card.scores) {
|
|
77
|
+
if (baseModelId(s.model) !== id || !matches(s, q) || s.n < minN || Date.parse(s.measuredAt) < oldest)
|
|
78
|
+
continue;
|
|
79
|
+
if (!best || s.n > best.n || (s.n === best.n && s.measuredAt > best.measuredAt))
|
|
80
|
+
best = s;
|
|
81
|
+
}
|
|
82
|
+
return best;
|
|
83
|
+
}
|
|
84
|
+
/** Puts measured models in order of their score, without moving anything else. The
|
|
85
|
+
* measured models take turns in the slots they already occupy; a model nobody has
|
|
86
|
+
* measured stays exactly where it was, so a good unmeasured model is never pushed
|
|
87
|
+
* down by a weaker measured one. Nothing is added or dropped. */
|
|
88
|
+
export function rankModels(models, cards, q, opts = {}) {
|
|
89
|
+
const measured = models.map((m, i) => ({ m, i, s: scoreFor(cards, m, q, opts) })).filter((x) => x.s !== undefined);
|
|
90
|
+
if (measured.length < 2)
|
|
91
|
+
return models;
|
|
92
|
+
const sorted = [...measured].sort((a, b) => (a.s.higherIsBetter ? b.s.value - a.s.value : a.s.value - b.s.value) || a.i - b.i);
|
|
93
|
+
const out = [...models];
|
|
94
|
+
measured.forEach((slot, k) => { out[slot.i] = sorted[k].m; });
|
|
95
|
+
return out;
|
|
96
|
+
}
|
package/dist/stats.d.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { AttemptEvent } from './lodestone.js';
|
|
2
|
+
import { type Scorecard } from './scorecard.js';
|
|
3
|
+
/** Records how each model does in live traffic and reports it as a scorecard: how often
|
|
4
|
+
* it answered, and how fast. This is the other half of Naija Eval's measurements: that
|
|
5
|
+
* says how good an answer is, this says whether you get one at all. Keeps the last
|
|
6
|
+
* `window` attempts per model, in memory. Save `toJSON()` in your store if you want it
|
|
7
|
+
* to survive restarts. Never sees prompts or answers. */
|
|
8
|
+
export interface StatsOptions {
|
|
9
|
+
/** Attempts kept per model and provider. Default 200. */
|
|
10
|
+
window?: number;
|
|
11
|
+
/** The `source` of the scorecard it produces. Default lodestone-live. */
|
|
12
|
+
source?: string;
|
|
13
|
+
now?: () => number;
|
|
14
|
+
}
|
|
15
|
+
interface Sample {
|
|
16
|
+
ok: boolean;
|
|
17
|
+
ms: number;
|
|
18
|
+
at: number;
|
|
19
|
+
}
|
|
20
|
+
export type StatsState = Record<string, Sample[]>;
|
|
21
|
+
export interface Stats {
|
|
22
|
+
record(e: AttemptEvent): void;
|
|
23
|
+
scorecard(): Scorecard;
|
|
24
|
+
toJSON(): StatsState;
|
|
25
|
+
load(state: StatsState): void;
|
|
26
|
+
}
|
|
27
|
+
export declare function createStats(opts?: StatsOptions): Stats;
|
|
28
|
+
export {};
|
package/dist/stats.js
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { SCORECARD_SCHEMA } from './scorecard.js';
|
|
2
|
+
const percentile = (sorted, p) => sorted[Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1)];
|
|
3
|
+
const key = (provider, model) => `${provider}\u0000${model}`;
|
|
4
|
+
const day = (ms) => new Date(ms).toISOString().slice(0, 10);
|
|
5
|
+
export function createStats(opts = {}) {
|
|
6
|
+
const window = opts.window ?? 200;
|
|
7
|
+
const now = opts.now ?? Date.now;
|
|
8
|
+
let state = {};
|
|
9
|
+
return {
|
|
10
|
+
record(e) {
|
|
11
|
+
const k = key(e.provider, e.model);
|
|
12
|
+
const list = (state[k] ??= []);
|
|
13
|
+
list.push({ ok: e.outcome === 'ok', ms: e.ms, at: now() });
|
|
14
|
+
if (list.length > window)
|
|
15
|
+
list.splice(0, list.length - window);
|
|
16
|
+
},
|
|
17
|
+
toJSON: () => state,
|
|
18
|
+
load(s) { state = s; },
|
|
19
|
+
scorecard() {
|
|
20
|
+
const scores = [];
|
|
21
|
+
for (const [k, list] of Object.entries(state)) {
|
|
22
|
+
if (!list.length)
|
|
23
|
+
continue;
|
|
24
|
+
const [provider, model] = k.split('\u0000');
|
|
25
|
+
const measuredAt = day(Math.max(...list.map((s) => s.at)));
|
|
26
|
+
const base = { model, n: list.length, measuredAt, slice: provider };
|
|
27
|
+
scores.push({ ...base, task: 'availability', metric: 'success_rate', value: Math.round((list.filter((s) => s.ok).length / list.length) * 1000) / 10, higherIsBetter: true });
|
|
28
|
+
const times = list.filter((s) => s.ok).map((s) => s.ms).sort((a, b) => a - b);
|
|
29
|
+
if (times.length) {
|
|
30
|
+
scores.push({ ...base, n: times.length, task: 'latency', metric: 'latency_p50_ms', value: percentile(times, 50), higherIsBetter: false });
|
|
31
|
+
scores.push({ ...base, n: times.length, task: 'latency', metric: 'latency_p95_ms', value: percentile(times, 95), higherIsBetter: false });
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return { schema: SCORECARD_SCHEMA, source: opts.source ?? 'lodestone-live', generatedAt: new Date(now()).toISOString(), scores };
|
|
35
|
+
},
|
|
36
|
+
};
|
|
37
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# Scorecards and candidates
|
|
2
|
+
|
|
3
|
+
Two small JSON formats let a model evaluator and Lodestone feed each other without knowing about each other's code. [Naija Eval](https://naija-eval.extrafemi.com) is the first evaluator to use them.
|
|
4
|
+
|
|
5
|
+
```
|
|
6
|
+
evaluator (Naija Eval) Lodestone (in your tool)
|
|
7
|
+
────────────────────── ────────────────────────
|
|
8
|
+
measures answer quality ── model-scorecard ──▶ orders models by it, per task
|
|
9
|
+
│
|
|
10
|
+
benchmarks what is in use ◀─ model-candidates ── lists the models it can call
|
|
11
|
+
│
|
|
12
|
+
(reads live reliability) ◀── model-scorecard ─── records success rate and speed
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Nothing here is automatic or hidden. A scorecard changes the order models are *tried* in, for tasks you name, and nothing else. A model nobody measured is never dropped, added or moved.
|
|
16
|
+
|
|
17
|
+
## `model-scorecard/1`
|
|
18
|
+
|
|
19
|
+
Measured results about models.
|
|
20
|
+
|
|
21
|
+
```json
|
|
22
|
+
{
|
|
23
|
+
"schema": "model-scorecard/1",
|
|
24
|
+
"source": "naija-eval",
|
|
25
|
+
"generatedAt": "2026-10-28T09:00:00Z",
|
|
26
|
+
"scores": [
|
|
27
|
+
{
|
|
28
|
+
"model": "anthropic/claude-sonnet-5.5",
|
|
29
|
+
"task": "translate",
|
|
30
|
+
"metric": "chrf++",
|
|
31
|
+
"value": 32.3,
|
|
32
|
+
"higherIsBetter": true,
|
|
33
|
+
"n": 100,
|
|
34
|
+
"measuredAt": "2026-10-08",
|
|
35
|
+
"language": "yo",
|
|
36
|
+
"slice": "en->yo"
|
|
37
|
+
}
|
|
38
|
+
]
|
|
39
|
+
}
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
| Field | Meaning |
|
|
43
|
+
|---|---|
|
|
44
|
+
| `model` | An OpenRouter model id. A model that can't be routed (run locally) uses `local/<name>`; Lodestone ignores those. A `:free` or other `:variant` suffix is ignored when matching, so a score for a model applies to its free version. |
|
|
45
|
+
| `task` | What was measured: `translate`, `transcribe`, `availability`, `latency`. |
|
|
46
|
+
| `metric` | How: `chrf++`, `wer`, `success_rate`, `latency_p50_ms`. |
|
|
47
|
+
| `value`, `higherIsBetter` | The number, and which way is good. Both are stated, never assumed. |
|
|
48
|
+
| `n` | Items the score rests on. |
|
|
49
|
+
| `measuredAt` | The day it was measured. |
|
|
50
|
+
| `language` | ISO 639-1 code, when the task is about a language. |
|
|
51
|
+
| `slice` | A finer cut: `en->yo`, `avg`, or for live stats the provider. |
|
|
52
|
+
|
|
53
|
+
`parseScorecard()` checks all of this and lists every problem. A scorecard with a different `schema` version is refused, not guessed at.
|
|
54
|
+
|
|
55
|
+
### Using one
|
|
56
|
+
|
|
57
|
+
```ts
|
|
58
|
+
import { createLodestone, fetchScorecard } from '@extrafemi-ai/lodestone';
|
|
59
|
+
|
|
60
|
+
let card; let at = 0;
|
|
61
|
+
const ai = createLodestone({
|
|
62
|
+
openrouter: { apiKey: process.env.OPENROUTER_API_KEY },
|
|
63
|
+
// Called per request; keep your own cache. If it fails or takes over 1.5 s, the order is left alone.
|
|
64
|
+
scorecards: async () => {
|
|
65
|
+
if (!card || Date.now() - at > 6 * 3_600_000) { card = await fetchScorecard('https://naija-eval.extrafemi.com/data/scorecard.json'); at = Date.now(); }
|
|
66
|
+
return [card];
|
|
67
|
+
},
|
|
68
|
+
profiles: {
|
|
69
|
+
'translate-yoruba': { tier: 'strong', quality: { task: 'translate', metric: 'chrf++', language: 'yo', slice: 'avg' } },
|
|
70
|
+
},
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
await ai.complete({ task: 'translate-yoruba', prompt });
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
The rules, so there are no surprises:
|
|
77
|
+
|
|
78
|
+
- Scores based on fewer than 30 items, or older than a year, are ignored (`minN`, `maxAgeDays` to change).
|
|
79
|
+
- Measured models are sorted among the slots they already hold. An unmeasured model keeps its slot, so a good unmeasured model is never pushed down by a weaker measured one.
|
|
80
|
+
- It applies to the free, paid and fallback model lists. It never reorders an explicit `route`.
|
|
81
|
+
- With two scorecards measuring the same thing, the one based on more items wins.
|
|
82
|
+
|
|
83
|
+
## Live reliability
|
|
84
|
+
|
|
85
|
+
An evaluator says how good an answer is. It can't say whether you get an answer at all. Lodestone records that itself:
|
|
86
|
+
|
|
87
|
+
```ts
|
|
88
|
+
import { createStats } from '@extrafemi-ai/lodestone';
|
|
89
|
+
|
|
90
|
+
const stats = createStats(); // last 200 attempts per model and provider
|
|
91
|
+
const ai = createLodestone({ /* ... */, stats });
|
|
92
|
+
|
|
93
|
+
stats.scorecard(); // success_rate, latency_p50_ms, latency_p95_ms per model, as a model-scorecard/1
|
|
94
|
+
stats.toJSON(); // save in your store; stats.load(saved) after a restart
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
The result is an ordinary scorecard (`source: "lodestone-live"`), so it can be served, shared, or used to order models: `quality: { task: 'availability', metric: 'success_rate' }` puts the model that answers most often first. Stats never see prompts or answers.
|
|
98
|
+
|
|
99
|
+
## `model-candidates/1`
|
|
100
|
+
|
|
101
|
+
The models a tool can actually call, so an evaluator benchmarks those and not a guess.
|
|
102
|
+
|
|
103
|
+
```ts
|
|
104
|
+
const candidates = await ai.candidates();
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
```json
|
|
108
|
+
{
|
|
109
|
+
"schema": "model-candidates/1",
|
|
110
|
+
"source": "lodestone",
|
|
111
|
+
"generatedAt": "2026-10-28T09:00:00Z",
|
|
112
|
+
"models": [
|
|
113
|
+
{ "id": "google/gemma-4-31b-it:free", "provider": "openrouter", "roles": ["free:strong", "free:vision"] },
|
|
114
|
+
{ "id": "qwen/qwen3.7-flash", "provider": "openrouter", "roles": ["paid"] }
|
|
115
|
+
]
|
|
116
|
+
}
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Serve it from an endpoint or write it to a file. Naija Eval's `eval/candidates.py` reads it and prints the exact commands for the models it hasn't measured yet.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@extrafemi-ai/lodestone",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"description": "Calls AI models so your tool keeps working when a model is withdrawn, slow or rate-limited.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
},
|
|
24
24
|
"files": [
|
|
25
25
|
"dist",
|
|
26
|
+
"docs",
|
|
26
27
|
"README.md",
|
|
27
28
|
"LICENSE"
|
|
28
29
|
],
|
|
@@ -39,7 +40,7 @@
|
|
|
39
40
|
"devDependencies": {
|
|
40
41
|
"@types/node": "^22.10.0",
|
|
41
42
|
"typescript": "^5.6.0",
|
|
42
|
-
"vitest": "^
|
|
43
|
+
"vitest": "^4.1.12"
|
|
43
44
|
},
|
|
44
45
|
"keywords": [
|
|
45
46
|
"ai",
|