@extrafemi-ai/lodestone 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -83,6 +83,10 @@ const { ok, withdrawn, shortTiers } = await ai.checkModels();
83
83
 
84
84
  `withdrawn` lists preferred models that have gone. That is harmless, they are already replaced. `ok` is false only when the catalogue can't be read or a tier is left with fewer than two real models. Without a daily run, a request that finds the saved catalogue over a day old refreshes it in the background.
85
85
 
86
+ ### Letting measurements decide the order
87
+
88
+ A scorecard (a small JSON file of measured results) can put the models that did best on your kind of task first, and Lodestone can publish its own live success rates and speeds in the same format. [Naija Eval](https://naija-eval.extrafemi.com) publishes translation and speech quality for Hausa, Igbo and Yoruba this way. It is opt-in, per task, and only changes the order models are tried in. See [docs/scorecards.md](docs/scorecards.md).
89
+
86
90
  ### Caching
87
91
 
88
92
  Pass `cacheKey` for a pure, repeatable task (for example a hash of a job page) and the answer is saved in the store and reused for `cacheTtlMs` (a day by default). Failures are never cached.
@@ -0,0 +1,16 @@
1
+ /** The models a tool would actually call, in a format an evaluator can read, so it
2
+ * benchmarks what is in use and not a guess. Naija Eval reads this to say which of
3
+ * them it has not measured yet. See docs/scorecards.md. */
4
+ export declare const CANDIDATES_SCHEMA = "model-candidates/1";
5
+ export interface Candidate {
6
+ id: string;
7
+ provider: string;
8
+ /** Where it is used: free:fast, free:strong, free:vision, paid, fallback. */
9
+ roles: string[];
10
+ }
11
+ export interface Candidates {
12
+ schema: typeof CANDIDATES_SCHEMA;
13
+ source: string;
14
+ generatedAt: string;
15
+ models: Candidate[];
16
+ }
@@ -0,0 +1,4 @@
1
+ /** The models a tool would actually call, in a format an evaluator can read, so it
2
+ * benchmarks what is in use and not a guess. Naija Eval reads this to say which of
3
+ * them it has not measured yet. See docs/scorecards.md. */
4
+ export const CANDIDATES_SCHEMA = 'model-candidates/1';
package/dist/index.d.ts CHANGED
@@ -16,3 +16,9 @@ export { anthropic } from './providers/anthropic.js';
16
16
  export { google } from './providers/google.js';
17
17
  export { mock } from './providers/mock.js';
18
18
  export type { Tier, Message, ContentPart, Provider, ProviderRequest, ProviderReply, KeyValueStore, Usage } from './types.js';
19
+ export { SCORECARD_SCHEMA, ScorecardError, parseScorecard, fetchScorecard, rankModels, scoreFor, baseModelId } from './scorecard.js';
20
+ export type { Scorecard, ModelScore, QualityQuery, RankOptions } from './scorecard.js';
21
+ export { createStats } from './stats.js';
22
+ export type { Stats, StatsOptions, StatsState } from './stats.js';
23
+ export { CANDIDATES_SCHEMA } from './candidates.js';
24
+ export type { Candidates, Candidate } from './candidates.js';
package/dist/index.js CHANGED
@@ -10,3 +10,6 @@ export { openrouter } from './providers/openrouter.js';
10
10
  export { anthropic } from './providers/anthropic.js';
11
11
  export { google } from './providers/google.js';
12
12
  export { mock } from './providers/mock.js';
13
+ export { SCORECARD_SCHEMA, ScorecardError, parseScorecard, fetchScorecard, rankModels, scoreFor, baseModelId } from './scorecard.js';
14
+ export { createStats } from './stats.js';
15
+ export { CANDIDATES_SCHEMA } from './candidates.js';
@@ -1,9 +1,14 @@
1
1
  import { type CatalogModel } from './catalog.js';
2
+ import { type Candidates } from './candidates.js';
3
+ import { type QualityQuery, type Scorecard } from './scorecard.js';
4
+ import type { Stats } from './stats.js';
2
5
  import type { KeyValueStore, Message, Provider, Tier, Usage } from './types.js';
3
6
  export interface Profile {
4
7
  tier: Tier;
5
8
  maxTokens?: number;
6
9
  temperature?: number;
10
+ /** Put the models a scorecard measured best on this first. See `scorecards`. */
11
+ quality?: QualityQuery;
7
12
  }
8
13
  export interface FallbackConfig {
9
14
  provider: Provider;
@@ -34,6 +39,13 @@ export interface LodestoneConfig {
34
39
  providers?: Record<string, Provider>;
35
40
  /** Other providers, each with its own key, tried after OpenRouter has failed. */
36
41
  fallbacks?: FallbackConfig[];
42
+ /** Measured results about models (see scorecard.ts). With a `quality` query on the task
43
+ * or the call, the free, paid and fallback models are put in order of their score. Models
44
+ * nobody has measured keep their place. A function is called per request, so it can fetch
45
+ * and cache; if it fails or takes over 1.5 s the order is left alone. */
46
+ scorecards?: Scorecard[] | (() => Promise<Scorecard[]>);
47
+ /** Records every attempt so you can publish how each model does (stats.scorecard()). */
48
+ stats?: Stats;
37
49
  /** Named tasks, so calls say what they are doing and not which model to use. */
38
50
  profiles?: Record<string, Profile>;
39
51
  /** Where the model catalogue and cached answers are kept. Default: memory. */
@@ -66,6 +78,8 @@ export interface CompleteOptions {
66
78
  route?: RouteStep[];
67
79
  /** A key in `profiles`. */
68
80
  task?: string;
81
+ /** Overrides the profile's quality query. */
82
+ quality?: QualityQuery;
69
83
  /** Overrides the profile's tier. Default: 'vision' if the messages contain an image, else 'fast'. */
70
84
  tier?: Tier;
71
85
  prompt?: string;
@@ -104,5 +118,6 @@ export declare function createLodestone(config?: LodestoneConfig): {
104
118
  withdrawn?: string[];
105
119
  shortTiers?: Tier[];
106
120
  }>;
121
+ candidates: () => Promise<Candidates>;
107
122
  };
108
123
  export type Lodestone = ReturnType<typeof createLodestone>;
package/dist/lodestone.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import { createCatalog, pickModels, ROUTER } from './catalog.js';
2
+ import { CANDIDATES_SCHEMA } from './candidates.js';
2
3
  import { LodestoneError, messageFrom, ProviderError } from './errors.js';
3
4
  import { tryParseJson } from './json.js';
4
5
  import { DEFAULT_MODELS, DEFAULT_PAID_MODELS } from './models.js';
@@ -6,6 +7,7 @@ import { readRateLimit } from './rateLimit.js';
6
7
  import { anthropic } from './providers/anthropic.js';
7
8
  import { google } from './providers/google.js';
8
9
  import { openrouter } from './providers/openrouter.js';
10
+ import { rankModels } from './scorecard.js';
9
11
  import { memoryStore } from './store.js';
10
12
  const hasImage = (messages) => messages.some((m) => Array.isArray(m.content) && m.content.some((p) => p.type === 'image'));
11
13
  export function createLodestone(config = {}) {
@@ -34,6 +36,20 @@ export function createLodestone(config = {}) {
34
36
  return prefs[tier];
35
37
  }
36
38
  }
39
+ /** The scorecards, or none: never throws and never holds a request up for long. */
40
+ async function cards() {
41
+ const src = config.scorecards;
42
+ if (!src)
43
+ return [];
44
+ if (Array.isArray(src))
45
+ return src;
46
+ try {
47
+ return await Promise.race([src(), new Promise((resolve) => setTimeout(() => resolve([]), 1_500).unref?.())]);
48
+ }
49
+ catch {
50
+ return [];
51
+ }
52
+ }
37
53
  async function complete(opts) {
38
54
  const messages = opts.messages ?? (opts.prompt !== undefined ? [{ role: 'user', content: opts.prompt }] : []);
39
55
  if (messages.length === 0)
@@ -45,6 +61,8 @@ export function createLodestone(config = {}) {
45
61
  const tier = opts.tier ?? profile?.tier ?? (images ? 'vision' : 'fast');
46
62
  const maxTokens = opts.maxTokens ?? profile?.maxTokens ?? 3500;
47
63
  const temperature = opts.temperature ?? profile?.temperature ?? (opts.route ? undefined : 0.2);
64
+ const quality = opts.quality ?? profile?.quality;
65
+ const order = async (models) => (quality && models.length > 1 ? rankModels(models, await cards(), quality, { now }) : models);
48
66
  const cacheSlot = opts.cacheKey ? `lodestone:cache:${opts.cacheKey}` : null;
49
67
  if (cacheSlot) {
50
68
  try {
@@ -90,7 +108,9 @@ export function createLodestone(config = {}) {
90
108
  else
91
109
  out = { kind: 'fail', status: 599, body: e.message };
92
110
  }
93
- config.onAttempt?.({ provider: provider.name, model, phase, outcome: out.kind === 'ok' ? 'ok' : out.kind === 'timeout' ? 'timeout' : 'failed', status: out.kind === 'fail' ? out.status : undefined, ms: now() - at });
111
+ const event = { provider: provider.name, model, phase, outcome: out.kind === 'ok' ? 'ok' : out.kind === 'timeout' ? 'timeout' : 'failed', status: out.kind === 'fail' ? out.status : undefined, ms: now() - at };
112
+ config.stats?.record(event);
113
+ config.onAttempt?.(event);
94
114
  last = out;
95
115
  return out;
96
116
  };
@@ -130,7 +150,7 @@ export function createLodestone(config = {}) {
130
150
  }
131
151
  // Phase 1: OpenRouter's free models. A withdrawn one has already been replaced by resolveModels.
132
152
  if (orKey) {
133
- const models = await resolveModels(tier);
153
+ const models = await order(await resolveModels(tier));
134
154
  for (let i = 0; i < models.length && freeLeft() >= t.minSlice; i++) {
135
155
  const slice = i >= models.length - 1 ? freeLeft() : Math.min(later ? t.freeSliceLater : t.freeSlice, freeLeft());
136
156
  const out = await attempt(router, orKey, models[i], 'free', slice);
@@ -147,7 +167,7 @@ export function createLodestone(config = {}) {
147
167
  }
148
168
  // Phase 2: a low-cost paid model, which is not under the free-model limits.
149
169
  if (paidAvailable && !authFailed) {
150
- for (const m of paidModels) {
170
+ for (const m of await order(paidModels)) {
151
171
  if (left() < t.minSlice)
152
172
  break;
153
173
  const out = await attempt(router, orKey, m, 'paid', left());
@@ -166,7 +186,7 @@ export function createLodestone(config = {}) {
166
186
  }
167
187
  // Phase 3: other providers, with their own keys.
168
188
  for (const fb of fbs) {
169
- for (const m of fb.models) {
189
+ for (const m of await order(fb.models)) {
170
190
  if (left() < t.minSlice)
171
191
  break;
172
192
  const out = await attempt(fb.provider, fb.apiKey, m, 'fallback', left());
@@ -224,5 +244,27 @@ export function createLodestone(config = {}) {
224
244
  const shortTiers = Object.keys(prefs).filter((tier) => pickModels(tier, models, prefs[tier]).filter((m) => m !== ROUTER).length < 2);
225
245
  return { ok: shortTiers.length === 0, catalogSize: models.length, withdrawn, shortTiers };
226
246
  }
227
- return { complete, json, resolveModels, refreshCatalog, checkModels };
247
+ /** Every model this instance can call, for an evaluator to benchmark (see candidates.ts). */
248
+ async function candidates() {
249
+ const byKey = new Map();
250
+ const add = (id, provider, role) => {
251
+ if (id === ROUTER)
252
+ return;
253
+ const k = `${provider}\u0000${id}`;
254
+ const c = byKey.get(k) ?? { id, provider, roles: [] };
255
+ if (!c.roles.includes(role))
256
+ c.roles.push(role);
257
+ byKey.set(k, c);
258
+ };
259
+ for (const tier of Object.keys(prefs))
260
+ for (const id of await resolveModels(tier))
261
+ add(id, 'openrouter', `free:${tier}`);
262
+ for (const id of paidModels)
263
+ add(id, 'openrouter', 'paid');
264
+ for (const fb of fallbacks)
265
+ for (const id of fb.models)
266
+ add(id, fb.provider.name, 'fallback');
267
+ return { schema: CANDIDATES_SCHEMA, source: 'lodestone', generatedAt: new Date(now()).toISOString(), models: [...byKey.values()] };
268
+ }
269
+ return { complete, json, resolveModels, refreshCatalog, checkModels, candidates };
228
270
  }
@@ -0,0 +1,68 @@
1
+ /** A model scorecard: measured results about models, in one format that anyone can
2
+ * produce and Lodestone can read. Naija Eval publishes one (translation and speech
3
+ * quality for Hausa, Igbo and Yoruba); Lodestone produces one of its own from live
4
+ * traffic (how often each model answered, and how fast). Lodestone uses a scorecard
5
+ * to put better models first, for tasks the scorecard measured. See docs/scorecards.md. */
6
+ export declare const SCORECARD_SCHEMA = "model-scorecard/1";
7
+ export interface ModelScore {
8
+ /** An OpenRouter model id (for example anthropic/claude-sonnet-5.5), or local/<name> for a model that is not routable. */
9
+ model: string;
10
+ /** What was measured: translate, transcribe, availability, latency... */
11
+ task: string;
12
+ /** How it was scored: chrf++, wer, success_rate, latency_p50_ms... */
13
+ metric: string;
14
+ value: number;
15
+ higherIsBetter: boolean;
16
+ /** How many items the score is based on. */
17
+ n: number;
18
+ /** The day it was measured, ISO (2026-10-08). */
19
+ measuredAt: string;
20
+ /** ISO 639-1 code of the language measured, when the task is about one. */
21
+ language?: string;
22
+ /** A finer cut, for example en->yo, or the provider for live stats. */
23
+ slice?: string;
24
+ }
25
+ export interface Scorecard {
26
+ schema: typeof SCORECARD_SCHEMA;
27
+ /** Who measured it: naija-eval, lodestone-live... */
28
+ source: string;
29
+ generatedAt: string;
30
+ scores: ModelScore[];
31
+ }
32
+ /** Which scores to look at when ordering models for a task. */
33
+ export interface QualityQuery {
34
+ task: string;
35
+ metric: string;
36
+ language?: string;
37
+ slice?: string;
38
+ }
39
+ export interface RankOptions {
40
+ /** Ignore scores based on fewer items than this. Default 30. */
41
+ minN?: number;
42
+ /** Ignore scores older than this. Default 365. */
43
+ maxAgeDays?: number;
44
+ now?: () => number;
45
+ }
46
+ export declare class ScorecardError extends Error {
47
+ problems: string[];
48
+ constructor(problems: string[]);
49
+ }
50
+ /** Checks a scorecard from outside (a file, a URL) and returns it typed. Throws a
51
+ * ScorecardError listing what is wrong. A scorecard with a schema version this
52
+ * library doesn't know is refused, not guessed at. */
53
+ export declare function parseScorecard(raw: unknown): Scorecard;
54
+ /** Fetches and checks a scorecard from a URL. Throws if it can't be read or isn't valid. */
55
+ export declare function fetchScorecard(url: string, opts?: {
56
+ fetch?: typeof fetch;
57
+ timeoutMs?: number;
58
+ }): Promise<Scorecard>;
59
+ /** An id without its variant suffix: anthropic/claude-x:free is the same model as anthropic/claude-x. */
60
+ export declare const baseModelId: (id: string) => string;
61
+ /** The score for a model on a query, or undefined if nobody measured it well enough.
62
+ * Where several scorecards measured the same thing, the one based on most items wins. */
63
+ export declare function scoreFor(cards: Scorecard[], model: string, q: QualityQuery, opts?: RankOptions): ModelScore | undefined;
64
+ /** Puts measured models in order of their score, without moving anything else. The
65
+ * measured models take turns in the slots they already occupy; a model nobody has
66
+ * measured stays exactly where it was, so a good unmeasured model is never pushed
67
+ * down by a weaker measured one. Nothing is added or dropped. */
68
+ export declare function rankModels(models: string[], cards: Scorecard[], q: QualityQuery, opts?: RankOptions): string[];
@@ -0,0 +1,96 @@
1
+ /** A model scorecard: measured results about models, in one format that anyone can
2
+ * produce and Lodestone can read. Naija Eval publishes one (translation and speech
3
+ * quality for Hausa, Igbo and Yoruba); Lodestone produces one of its own from live
4
+ * traffic (how often each model answered, and how fast). Lodestone uses a scorecard
5
+ * to put better models first, for tasks the scorecard measured. See docs/scorecards.md. */
6
+ export const SCORECARD_SCHEMA = 'model-scorecard/1';
7
+ export class ScorecardError extends Error {
8
+ problems;
9
+ constructor(problems) {
10
+ super(`not a valid model scorecard: ${problems.slice(0, 3).join('; ')}${problems.length > 3 ? `; and ${problems.length - 3} more` : ''}`);
11
+ this.problems = problems;
12
+ this.name = 'ScorecardError';
13
+ }
14
+ }
15
+ const isStr = (v) => typeof v === 'string' && v.length > 0;
16
+ const isNum = (v) => typeof v === 'number' && Number.isFinite(v);
17
+ const isDay = (v) => typeof v === 'string' && /^\d{4}-\d{2}-\d{2}/.test(v) && !Number.isNaN(Date.parse(v));
18
+ /** Checks a scorecard from outside (a file, a URL) and returns it typed. Throws a
19
+ * ScorecardError listing what is wrong. A scorecard with a schema version this
20
+ * library doesn't know is refused, not guessed at. */
21
+ export function parseScorecard(raw) {
22
+ const problems = [];
23
+ const c = raw;
24
+ if (!c || typeof c !== 'object')
25
+ throw new ScorecardError(['not an object']);
26
+ if (c.schema !== SCORECARD_SCHEMA)
27
+ problems.push(`schema is ${JSON.stringify(c.schema)}, expected ${SCORECARD_SCHEMA}`);
28
+ if (!isStr(c.source))
29
+ problems.push('source is missing');
30
+ if (!isDay(c.generatedAt))
31
+ problems.push('generatedAt is missing or not a date');
32
+ if (!Array.isArray(c.scores))
33
+ problems.push('scores is not a list');
34
+ else
35
+ c.scores.forEach((s, i) => {
36
+ const at = `scores[${i}]`;
37
+ if (!isStr(s?.model))
38
+ problems.push(`${at}.model is missing`);
39
+ if (!isStr(s?.task))
40
+ problems.push(`${at}.task is missing`);
41
+ if (!isStr(s?.metric))
42
+ problems.push(`${at}.metric is missing`);
43
+ if (!isNum(s?.value))
44
+ problems.push(`${at}.value is not a number`);
45
+ if (typeof s?.higherIsBetter !== 'boolean')
46
+ problems.push(`${at}.higherIsBetter is not true or false`);
47
+ if (!isNum(s?.n) || s.n < 0)
48
+ problems.push(`${at}.n is not a count`);
49
+ if (!isDay(s?.measuredAt))
50
+ problems.push(`${at}.measuredAt is not a date`);
51
+ });
52
+ if (problems.length)
53
+ throw new ScorecardError(problems);
54
+ return c;
55
+ }
56
+ /** Fetches and checks a scorecard from a URL. Throws if it can't be read or isn't valid. */
57
+ export async function fetchScorecard(url, opts = {}) {
58
+ const res = await (opts.fetch ?? fetch)(url, { signal: AbortSignal.timeout(opts.timeoutMs ?? 8_000) });
59
+ if (!res.ok)
60
+ throw new Error(`scorecard ${url}: ${res.status}`);
61
+ return parseScorecard(await res.json());
62
+ }
63
+ /** An id without its variant suffix: anthropic/claude-x:free is the same model as anthropic/claude-x. */
64
+ export const baseModelId = (id) => id.split(':')[0].toLowerCase();
65
+ function matches(s, q) {
66
+ return s.task === q.task && s.metric === q.metric && (q.language === undefined || s.language === q.language) && (q.slice === undefined || s.slice === q.slice);
67
+ }
68
+ /** The score for a model on a query, or undefined if nobody measured it well enough.
69
+ * Where several scorecards measured the same thing, the one based on most items wins. */
70
+ export function scoreFor(cards, model, q, opts = {}) {
71
+ const minN = opts.minN ?? 30;
72
+ const oldest = (opts.now ?? Date.now)() - (opts.maxAgeDays ?? 365) * 86_400_000;
73
+ const id = baseModelId(model);
74
+ let best;
75
+ for (const card of cards)
76
+ for (const s of card.scores) {
77
+ if (baseModelId(s.model) !== id || !matches(s, q) || s.n < minN || Date.parse(s.measuredAt) < oldest)
78
+ continue;
79
+ if (!best || s.n > best.n || (s.n === best.n && s.measuredAt > best.measuredAt))
80
+ best = s;
81
+ }
82
+ return best;
83
+ }
84
+ /** Puts measured models in order of their score, without moving anything else. The
85
+ * measured models take turns in the slots they already occupy; a model nobody has
86
+ * measured stays exactly where it was, so a good unmeasured model is never pushed
87
+ * down by a weaker measured one. Nothing is added or dropped. */
88
+ export function rankModels(models, cards, q, opts = {}) {
89
+ const measured = models.map((m, i) => ({ m, i, s: scoreFor(cards, m, q, opts) })).filter((x) => x.s !== undefined);
90
+ if (measured.length < 2)
91
+ return models;
92
+ const sorted = [...measured].sort((a, b) => (a.s.higherIsBetter ? b.s.value - a.s.value : a.s.value - b.s.value) || a.i - b.i);
93
+ const out = [...models];
94
+ measured.forEach((slot, k) => { out[slot.i] = sorted[k].m; });
95
+ return out;
96
+ }
@@ -0,0 +1,28 @@
1
+ import type { AttemptEvent } from './lodestone.js';
2
+ import { type Scorecard } from './scorecard.js';
3
+ /** Records how each model does in live traffic and reports it as a scorecard: how often
4
+ * it answered, and how fast. This is the other half of Naija Eval's measurements: that
5
+ * says how good an answer is, this says whether you get one at all. Keeps the last
6
+ * `window` attempts per model, in memory. Save `toJSON()` in your store if you want it
7
+ * to survive restarts. Never sees prompts or answers. */
8
+ export interface StatsOptions {
9
+ /** Attempts kept per model and provider. Default 200. */
10
+ window?: number;
11
+ /** The `source` of the scorecard it produces. Default lodestone-live. */
12
+ source?: string;
13
+ now?: () => number;
14
+ }
15
+ interface Sample {
16
+ ok: boolean;
17
+ ms: number;
18
+ at: number;
19
+ }
20
+ export type StatsState = Record<string, Sample[]>;
21
+ export interface Stats {
22
+ record(e: AttemptEvent): void;
23
+ scorecard(): Scorecard;
24
+ toJSON(): StatsState;
25
+ load(state: StatsState): void;
26
+ }
27
+ export declare function createStats(opts?: StatsOptions): Stats;
28
+ export {};
package/dist/stats.js ADDED
@@ -0,0 +1,37 @@
1
+ import { SCORECARD_SCHEMA } from './scorecard.js';
2
+ const percentile = (sorted, p) => sorted[Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1)];
3
+ const key = (provider, model) => `${provider}\u0000${model}`;
4
+ const day = (ms) => new Date(ms).toISOString().slice(0, 10);
5
+ export function createStats(opts = {}) {
6
+ const window = opts.window ?? 200;
7
+ const now = opts.now ?? Date.now;
8
+ let state = {};
9
+ return {
10
+ record(e) {
11
+ const k = key(e.provider, e.model);
12
+ const list = (state[k] ??= []);
13
+ list.push({ ok: e.outcome === 'ok', ms: e.ms, at: now() });
14
+ if (list.length > window)
15
+ list.splice(0, list.length - window);
16
+ },
17
+ toJSON: () => state,
18
+ load(s) { state = s; },
19
+ scorecard() {
20
+ const scores = [];
21
+ for (const [k, list] of Object.entries(state)) {
22
+ if (!list.length)
23
+ continue;
24
+ const [provider, model] = k.split('\u0000');
25
+ const measuredAt = day(Math.max(...list.map((s) => s.at)));
26
+ const base = { model, n: list.length, measuredAt, slice: provider };
27
+ scores.push({ ...base, task: 'availability', metric: 'success_rate', value: Math.round((list.filter((s) => s.ok).length / list.length) * 1000) / 10, higherIsBetter: true });
28
+ const times = list.filter((s) => s.ok).map((s) => s.ms).sort((a, b) => a - b);
29
+ if (times.length) {
30
+ scores.push({ ...base, n: times.length, task: 'latency', metric: 'latency_p50_ms', value: percentile(times, 50), higherIsBetter: false });
31
+ scores.push({ ...base, n: times.length, task: 'latency', metric: 'latency_p95_ms', value: percentile(times, 95), higherIsBetter: false });
32
+ }
33
+ }
34
+ return { schema: SCORECARD_SCHEMA, source: opts.source ?? 'lodestone-live', generatedAt: new Date(now()).toISOString(), scores };
35
+ },
36
+ };
37
+ }
@@ -0,0 +1,119 @@
1
+ # Scorecards and candidates
2
+
3
+ Two small JSON formats let a model evaluator and Lodestone feed each other without knowing about each other's code. [Naija Eval](https://naija-eval.extrafemi.com) is the first evaluator to use them.
4
+
5
+ ```
6
+ evaluator (Naija Eval) Lodestone (in your tool)
7
+ ────────────────────── ────────────────────────
8
+ measures answer quality ── model-scorecard ──▶ orders models by it, per task
9
+ │
10
+ benchmarks what is in use ◀─ model-candidates ── lists the models it can call
11
+ │
12
+ (reads live reliability) ◀── model-scorecard ─── records success rate and speed
13
+ ```
14
+
15
+ Nothing here is automatic or hidden. A scorecard changes the order models are *tried* in, for tasks you name, and nothing else. A model nobody measured is never dropped, added or moved.
16
+
17
+ ## `model-scorecard/1`
18
+
19
+ Measured results about models.
20
+
21
+ ```json
22
+ {
23
+ "schema": "model-scorecard/1",
24
+ "source": "naija-eval",
25
+ "generatedAt": "2026-10-28T09:00:00Z",
26
+ "scores": [
27
+ {
28
+ "model": "anthropic/claude-sonnet-5.5",
29
+ "task": "translate",
30
+ "metric": "chrf++",
31
+ "value": 32.3,
32
+ "higherIsBetter": true,
33
+ "n": 100,
34
+ "measuredAt": "2026-10-08",
35
+ "language": "yo",
36
+ "slice": "en->yo"
37
+ }
38
+ ]
39
+ }
40
+ ```
41
+
42
+ | Field | Meaning |
43
+ |---|---|
44
+ | `model` | An OpenRouter model id. A model that can't be routed (run locally) uses `local/<name>`; Lodestone ignores those. A `:free` or other `:variant` suffix is ignored when matching, so a score for a model applies to its free version. |
45
+ | `task` | What was measured: `translate`, `transcribe`, `availability`, `latency`. |
46
+ | `metric` | How: `chrf++`, `wer`, `success_rate`, `latency_p50_ms`. |
47
+ | `value`, `higherIsBetter` | The number, and which way is good. Both are stated, never assumed. |
48
+ | `n` | Items the score rests on. |
49
+ | `measuredAt` | The day it was measured. |
50
+ | `language` | ISO 639-1 code, when the task is about a language. |
51
+ | `slice` | A finer cut: `en->yo`, `avg`, or for live stats the provider. |
52
+
53
+ `parseScorecard()` checks all of this and lists every problem. A scorecard with a different `schema` version is refused, not guessed at.
54
+
55
+ ### Using one
56
+
57
+ ```ts
58
+ import { createLodestone, fetchScorecard } from '@extrafemi-ai/lodestone';
59
+
60
+ let card; let at = 0;
61
+ const ai = createLodestone({
62
+ openrouter: { apiKey: process.env.OPENROUTER_API_KEY },
63
+ // Called per request; keep your own cache. If it fails or takes over 1.5 s, the order is left alone.
64
+ scorecards: async () => {
65
+ if (!card || Date.now() - at > 6 * 3_600_000) { card = await fetchScorecard('https://naija-eval.extrafemi.com/data/scorecard.json'); at = Date.now(); }
66
+ return [card];
67
+ },
68
+ profiles: {
69
+ 'translate-yoruba': { tier: 'strong', quality: { task: 'translate', metric: 'chrf++', language: 'yo', slice: 'avg' } },
70
+ },
71
+ });
72
+
73
+ await ai.complete({ task: 'translate-yoruba', prompt });
74
+ ```
75
+
76
+ The rules, so there are no surprises:
77
+
78
+ - Scores based on fewer than 30 items, or older than a year, are ignored (`minN`, `maxAgeDays` to change).
79
+ - Measured models are sorted among the slots they already hold. An unmeasured model keeps its slot, so a good unmeasured model is never pushed down by a weaker measured one.
80
+ - It applies to the free, paid and fallback model lists. It never reorders an explicit `route`.
81
+ - With two scorecards measuring the same thing, the one based on more items wins.
82
+
83
+ ## Live reliability
84
+
85
+ An evaluator says how good an answer is. It can't say whether you get an answer at all. Lodestone records that itself:
86
+
87
+ ```ts
88
+ import { createStats } from '@extrafemi-ai/lodestone';
89
+
90
+ const stats = createStats(); // last 200 attempts per model and provider
91
+ const ai = createLodestone({ /* ... */, stats });
92
+
93
+ stats.scorecard(); // success_rate, latency_p50_ms, latency_p95_ms per model, as a model-scorecard/1
94
+ stats.toJSON(); // save in your store; stats.load(saved) after a restart
95
+ ```
96
+
97
+ The result is an ordinary scorecard (`source: "lodestone-live"`), so it can be served, shared, or used to order models: `quality: { task: 'availability', metric: 'success_rate' }` puts the model that answers most often first. Stats never see prompts or answers.
98
+
99
+ ## `model-candidates/1`
100
+
101
+ The models a tool can actually call, so an evaluator benchmarks those and not a guess.
102
+
103
+ ```ts
104
+ const candidates = await ai.candidates();
105
+ ```
106
+
107
+ ```json
108
+ {
109
+ "schema": "model-candidates/1",
110
+ "source": "lodestone",
111
+ "generatedAt": "2026-10-28T09:00:00Z",
112
+ "models": [
113
+ { "id": "google/gemma-4-31b-it:free", "provider": "openrouter", "roles": ["free:strong", "free:vision"] },
114
+ { "id": "qwen/qwen3.7-flash", "provider": "openrouter", "roles": ["paid"] }
115
+ ]
116
+ }
117
+ ```
118
+
119
+ Serve it from an endpoint or write it to a file. Naija Eval's `eval/candidates.py` reads it and prints the exact commands for the models it hasn't measured yet.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@extrafemi-ai/lodestone",
3
- "version": "0.2.0",
3
+ "version": "0.3.0",
4
4
  "description": "Calls AI models so your tool keeps working when a model is withdrawn, slow or rate-limited.",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -23,6 +23,7 @@
23
23
  },
24
24
  "files": [
25
25
  "dist",
26
+ "docs",
26
27
  "README.md",
27
28
  "LICENSE"
28
29
  ],