@extrafemi-ai/lodestone 0.6.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +44 -21
  3. package/SECURITY.md +17 -0
  4. package/dist/catalog.js +11 -2
  5. package/dist/errors.d.ts +6 -0
  6. package/dist/errors.js +1 -2
  7. package/dist/explain.js +4 -0
  8. package/dist/health.js +19 -3
  9. package/dist/index.d.ts +10 -4
  10. package/dist/index.js +4 -1
  11. package/dist/limiter.d.ts +34 -0
  12. package/dist/limiter.js +35 -0
  13. package/dist/lodestone.d.ts +25 -12
  14. package/dist/lodestone.js +56 -10
  15. package/dist/providers/anthropic.d.ts +2 -0
  16. package/dist/providers/anthropic.js +1 -1
  17. package/dist/providers/openaiCompatible.d.ts +2 -0
  18. package/dist/providers/openaiCompatible.js +2 -0
  19. package/dist/scorecard.d.ts +9 -3
  20. package/dist/scorecard.js +45 -2
  21. package/dist/spend.d.ts +33 -0
  22. package/dist/spend.js +72 -0
  23. package/dist/store.d.ts +2 -2
  24. package/dist/store.js +19 -3
  25. package/dist/stores/cloudflare.d.ts +10 -0
  26. package/dist/stores/cloudflare.js +11 -0
  27. package/dist/stores/deno.d.ts +14 -0
  28. package/dist/stores/deno.js +8 -0
  29. package/dist/stores/index.d.ts +9 -0
  30. package/dist/stores/index.js +5 -0
  31. package/dist/stores/netlify.d.ts +12 -0
  32. package/dist/stores/netlify.js +9 -0
  33. package/dist/stores/safeKey.d.ts +10 -0
  34. package/dist/stores/safeKey.js +20 -0
  35. package/dist/stores/upstash.d.ts +14 -0
  36. package/dist/stores/upstash.js +50 -0
  37. package/dist/types.d.ts +15 -3
  38. package/dist/util.d.ts +7 -0
  39. package/dist/util.js +17 -0
  40. package/docs/free-model-index-spec.md +141 -0
  41. package/docs/protecting-your-budget.md +48 -0
  42. package/docs/runtimes.md +19 -0
  43. package/docs/scorecards.md +20 -0
  44. package/docs/stores.md +45 -0
  45. package/examples/netlify-function.ts +10 -7
  46. package/package.json +5 -2
package/dist/util.d.ts ADDED
@@ -0,0 +1,7 @@
1
+ /** The first `bytes` bytes of a SHA-256, in hex. Web Crypto exists in Node 20+, Workers, Deno and Bun. */
2
+ export declare function sha256Hex(text: string, bytes?: number): Promise<string>;
3
+ /** Runs the functions it is given one at a time, so two calls in the same process cannot read the same
4
+ * counter and both write back. It does nothing for two processes: only an atomic store can do that. */
5
+ export declare function createMutex(): <T>(fn: () => Promise<T>) => Promise<T>;
6
+ export declare const isRecord: (v: unknown) => v is Record<string, unknown>;
7
+ export declare const isNonNegative: (v: unknown) => v is number;
package/dist/util.js ADDED
@@ -0,0 +1,17 @@
1
+ /** The first `bytes` bytes of a SHA-256, in hex. Web Crypto exists in Node 20+, Workers, Deno and Bun. */
2
+ export async function sha256Hex(text, bytes = 16) {
3
+ const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(text)));
4
+ return [...digest.slice(0, bytes)].map((b) => b.toString(16).padStart(2, '0')).join('');
5
+ }
6
+ /** Runs the functions it is given one at a time, so two calls in the same process cannot read the same
7
+ * counter and both write back. It does nothing for two processes: only an atomic store can do that. */
8
+ export function createMutex() {
9
+ let tail = Promise.resolve();
10
+ return (fn) => {
11
+ const run = tail.then(fn, fn);
12
+ tail = run.catch(() => undefined);
13
+ return run;
14
+ };
15
+ }
16
+ export const isRecord = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
17
+ export const isNonNegative = (v) => typeof v === 'number' && Number.isFinite(v) && v >= 0;
@@ -0,0 +1,141 @@
1
+ # Free Model Index: spec (draft 1, 2 Nov 2026)
2
+
3
+ > **Status:** milestone 1 (harness, validity rule, outcome classification, daily rollup) is built and public at https://github.com/obasaolorunfemi-ai/free-model-index. The scheduler, `scorecard.json`, the page and the Naija Eval join are not.
4
+
5
+ A public, open, scheduled measurement of how reliably OpenRouter's free models answer a real request, published as an open dataset, a methodology page and a weekly update. It is the Lodestone version of what makes SemiAnalysis's InferenceX worth citing: every number comes from a public run anyone can inspect, and it is updated continuously.
6
+
7
+ ## What it is, and is not
8
+
9
+ - **Is:** for each free model, how often a probe got a usable answer inside a serverless time budget, how fast, and how it failed when it did not. One uniform probe, so the rates are comparable between models.
10
+ - **Is not:** a quality ranking. A fast, reliable model can still be a poor translator. Quality lives in Naija Eval (below) and, for other tasks, nowhere yet.
11
+ - **Is not:** a measurement of paid models, or of every region. The probes run from GitHub's runners, not from Lagos.
12
+ - **Differs from the "weather" feed:** weather counts what one app's traffic happened to try, so a model is tried only because earlier ones failed, and its rate says little. The index sends the same request to every model on a schedule, so a rate here means something.
13
+
14
+ ## Name, versions, cadence
15
+
16
+ - **Name:** Free Model Index. Avoid the "MAX" and "X" suffixes SemiAnalysis uses on its own named products.
17
+ - **Methodology versions:** `v1.0` now. Any change to the probe, the validity rule or the timeout bumps the version; every datapoint carries the version it was taken under, and rates are never compared across versions without saying so.
18
+ - **Weekly updates:** numbered, published Mondays (Update 1, 2, ...). Each is the JSON for the week, a short written summary, and the list of models added or withdrawn.
19
+ - **Announce versions, not the daily data.** `v1` is the first public week with at least 14 days of history.
20
+
21
+ ## Where it lives, and licences
22
+
23
+ - **Repo:** a new public repo, `free-model-index`, under the same account. Separate from the library so the index can be cited and forked on its own, and so the library's releases do not depend on it.
24
+ - **Code (probe harness, aggregation, site page):** Apache-2.0, like InferenceX's benchmark repo.
25
+ - **Data:** CC BY 4.0, so a citation is the price of reuse. Attribution is the point.
26
+ - **README rule, borrowed from InferenceX:** only this repo contains the official index. Forks and reruns are unofficial and must say so.
27
+ - **Branch layout:** `main` holds code and the methodology; an append-only `data` branch holds the probe records, so the code history is not buried in one commit per run.
28
+
29
+ ## The probe
30
+
31
+ - **One fixed request**, published in the repo. A short JSON extraction (a one-line job posting to `{title, remote}`), temperature 0, `max_tokens` about 200, no reasoning option set, no system prompt. This is the kind of call Lodestone is for, and a model that "thinks" for thousands of tokens will time out, which is a real result for a serverless tool.
32
+ - **Validity rule, in code:** the answer parses as JSON, `title` mentions the role, `remote` is true. Anything else is "answered, unusable".
33
+ - **Timing:** measured to the last byte of the body, never to the headers. (28 Oct 2026: headers arrive seconds early and made every model look faster.)
34
+ - **Time limits:** a hard timeout at 45 s (the hang length seen on 28 Oct) and a "serverless budget" line at 24 s (Lodestone's default).
35
+ - **Explicit model ids only.** The harness refuses any id that does not end in `:free` and whose catalogue price is not zero, so a probe can never spend money. It does not use OpenRouter's router model.
36
+
37
+ ## Outcomes
38
+
39
+ Every probe ends in exactly one outcome, using the same names as Lodestone's failure kinds where they exist:
40
+
41
+ | Outcome | Meaning |
42
+ |---|---|
43
+ | `usable` | valid answer within 24 s |
44
+ | `usable_late` | valid answer between 24 s and the 45 s timeout |
45
+ | `unusable` | answered, but empty or failed the validity rule |
46
+ | `timeout` | no complete body within 45 s |
47
+ | `rate_limited_upstream` / `rate_limited_other` | a 429 from the model's own provider, or one we could not tell apart (classified with `readRateLimit()`) |
48
+ | `withdrawn` | 404 |
49
+ | `unavailable` | 5xx |
50
+ | `other` | anything else, with the status |
51
+ | `restricted` | 403: the model refuses direct API calls (for example it is offered only through approved apps). Recorded with OpenRouter's message, listed, never rated: it says nothing about reliability |
52
+ | `probe_budget` | our own account's daily or per-minute limit. OpenRouter's "daily" and "minute" 429s apply to the whole account, so they say nothing about a model. Recorded, never counted for or against it |
53
+
54
+ Headline metric per model per window: **usable rate** = `usable` / all counted outcomes. Also published: `usable_late`, `unusable`, `timeout` and rate-limit shares, and p50 and p95 latency of answered probes.
55
+
56
+ ## Budget: the constraint that shapes everything
57
+
58
+ OpenRouter's documented limits for `:free` models are 20 requests a minute, and 50 a day with under 10 credits purchased or 1,000 a day with 10 or more. Limits are per account, and extra keys do not raise them.
59
+
60
+ - **Use a dedicated account and key.** Probes must not share a cap with KeyStone, Inkstone, TouchStone or Naija Eval, or they would eat those tools' allowance and the tools would corrupt the index.
61
+ - **Option A, no purchase (50 a day):** on 2 Nov 2026 OpenRouter's catalogue lists 14 free text models, of which 12 are general chat models (a safety classifier and a code-only model are left out, as Lodestone leaves them out). 12 models x 3 probes a day (every 8 hours) is 36 a day, inside the cap with room for new models. Each model reaches n >= 30 in about 10 days, so the published window is trailing 14 days.
62
+ - **Option B, one-time 10-credit purchase (1,000 a day):** up to about 40 models, one probe an hour each, about 960 a day. n is about 168 a week per model, so a weekly window works. The credit is not consumed by free calls. This is the recommendation, and it is the owner's decision to spend it.
63
+ - **If there are more free models than the budget covers:** probe the most-used first (the models in Lodestone's default lists and the ones KeyStone, Inkstone and TouchStone call), and the rest on a rota, labelled "sampled".
64
+ - **Back-off:** honour `Retry-After`. Never retry a failed probe within the same hour; a retry would measure our persistence, not the model.
65
+
66
+ ## Finding new models the day they appear
67
+
68
+ - An hourly job reads OpenRouter's catalogue (not a completion, so it does not use the free-request cap), keeps the `:free` models, and diffs against the last list.
69
+ - A new free model gets its first probe in the same run and is added to the schedule. The weekly update names it ("added Tuesday 14:00 UTC").
70
+ - A model that is withdrawn (404) or whose expiry date is within a week is marked, not deleted. Its history stays.
71
+
72
+ ## Schedule and storage
73
+
74
+ - **Scheduler:** GitHub Actions cron on a public repo (standard runners are free for public repos). GitHub documents that scheduled runs can start late and that it disables schedules in a public repo with no activity for 60 days; recording the real start time of every probe and committing the results keeps it active.
75
+ - **Records:** `data/probes/YYYY-MM-DD.jsonl`, one line per probe: `{ts, version, model, outcome, status, ms, answered_chars, account_remaining}`. No model answers are stored beyond a length and the validity result; a weekly audit sample of 20 raw answers is kept for checking.
76
+ - **Rollups:** `data/daily/YYYY-MM-DD.json` and `data/weekly/YYYY-WW.json`.
77
+ - **Public file:** `scorecard.json` in the repo's published output, in `model-scorecard/1`, so Lodestone can read it unchanged:
78
+ `source: "free-model-index"`, `task: "availability"`, `metric: "usable_rate"` (and `latency_p50_ms`, `latency_p95_ms`), `n`, `measuredAt`, `slice: "openrouter-free"`.
79
+ - **Stable URL on the Lodestone site**, for example `https://lodestone.extrafemi.com/data/free-model-index.json`, with `Access-Control-Allow-Origin: *` so anyone can read it and the site's own CSP does not need a new origin.
80
+
81
+ ## The page
82
+
83
+ A page on the Lodestone site, `/free-model-index/`, built the way the Runs on Lodestone page is: static, reading the published JSON.
84
+
85
+ - A sortable table: model, usable rate, p50 and p95, timeouts, rate limits, first seen, last probed, n. Models with n < 30 in the window are listed but not rated.
86
+ - A 14-day strip per model (one cell a day) so a change is visible.
87
+ - "Updated" time, the methodology version, a link to the data and the repo, and a copy-ready citation line.
88
+ - Plain-language note at the top: "A low rate is not a bad model. These are the free tiers, which are rationed."
89
+ - Emit `Dataset` structured data and add the page to `llms.txt` and the sitemap, so search engines and AI assistants can find and quote it.
90
+
91
+ ## How Naija Eval connects
92
+
93
+ Two halves of the same question, "can I build on this free model?": the index says whether it answers at all, Naija Eval says how good the answer is in Hausa, Igbo and Yoruba.
94
+
95
+ - **Same format.** Both publish `model-scorecard/1`. Lodestone already reads several scorecards and orders models with `quality: { task, metric, language }`. A Hausa task can sort by Naija Eval's `chrf++`; any other task by the index's `usable_rate`.
96
+ - **The index gates Naija Eval's measurements.** On 28 Oct 2026 Naija Eval's free-model translation run got 0 of 180 answers from gemma-4-31b and nemotron-3.5-lightning, 4 of 180 from nemotron-3.5-super-vl, and 180 of 180 from nemotron-3-ultra. It correctly refused to publish a row of zeros. With the index, Naija Eval can run a free model's quality test only when that model has recently been usable, and can say "not measured: answers too rarely" instead of nothing.
97
+ - **The index feeds candidates both ways.** Lodestone's `model-candidates/1` already tells Naija Eval which models are in use. The index adds which free models exist and are alive, so Naija Eval benchmarks live models and skips dead ones.
98
+ - **A joined view.** One extra table, "up and good": usable rate (index) beside average chrF++ for Hausa, Igbo and Yoruba (Naija Eval, n = 100 per direction, with its date). A quality number is shown only for a model whose usable rate is above a stated floor in the same window, and always with its own n and date. Nobody else publishes this for African languages.
99
+ - **One rule, stated once:** quality and reliability stay separate columns and are never blended into one score.
100
+ - **Account separation again.** Naija Eval's all-day free runs also draw on an OpenRouter cap. It must use its own key, not the index's.
101
+
102
+ ## Methodology and independence note (published with v1)
103
+
104
+ - What a probe is, the exact prompt, the validity rule, the timeouts, the outcome table, the window, the minimum n.
105
+ - Limits, stated up front: one prompt; GitHub's runners and region; free tiers are rationed and change without notice; ranks are not a judgement of the model.
106
+ - Independence: we sell nothing whose price or availability depends on a model's rank. We take no payment from any model provider or gateway.
107
+ - Disclosure: Extrafemi's own tools (KeyStone, Inkstone, TouchStone) and Naija Eval use these models, and Lodestone is the library that routes around the ones that fail. That is a reason we measure this and a reason to read it critically; the harness and data are open so anyone can check.
108
+ - Corrections policy: a wrong number is corrected in the open, with the old value kept in the history.
109
+ - Nothing is implied about OpenRouter's view of the index; it has not reviewed it.
110
+
111
+ ## Funnel (later, not part of v1)
112
+
113
+ Library and index free, then hosted alerts (tell me when a model in my chain goes down or is withdrawn, using the index plus my Lodestone config), then a managed store for the circuit breaker and rate limits, then Extrafemi integration work. None of these should change what the index says.
114
+
115
+ ## Success measures
116
+
117
+ Referrals to the page and the JSON, GitHub stars and forks of the repo, npm downloads of the library in the weeks after each update, AI-crawler and assistant citations of the page, and inbound messages that mention the index. Review after 8 weekly updates.
118
+
119
+ ## Milestones
120
+
121
+ 1. **Harness and data repo:** probe, validity rule, outcome classification, dedicated key, a manual run, the raw records and the daily rollup. About 2 days.
122
+ 2. **Scheduler and catalogue watcher:** hourly or 8-hourly runs on GitHub Actions, new-model detection, the `data` branch. About 1 day.
123
+ 3. **`scorecard.json` and the page:** the table, the strip, structured data, `llms.txt`. About 2 days.
124
+ 4. **Methodology page and the first 14 days of history,** then **Update 1** and the announcement post.
125
+ 5. **Naija Eval join:** the gate and the joined table, once Naija Eval is ready to read the scorecard.
126
+
127
+ ## Risks
128
+
129
+ - **The free tier changes.** Limits, model availability and rules can change overnight. The methodology says what was true when, and the version bumps.
130
+ - **"One prompt proves nothing."** True, and stated. A second probe type (a longer reasoning-light task) is a v1.1 candidate, not a v1 requirement.
131
+ - **Our probes distort the thing measured.** A dedicated account and a once-an-hour rate keep that small, and `probe_budget` outcomes are excluded.
132
+ - **Cloudflare or an upstream blocks the runners.** Recorded as `other` with the status. If it is persistent, move the scheduler to a Worker cron, which is also free.
133
+ - **Credibility.** The index is only as good as the care in the methodology. Publish the failures and the corrections.
134
+ - **Terms.** OpenRouter's limits page has no rule about benchmarking, only that heavy abuse is blocked and limits are global. Re-read it before launch.
135
+
136
+ ## Open decisions for the owner
137
+
138
+ 1. Spend the one-time 10 credits on a dedicated OpenRouter account (Option B), or start on Option A with 15 models?
139
+ 2. A separate `free-model-index` repo, or a folder inside the Lodestone repo?
140
+ 3. Data licence: CC BY 4.0 (recommended) or CC0?
141
+ 4. First scorecard consumer: ship the Lodestone site page first, or the Naija Eval join first?
@@ -0,0 +1,48 @@
1
+ # Protecting your budget on a public tool
2
+
3
+ A free public tool that calls a paid model has one big risk: somebody finds the endpoint and calls it in a loop. Lodestone gives you three layers, from the one that stops one person to the one that stops everyone.
4
+
5
+ ## 1. Per caller: `rateLimit`
6
+
7
+ ```ts
8
+ import { createLodestone, createRateLimiter, upstashStore } from '@extrafemi-ai/lodestone';
9
+
10
+ const store = upstashStore({ url: process.env.UPSTASH_URL!, token: process.env.UPSTASH_TOKEN! });
11
+ const ai = createLodestone({
12
+ /* ... */
13
+ rateLimit: {
14
+ limiter: createRateLimiter({ store, max: 20, windowMs: 3_600_000 }), // 20 calls an hour each
15
+ whenOver: 'free-only', // or 'reject' (the default)
16
+ },
17
+ });
18
+
19
+ await ai.complete({ prompt, caller: userId ?? clientIp });
20
+ ```
21
+
22
+ - A call that names a `caller` counts against that caller's allowance. A call with no `caller`, a call on the caller's own key (their money) and an explicit `route` are not limited. **Pass `caller` on every public call.**
23
+ - Over the allowance, `whenOver: 'reject'` fails with `kind: 'throttled'` and `err.detail.retryAfterMs`; `explain(err)` says "Try again in about N minutes". `'free-only'` still answers, from free models only, so the abuser can never cost you anything.
24
+ - Only a SHA-256 hash of the caller is stored, never the caller, so an IP address does not end up in your store.
25
+ - Exactness depends on the store. With `upstashStore` (or `memoryStore` in one process) the count is atomic. With a plain get/set store it is exact inside one process and approximate between processes: two functions answering the same caller at the same moment can let a call or two through. For a tool that is attacked, use Upstash.
26
+ - If the store cannot be reached the limiter lets the call through by default (`onStoreError: 'allow'`) so a broken store does not take your tool down; the spend limit below is the backstop. Set `'deny'` to fail closed.
27
+
28
+ ## 2. Everyone together: `createSpendLimit`
29
+
30
+ ```ts
31
+ const spend = createSpendLimit({ store, dailyUsd: 2, monthlyUsd: 30, holdUsd: 0.01 });
32
+ const ai = createLodestone({ /* ... */, spend });
33
+ await spend.report(); // { todayUsd, monthUsd, heldUsd, ... }
34
+ ```
35
+
36
+ A plain "is the total under the limit?" check lets a burst through: fifty requests all read the same total, all pass, all spend. `createSpendLimit` takes a **hold** (`holdUsd`, roughly the most one call can cost) for each paid attempt when it is allowed, and counts holds against the limit, so concurrent requests see each other. The hold is replaced by the real cost when the attempt answers and given back when it does not; a hold that is never settled (the function was killed) expires after two minutes.
37
+
38
+ Be honest about the limits of it: inside one process it is exact. Between processes, with a get/set store, two functions can each take a hold the other did not see, so leave headroom (set the limit a little under what you can afford). If the ledger cannot be read, the paid attempt is refused.
39
+
40
+ Bring your own: `spend` is just `{ allow, record, release? }`, and `allow`/`record`/`release` receive an `id` that names one attempt so you can do the same with a database that has transactions.
41
+
42
+ ## 3. The ceiling on one call
43
+
44
+ `maxTokens` (3,500 by default) bounds what one call can produce, and `budgetMs` bounds how long it can run. Set `maxTokens` per task (`profiles`) to what the task needs.
45
+
46
+ ## What this does not do
47
+
48
+ Lodestone cannot tell a real person from a script: `caller` is whatever you pass. If your callers are anonymous, an IP address is a weak identity (shared networks, rotating addresses). For a tool that is attacked, put a bot check (a CAPTCHA or a signed token) in front of the endpoint, and treat the layers above as the part that limits the damage when one gets through.
@@ -0,0 +1,19 @@
1
+ # Runtimes
2
+
3
+ Lodestone uses only web-standard APIs (`fetch`, `Response`, `AbortSignal`, `URL`, `crypto.subtle`, `TextEncoder`) and has no dependencies, so it is not tied to Node.
4
+
5
+ | Runtime | How it is checked | Where |
6
+ |---|---|---|
7
+ | Node 20, 22 and 24 | the full test suite and the smoke checks | `npm test`, `npm run smoke` |
8
+ | Cloudflare Workers | the smoke checks run inside `workerd` through Miniflare, **without** the `nodejs_compat` flag | `npm run smoke:workers` |
9
+ | Deno | the smoke checks | `deno run --allow-read scripts/smoke.mjs` |
10
+ | Bun | the smoke checks | `bun scripts/smoke.mjs` |
11
+
12
+ All four run in CI on every push. The smoke checks (`scripts/smoke-core.mjs`) are the same on every runtime: failover, cutting off a model that hangs, `json()`, `explain()`, the circuit breaker, the spend limit under a burst, per-caller rate limiting, private calls not being cached, and the Upstash, Workers KV and scorecard code paths. They use fake providers, so they need no network and no keys.
13
+
14
+ This shows the library's code runs on those runtimes. It does not show that each provider's API behaves the same from each platform's network, which only your own deployment can tell you.
15
+
16
+ ## Things to know on Workers
17
+
18
+ - Use `cloudflareKvStore(env.KV)` for the catalogue and cached answers. KV is eventually consistent and about one write per second per key, so use `upstashStore` for rate-limit and spend counters.
19
+ - A Worker has CPU and subrequest limits. A call that tries three models and two providers makes up to five subrequests; keep `ACTIVE` small, or set `budgetMs` under your plan's wall-clock limit.
@@ -14,6 +14,26 @@ Two small JSON formats let a model evaluator and Lodestone feed each other witho
14
14
 
15
15
  Nothing here is automatic or hidden. A scorecard changes the order models are *tried* in, for tasks you name, and nothing else. A model nobody measured is never dropped, added or moved.
16
16
 
17
+ ## Why Naija Eval is a good evaluator to start with
18
+
19
+ [Naija Eval](https://naija-eval.extrafemi.com) tests Nigeria's own language model, N-ATLaS, against the Llama model it was built from and four hosted models, on Hausa, Igbo and Yoruba. Four things make it a sound first source for Lodestone, and its limits are part of the answer.
20
+
21
+ - **It measures what a price list cannot tell you.** Whether a model can translate between English and Hausa, Igbo or Yoruba, or transcribe speech in those languages, is not something a provider's own claims settle. Naija Eval scores each model on six translation directions (chrF++) and three speech languages (word error rate), and also shows each score ignoring tone marks, so a missing mark can be told apart from a wrong word.
22
+ - **Anyone can re-run it.** It uses public test sets (Google's FLEURS, CC BY 4.0, and Meta's FLORES, CC BY-SA 4.0), one fixed prompt for every model and temperature 0. A script for each step, the results file (`summary.json`) and the scorecard (`scorecard.json`) are public, and repeating the four hosted-model runs cost $1.59. A number that can be repeated can also be checked, and corrected.
23
+ - **It states what it did not do.** Each score rests on 100 items per language and direction, and Naija Eval calls itself a first pass, not a leaderboard. The 4-bit local models may score a point or two below full precision. The test sets are public, so any model may have seen them in training. FLEURS is clean read speech, and street voice notes will score differently.
24
+ - **It publishes every number with what it rests on.** In `model-scorecard/1` each score carries its metric, which way is good, the number of items (`n`) and the day it was measured, so Lodestone never has to guess how much weight a figure deserves.
25
+
26
+ ## Why its results are reliable enough for Lodestone
27
+
28
+ "Reliable" here means something narrow. Lodestone uses a scorecard to put models in a better order for a task you name, and it is built so that a wrong number cannot do much harm:
29
+
30
+ - A scorecard only changes the *order* models are tried in. Nothing is added, dropped or called that you did not list.
31
+ - A score resting on fewer than 30 items, or measured more than a year ago, is ignored. Naija Eval's scores rest on 100 items each.
32
+ - A model nobody measured keeps its slot, an explicit `route` is never reordered, and if the scorecard cannot be fetched within 1.5 seconds the order is left alone.
33
+ - The worst a bad number can do is put a slightly worse model first for that task. A failure or a slow answer still moves the call to the next model, as it always does.
34
+
35
+ What it does not tell you: Naija Eval measures translation and speech in Hausa, Igbo and Yoruba, so use it only for tasks you tag that way. It says nothing about extracting a job posting or scoring a CV in English. And one disclosure: Naija Eval and Lodestone are both Extrafemi projects, so the evaluator is not independent of Lodestone's author. The protection is that its method and data are public, so anyone can check it.
36
+
17
37
  ## `model-scorecard/1`
18
38
 
19
39
  Measured results about models.
package/docs/stores.md ADDED
@@ -0,0 +1,45 @@
1
+ # Stores
2
+
3
+ Lodestone keeps a little state: the model catalogue, which models are failing (the circuit breaker), saved answers, and, if you use them, rate-limit and spend counters. By default that state lives in memory, which is lost when a serverless function ends and is not shared between functions. For anything real, give Lodestone a store.
4
+
5
+ A store is two methods:
6
+
7
+ ```ts
8
+ interface KeyValueStore {
9
+ get(key: string): Promise<unknown>;
10
+ set(key: string, value: unknown, opts?: { ttlMs?: number }): Promise<void>;
11
+ }
12
+ ```
13
+
14
+ `ttlMs` says how long a value is worth keeping; a store that cannot expire values may ignore it. A store that can also count atomically (`incrBy`) makes the rate limiter exact across every function. Ready-made adapters need no client library: they take the binding or token you already have.
15
+
16
+ ```ts
17
+ import { createHealth, createLodestone, netlifyBlobsStore } from '@extrafemi-ai/lodestone';
18
+ import { getStore } from '@netlify/blobs';
19
+
20
+ const store = netlifyBlobsStore(getStore('lodestone'));
21
+ const ai = createLodestone({ store, health: createHealth({ store }), /* ... */ });
22
+ ```
23
+
24
+ | Adapter | Use it on | Expires values | Counts atomically | Notes |
25
+ |---|---|---|---|---|
26
+ | `netlifyBlobsStore(getStore('x'))` | Netlify Functions | by Lodestone (a value carries its own expiry) | no | Fine for the catalogue, cached answers and the breaker. No atomic add, and writes from separate functions can overwrite each other. |
27
+ | `cloudflareKvStore(env.KV)` | Cloudflare Workers | by KV | no | Eventually consistent, and one key can be written about once a second. Fine for the catalogue and cached answers; the breaker learns a little slower. Not for counters. |
28
+ | `upstashStore({ url, token })` | anywhere with `fetch`, including Vercel KV | by Redis | **yes** | The one to use for per-caller rate limits and spend counters. Plain `fetch`, so it runs on Workers, Deno, Bun and Node. |
29
+ | `denoKvStore(await Deno.openKv())` | Deno and Deno Deploy | by Deno KV | no | |
30
+ | `memoryStore()` | tests, one long-running process | yes | yes (in this process) | The default. |
31
+
32
+ ## Which store for which job
33
+
34
+ | Job | A plain store (Blobs, KV, Deno KV) | Upstash |
35
+ |---|---|---|
36
+ | Model catalogue, cached answers | fine | fine |
37
+ | Circuit breaker | fine | fine |
38
+ | Per-caller rate limit | exact inside one process, approximate between processes | exact |
39
+ | Spend limit (`createSpendLimit`) | exact inside one process, approximate between processes | as good as a get/set ledger can be (see [protecting your budget](protecting-your-budget.md)) |
40
+
41
+ ## Writing your own
42
+
43
+ Anything that gets and sets JSON by key will do. Keys contain colons (`lodestone:health`); if your store dislikes them, `safeKey(key)` returns a version it will accept (anything outside letters, digits, `.`, `_` and `-` becomes `~` and two hex digits, so two different keys never collide).
44
+
45
+ What comes back from a store is checked before it is used: a catalogue, breaker record or ledger of the wrong shape or an absurd size is ignored, so a store that someone else can write to cannot make Lodestone hide every model or grow without limit.
@@ -1,20 +1,22 @@
1
1
  // A Netlify function that asks a model for JSON, written so it keeps working when a model is
2
2
  // withdrawn, slow or rate-limited. Needs OPENROUTER_API_KEY (and, for a second route, GROQ_API_KEY).
3
- import { createHealth, createLodestone, explain, groq, LodestoneError } from '@extrafemi-ai/lodestone';
3
+ import { createHealth, createLodestone, createRateLimiter, createSpendLimit, explain, groq, LodestoneError, netlifyBlobsStore, upstashStore } from '@extrafemi-ai/lodestone';
4
4
  import { getStore } from '@netlify/blobs';
5
5
  import type { Handler } from '@netlify/functions';
6
6
 
7
7
  // Where Lodestone keeps what it learns: the model catalogue, which models are failing, saved answers.
8
- // Blob keys cannot contain colons, so they are made simple.
9
- const blobs = getStore('lodestone');
10
- const simple = (k: string) => k.replace(/[^a-z0-9._-]/gi, '-');
11
- export const store = { get: (k: string) => blobs.get(simple(k), { type: 'json' }), set: (k: string, v: unknown) => blobs.setJSON(simple(k), v) };
8
+ // Netlify Blobs cannot count atomically, so the per-caller limit and the spend ledger go in Upstash when it is set up
9
+ // (UPSTASH_URL and UPSTASH_TOKEN); without it they are exact inside one function instance and approximate between instances.
10
+ export const store = netlifyBlobsStore(getStore('lodestone'));
11
+ const counters = process.env.UPSTASH_URL ? upstashStore({ url: process.env.UPSTASH_URL, token: process.env.UPSTASH_TOKEN! }) : store;
12
12
 
13
13
  export const ai = createLodestone({
14
14
  openrouter: { apiKey: process.env.OPENROUTER_API_KEY },
15
15
  fallbacks: process.env.GROQ_API_KEY ? [{ provider: groq(), apiKey: process.env.GROQ_API_KEY, models: ['llama-3.3-70b-versatile'], free: true }] : [],
16
16
  store,
17
17
  health: createHealth({ store }), // a model that keeps failing is skipped by every function until it recovers
18
+ spend: createSpendLimit({ store: counters, dailyUsd: 2 }), // holds, so a burst cannot overshoot
19
+ rateLimit: { limiter: createRateLimiter({ store: counters, max: 20 }), whenOver: 'free-only' }, // one person cannot empty it
18
20
  budgetMs: 24_000, // Netlify cuts a function off at 30 seconds
19
21
  });
20
22
 
@@ -26,10 +28,11 @@ export const handler: Handler = async (event) => {
26
28
  const { text } = JSON.parse(event.body ?? '{}') as { text?: string };
27
29
  if (!text) return { statusCode: 400, body: JSON.stringify({ error: 'text is required' }) };
28
30
  // A model whose answer is not valid JSON of this shape is skipped for the next one.
29
- const r = await ai.json<Summary>({ prompt: `Summarise as JSON {"title": string, "points": string[]}:\n\n${text}`, tier: 'fast', validate: isSummary });
31
+ const caller = event.headers['x-nf-client-connection-ip'] ?? 'unknown'; // name who is calling, so each caller has their own allowance
32
+ const r = await ai.json<Summary>({ prompt: `Summarise as JSON {"title": string, "points": string[]}:\n\n${text}`, tier: 'fast', validate: isSummary, caller });
30
33
  return { statusCode: 200, body: JSON.stringify(r.data) };
31
34
  } catch (err) {
32
35
  // A sentence a person can be shown, not a stack trace.
33
- return { statusCode: err instanceof LodestoneError && err.kind === 'rate_limit' ? 429 : 502, body: JSON.stringify({ error: explain(err, { product: 'MyApp' }) }) };
36
+ return { statusCode: err instanceof LodestoneError && (err.kind === 'rate_limit' || err.kind === 'throttled') ? 429 : 502, body: JSON.stringify({ error: explain(err, { product: 'MyApp' }) }) };
34
37
  }
35
38
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@extrafemi-ai/lodestone",
3
- "version": "0.6.0",
3
+ "version": "0.7.1",
4
4
  "description": "Calls AI models so your tool keeps working when a model is withdrawn, slow, rate-limited or down: failover across providers, self-updating model lists, spend limits.",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -40,10 +40,13 @@
40
40
  "test": "vitest run",
41
41
  "prepublishOnly": "npm run typecheck && npm run typecheck:examples && npm test && npm run build",
42
42
  "typecheck:examples": "tsc -p tsconfig.examples.json",
43
- "site": "npm run build && node site/build.mjs"
43
+ "site": "npm run build && node site/build.mjs",
44
+ "smoke": "npm run build && node scripts/smoke.mjs",
45
+ "smoke:workers": "node scripts/run-workers.mjs"
44
46
  },
45
47
  "devDependencies": {
46
48
  "@types/node": "^22.10.0",
49
+ "miniflare": "^4.20260730.0",
47
50
  "typescript": "^5.6.0",
48
51
  "vitest": "^4.1.12"
49
52
  },