@extrafemi-ai/lodestone 0.6.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +44 -21
  3. package/SECURITY.md +17 -0
  4. package/dist/catalog.js +11 -2
  5. package/dist/errors.d.ts +6 -0
  6. package/dist/errors.js +1 -2
  7. package/dist/explain.js +4 -0
  8. package/dist/health.js +19 -3
  9. package/dist/index.d.ts +10 -4
  10. package/dist/index.js +4 -1
  11. package/dist/limiter.d.ts +34 -0
  12. package/dist/limiter.js +35 -0
  13. package/dist/lodestone.d.ts +25 -12
  14. package/dist/lodestone.js +56 -10
  15. package/dist/providers/anthropic.d.ts +2 -0
  16. package/dist/providers/anthropic.js +1 -1
  17. package/dist/providers/openaiCompatible.d.ts +2 -0
  18. package/dist/providers/openaiCompatible.js +2 -0
  19. package/dist/scorecard.d.ts +9 -3
  20. package/dist/scorecard.js +45 -2
  21. package/dist/spend.d.ts +33 -0
  22. package/dist/spend.js +72 -0
  23. package/dist/store.d.ts +2 -2
  24. package/dist/store.js +19 -3
  25. package/dist/stores/cloudflare.d.ts +10 -0
  26. package/dist/stores/cloudflare.js +11 -0
  27. package/dist/stores/deno.d.ts +14 -0
  28. package/dist/stores/deno.js +8 -0
  29. package/dist/stores/index.d.ts +9 -0
  30. package/dist/stores/index.js +5 -0
  31. package/dist/stores/netlify.d.ts +12 -0
  32. package/dist/stores/netlify.js +9 -0
  33. package/dist/stores/safeKey.d.ts +10 -0
  34. package/dist/stores/safeKey.js +20 -0
  35. package/dist/stores/upstash.d.ts +14 -0
  36. package/dist/stores/upstash.js +50 -0
  37. package/dist/types.d.ts +15 -3
  38. package/dist/util.d.ts +7 -0
  39. package/dist/util.js +17 -0
  40. package/docs/free-model-index-spec.md +141 -0
  41. package/docs/protecting-your-budget.md +48 -0
  42. package/docs/runtimes.md +19 -0
  43. package/docs/scorecards.md +20 -0
  44. package/docs/stores.md +45 -0
  45. package/examples/netlify-function.ts +10 -7
  46. package/package.json +5 -2
package/dist/lodestone.js CHANGED
@@ -10,6 +10,7 @@ import { google } from './providers/google.js';
10
10
  import { openrouter } from './providers/openrouter.js';
11
11
  import { rankModels } from './scorecard.js';
12
12
  import { memoryStore } from './store.js';
13
+ import { sha256Hex } from './util.js';
13
14
  const hasImage = (messages) => messages.some((m) => Array.isArray(m.content) && m.content.some((p) => p.type === 'image'));
14
15
  /** How many candidates are kept in the background for a tier. A request still tries at most three. */
15
16
  const POOL = 8;
@@ -60,13 +61,19 @@ export function createLodestone(config = {}) {
60
61
  /** A fallback provider's models for a tier: the app's own, pruned and topped up from the provider's catalogue. */
61
62
  async function resolveFallback(fb, tier, strict) {
62
63
  const cat = fbCatalogs.get(fb);
64
+ // A private call may only go where something says prompts are not kept or trained on: the app's own word, or the provider's list.
65
+ const vouched = strict && fb.privacy === 'no-training';
63
66
  if (!cat)
64
- return fb.models;
67
+ return strict && !vouched ? [] : fb.models;
65
68
  try {
66
- return resolveProviderModels(fb.models, await cat.load(), { tier, policy, strict, fill: fb.discover?.fill, max: fb.discover?.max, now: now() });
69
+ const listed = await cat.load();
70
+ if (strict && !vouched && !listed?.length)
71
+ return []; // no list, so nothing to check the models against
72
+ const p = strict && !vouched ? { ...policy, requireNoTraining: true } : policy;
73
+ return resolveProviderModels(fb.models, listed, { tier, policy: p, strict: strict && !vouched, fill: fb.discover?.fill, max: fb.discover?.max, now: now() });
67
74
  }
68
75
  catch {
69
- return fb.models;
76
+ return strict && !vouched ? [] : fb.models;
70
77
  }
71
78
  }
72
79
  /** The scorecards, or none: never throws and never holds a request up for long. */
@@ -108,15 +115,29 @@ export function createLodestone(config = {}) {
108
115
  throw new LodestoneError('aborted');
109
116
  const quality = opts.quality ?? profile?.quality;
110
117
  const order = async (models) => (quality && models.length > 1 ? rankModels(models, await cards(), quality, { now }) : models);
111
- const cacheSlot = opts.cacheKey ? `lodestone:cache:${opts.cacheKey}` : null;
118
+ // A private call is never read from or written to the cache: an answer made from a CV must not sit in a shared store.
119
+ const cacheSlot = opts.cacheKey && !strict ? `lodestone:cache:${opts.cacheKey}` : null;
120
+ const cacheTtl = opts.cacheTtlMs ?? 24 * 3_600_000;
121
+ const fingerprint = cacheSlot ? await sha256Hex(JSON.stringify([opts.system ?? '', messages, tier, opts.task ?? ''])) : '';
112
122
  if (cacheSlot) {
113
123
  try {
114
124
  const hit = (await store.get(cacheSlot));
115
- if (hit?.text && now() - hit.at < (opts.cacheTtlMs ?? 24 * 3_600_000) && (!opts.accept || opts.accept(hit.text)))
125
+ const mine = typeof hit?.text === 'string' && typeof hit.at === 'number' && hit.fp === fingerprint; // an answer for different input is a miss
126
+ if (mine && now() - hit.at < cacheTtl && (!opts.accept || opts.accept(hit.text)))
116
127
  return { text: hit.text, provider: hit.provider, model: hit.model, cached: true };
117
128
  }
118
129
  catch { /* a store that cannot be read is a cache miss */ }
119
130
  }
131
+ // Each caller has their own allowance, so one of them cannot empty the platform's budget for the rest.
132
+ let throttled = null;
133
+ if (config.rateLimit && opts.caller && !opts.apiKey && !opts.route?.length) {
134
+ const r = await config.rateLimit.limiter.take(opts.caller).catch(() => null);
135
+ if (r && !r.ok) {
136
+ if ((config.rateLimit.whenOver ?? 'reject') === 'reject')
137
+ throw new LodestoneError('throttled', { retryAfterMs: r.retryAfterMs });
138
+ throttled = { retryAfterMs: r.retryAfterMs };
139
+ }
140
+ }
120
141
  const started = now();
121
142
  const left = () => budget - (now() - started);
122
143
  const orKey = opts.apiKey || config.openrouter?.apiKey;
@@ -147,11 +168,22 @@ export function createLodestone(config = {}) {
147
168
  return keep;
148
169
  };
149
170
  /** May the platform's money be spent on this attempt? A caller's own key is their money, not ours. */
171
+ let held = null;
150
172
  const mayUse = async (provider, model, phase, free = false) => {
151
- if (callerKey || free || !config.spend)
173
+ if (callerKey || free)
174
+ return true;
175
+ // A caller over their allowance (with 'free-only') costs the platform nothing.
176
+ if (throttled) {
177
+ config.onAttempt?.({ provider: provider.name, model, phase, outcome: 'skipped', ms: 0 });
178
+ return false;
179
+ }
180
+ if (!config.spend)
152
181
  return true;
153
- if (await config.spend.allow({ provider: provider.name, model, phase }))
182
+ const info = { id: crypto.randomUUID(), provider: provider.name, model, phase };
183
+ if (await config.spend.allow(info)) {
184
+ held = info;
154
185
  return true;
186
+ }
155
187
  spendBlocked = true;
156
188
  config.onAttempt?.({ provider: provider.name, model, phase, outcome: 'skipped', ms: 0 });
157
189
  return false;
@@ -182,6 +214,15 @@ export function createLodestone(config = {}) {
182
214
  else
183
215
  out = { kind: 'fail', status: 599, body: e.message };
184
216
  }
217
+ // An attempt that did not answer gives back the hold it took on the spend limit.
218
+ const hold = held;
219
+ if (out.kind !== 'ok' && hold && hold.provider === provider.name && hold.model === model) {
220
+ held = null;
221
+ try {
222
+ await config.spend?.release?.(hold);
223
+ }
224
+ catch { /* the hold expires by itself */ }
225
+ }
185
226
  if (out.kind === 'aborted')
186
227
  throw new LodestoneError('aborted'); // the person left: nothing more to try
187
228
  const event = { provider: provider.name, model, phase, outcome: out.kind === 'ok' ? 'ok' : out.kind === 'timeout' ? 'timeout' : 'failed', status: out.kind === 'fail' ? out.status : undefined, ms: now() - at };
@@ -200,14 +241,17 @@ export function createLodestone(config = {}) {
200
241
  const result = { text: reply.text, provider: provider.name, model, usage: reply.usage };
201
242
  if (phase && !callerKey && !free && config.spend) {
202
243
  const costUsd = reply.usage?.costUsd ?? (await estimateCost(model, reply.usage));
244
+ const hold = held;
245
+ const id = hold && hold.provider === provider.name && hold.model === model ? hold.id : crypto.randomUUID();
246
+ held = null;
203
247
  // A ledger that fails must never fail an answer the person is waiting for.
204
248
  try {
205
- await config.spend.record({ provider: provider.name, model, phase, usage: reply.usage, costUsd });
249
+ await config.spend.record({ id, provider: provider.name, model, phase, usage: reply.usage, costUsd });
206
250
  }
207
251
  catch { /* not counted */ }
208
252
  }
209
253
  if (cacheSlot)
210
- await store.set(cacheSlot, { ...result, at: now() }).catch(() => { });
254
+ await store.set(cacheSlot, { ...result, at: now(), fp: fingerprint }, { ttlMs: cacheTtl }).catch(() => { });
211
255
  return result;
212
256
  };
213
257
  // An explicit route: exactly the steps the caller named, in order.
@@ -339,6 +383,8 @@ export function createLodestone(config = {}) {
339
383
  }
340
384
  // Nothing answered. Say why, using what the first useful failure told us.
341
385
  const ownKey = Boolean(opts.apiKey);
386
+ if (throttled)
387
+ throw new LodestoneError('throttled', throttled);
342
388
  if (spendBlocked)
343
389
  throw new LodestoneError('budget');
344
390
  if (!attempted)
@@ -416,7 +462,7 @@ export function createLodestone(config = {}) {
416
462
  }
417
463
  return { ok: shortTiers.length === 0, catalogSize: models.length, withdrawn, expiring, shortTiers, ...(Object.keys(providers).length ? { providers } : {}) };
418
464
  }
419
- /** An answer already saved under this key, without calling any model: so a caller can serve it without counting it as a request. */
465
+ /** An answer already saved under this key, without calling any model: so a caller can serve it without counting it as a request. It cannot see the input, so the key must stand for all of it; complete() checks the input itself and will not serve an answer made for something else. */
420
466
  async function peek(cacheKey, ttlMs = 24 * 3_600_000) {
421
467
  try {
422
468
  const hit = (await store.get(`lodestone:cache:${cacheKey}`));
@@ -1,6 +1,8 @@
1
1
  import type { Provider } from '../types.js';
2
2
  export interface AnthropicOptions {
3
3
  fetch?: typeof fetch;
4
+ /** Send the request's temperature. Default false. Anthropic's current models reject a temperature: Claude 4.7 and later answer 400 to a non-default one, and API version 2026-10-21 removes the parameter for every model. Lodestone's calls default to 0.2, so by default it is left out and the model's own setting applies. Turn this on only for an older model that still accepts one. */
5
+ sendTemperature?: boolean;
4
6
  }
5
7
  /** Anthropic's own API, for use as a fallback with a separate key from OpenRouter. */
6
8
  export declare function anthropic(opts?: AnthropicOptions): Provider;
@@ -22,7 +22,7 @@ export function anthropic(opts = {}) {
22
22
  body: JSON.stringify({
23
23
  model: req.model,
24
24
  max_tokens: req.maxTokens,
25
- ...(req.temperature === undefined ? {} : { temperature: req.temperature }),
25
+ ...(opts.sendTemperature && req.temperature !== undefined ? { temperature: req.temperature } : {}),
26
26
  ...(req.system ? { system: req.system } : {}),
27
27
  messages: req.messages.map(wire),
28
28
  }),
@@ -38,6 +38,8 @@ export declare const requesty: (o?: PresetOptions & {
38
38
  export declare const llmapi: (o?: PresetOptions) => Provider;
39
39
  /** https://groq.com: fast inference with a free developer tier. */
40
40
  export declare const groq: (o?: PresetOptions) => Provider;
41
+ /** https://omniakey.com: a prepaid gateway to text, image, video and music models, with an OpenAI-compatible text endpoint (api.omniakey.com/v1). Its model list needs a key (401 without one), and its site says it does not switch a request to a different model by itself, which is what Lodestone does on top of it. Not yet called live. */
42
+ export declare const omniakey: (o?: PresetOptions) => Provider;
41
43
  /** https://cerebras.ai: fast inference with a free tier. */
42
44
  export declare const cerebras: (o?: PresetOptions) => Provider;
43
45
  export declare const openai: (o?: PresetOptions) => Provider;
@@ -89,6 +89,8 @@ export const requesty = (o = {}) => openaiCompatible({ name: 'requesty', baseUrl
89
89
  export const llmapi = (o = {}) => openaiCompatible({ name: 'llmapi', baseUrl: 'https://api.llmapi.ai/v1', normalize: normalizeLlmapi, ...o });
90
90
  /** https://groq.com: fast inference with a free developer tier. */
91
91
  export const groq = (o = {}) => openaiCompatible({ name: 'groq', baseUrl: 'https://api.groq.com/openai/v1', normalize: normalizeGroq, ...o });
92
+ /** https://omniakey.com: a prepaid gateway to text, image, video and music models, with an OpenAI-compatible text endpoint (api.omniakey.com/v1). Its model list needs a key (401 without one), and its site says it does not switch a request to a different model by itself, which is what Lodestone does on top of it. Not yet called live. */
93
+ export const omniakey = (o = {}) => openaiCompatible({ name: 'omniakey', baseUrl: 'https://api.omniakey.com/v1', ...o });
92
94
  /** https://cerebras.ai: fast inference with a free tier. */
93
95
  export const cerebras = (o = {}) => openaiCompatible({ name: 'cerebras', baseUrl: 'https://api.cerebras.ai/v1', ...o });
94
96
  export const openai = (o = {}) => openaiCompatible({ name: 'openai', baseUrl: 'https://api.openai.com/v1', ...o });
@@ -51,11 +51,17 @@ export declare class ScorecardError extends Error {
51
51
  * ScorecardError listing what is wrong. A scorecard with a schema version this
52
52
  * library doesn't know is refused, not guessed at. */
53
53
  export declare function parseScorecard(raw: unknown): Scorecard;
54
- /** Fetches and checks a scorecard from a URL. Throws if it can't be read or isn't valid. */
55
- export declare function fetchScorecard(url: string, opts?: {
54
+ export interface FetchScorecardOptions {
56
55
  fetch?: typeof fetch;
57
56
  timeoutMs?: number;
58
- }): Promise<Scorecard>;
57
+ /** Refuse a body bigger than this. Default 1 MB. */
58
+ maxBytes?: number;
59
+ /** Only fetch from these hosts (the final host, after any redirect). Default: any host. */
60
+ allowedHosts?: string[];
61
+ }
62
+ /** Fetches and checks a scorecard from a URL. Throws if it can't be read, is too big, comes from a host you did not allow, or isn't valid.
63
+ * Only https is accepted (http only for localhost). A scorecard can only change the order models are tried in, but a body of any size from anywhere is still not worth reading. */
64
+ export declare function fetchScorecard(url: string, opts?: FetchScorecardOptions): Promise<Scorecard>;
59
65
  /** An id without its variant suffix: anthropic/claude-x:free is the same model as anthropic/claude-x. */
60
66
  export declare const baseModelId: (id: string) => string;
61
67
  /** The score for a model on a query, or undefined if nobody measured it well enough.
package/dist/scorecard.js CHANGED
@@ -12,6 +12,7 @@ export class ScorecardError extends Error {
12
12
  this.name = 'ScorecardError';
13
13
  }
14
14
  }
15
+ const MAX_SCORES = 20_000;
15
16
  const isStr = (v) => typeof v === 'string' && v.length > 0;
16
17
  const isNum = (v) => typeof v === 'number' && Number.isFinite(v);
17
18
  const isDay = (v) => typeof v === 'string' && /^\d{4}-\d{2}-\d{2}/.test(v) && !Number.isNaN(Date.parse(v));
@@ -31,6 +32,8 @@ export function parseScorecard(raw) {
31
32
  problems.push('generatedAt is missing or not a date');
32
33
  if (!Array.isArray(c.scores))
33
34
  problems.push('scores is not a list');
35
+ else if (c.scores.length > MAX_SCORES)
36
+ problems.push(`scores has more than ${MAX_SCORES} entries`);
34
37
  else
35
38
  c.scores.forEach((s, i) => {
36
39
  const at = `scores[${i}]`;
@@ -53,12 +56,52 @@ export function parseScorecard(raw) {
53
56
  throw new ScorecardError(problems);
54
57
  return c;
55
58
  }
56
- /** Fetches and checks a scorecard from a URL. Throws if it can't be read or isn't valid. */
59
+ /** Fetches and checks a scorecard from a URL. Throws if it can't be read, is too big, comes from a host you did not allow, or isn't valid.
60
+ * Only https is accepted (http only for localhost). A scorecard can only change the order models are tried in, but a body of any size from anywhere is still not worth reading. */
57
61
  export async function fetchScorecard(url, opts = {}) {
62
+ const u = new URL(url);
63
+ const local = u.hostname === 'localhost' || u.hostname === '127.0.0.1' || u.hostname === '[::1]';
64
+ if (u.protocol !== 'https:' && !(u.protocol === 'http:' && local))
65
+ throw new Error(`scorecard ${url}: only https is accepted`);
66
+ const allowed = (host) => !opts.allowedHosts || opts.allowedHosts.map((h) => h.toLowerCase()).includes(host.toLowerCase());
67
+ if (!allowed(u.hostname))
68
+ throw new Error(`scorecard ${url}: host not allowed`);
58
69
  const res = await (opts.fetch ?? fetch)(url, { signal: AbortSignal.timeout(opts.timeoutMs ?? 8_000) });
59
70
  if (!res.ok)
60
71
  throw new Error(`scorecard ${url}: ${res.status}`);
61
- return parseScorecard(await res.json());
72
+ if (res.url && !allowed(new URL(res.url).hostname))
73
+ throw new Error(`scorecard ${url}: redirected to a host that is not allowed`);
74
+ const max = opts.maxBytes ?? 1_000_000;
75
+ const declared = Number(res.headers.get('content-length'));
76
+ if (Number.isFinite(declared) && declared > max)
77
+ throw new Error(`scorecard ${url}: larger than ${max} bytes`);
78
+ let text = '';
79
+ if (res.body) {
80
+ const reader = res.body.getReader();
81
+ const chunks = [];
82
+ let size = 0;
83
+ for (;;) {
84
+ const { done, value } = await reader.read();
85
+ if (done)
86
+ break;
87
+ size += value.byteLength;
88
+ if (size > max) {
89
+ await reader.cancel().catch(() => { });
90
+ throw new Error(`scorecard ${url}: larger than ${max} bytes`);
91
+ }
92
+ chunks.push(value);
93
+ }
94
+ const all = new Uint8Array(size);
95
+ let at = 0;
96
+ for (const c of chunks) {
97
+ all.set(c, at);
98
+ at += c.byteLength;
99
+ }
100
+ text = new TextDecoder().decode(all);
101
+ }
102
+ else
103
+ text = await res.text();
104
+ return parseScorecard(JSON.parse(text));
62
105
  }
63
106
  /** An id without its variant suffix: anthropic/claude-x:free is the same model as anthropic/claude-x. */
64
107
  export const baseModelId = (id) => id.split(':')[0].toLowerCase();
@@ -0,0 +1,33 @@
1
+ import type { SpendControl } from './lodestone.js';
2
+ import type { KeyValueStore } from './types.js';
3
+ /** A daily and a monthly spend limit that holds under load. A bare `allow()` then `record()` lets a burst of
4
+ * requests all pass the check before any of them has been counted. This takes a hold for each paid attempt
5
+ * when it is allowed, and counts the holds against the limit, so concurrent requests see each other. The
6
+ * hold is replaced by the real cost when the attempt answers, and handed back when it does not. A hold that
7
+ * is never settled (the function was killed) expires by itself.
8
+ *
9
+ * Exact inside one process. Between processes it is as good as the store: with a plain get/set store, two
10
+ * functions allowing at the same moment can each take a hold the other did not see, so size `holdUsd` and
11
+ * the limit with that in mind. If the store cannot be read, the attempt is refused (the money is yours). */
12
+ export interface SpendLimitOptions {
13
+ store: KeyValueStore;
14
+ dailyUsd?: number;
15
+ monthlyUsd?: number;
16
+ /** Held for each paid attempt until it settles: roughly the most one call can cost. Default $0.01. */
17
+ holdUsd?: number;
18
+ /** A hold that is never settled goes away after this. Default 2 minutes. */
19
+ holdTtlMs?: number;
20
+ /** Where the ledger is kept. Default lodestone:spend */
21
+ key?: string;
22
+ now?: () => number;
23
+ }
24
+ export interface SpendReport {
25
+ todayUsd: number;
26
+ monthUsd: number;
27
+ heldUsd: number;
28
+ dailyUsd?: number;
29
+ monthlyUsd?: number;
30
+ }
31
+ export declare function createSpendLimit(o: SpendLimitOptions): Required<SpendControl> & {
32
+ report(): Promise<SpendReport>;
33
+ };
package/dist/spend.js ADDED
@@ -0,0 +1,72 @@
1
+ import { createMutex, isNonNegative, isRecord } from './util.js';
2
+ const MAX_HOLDS = 500;
3
+ export function createSpendLimit(o) {
4
+ const now = o.now ?? Date.now;
5
+ const key = o.key ?? 'lodestone:spend';
6
+ const hold = o.holdUsd ?? 0.01;
7
+ const ttl = o.holdTtlMs ?? 120_000;
8
+ const exclusive = createMutex();
9
+ /** What is in the store, checked: a ledger somebody else wrote is not trusted to be the right shape. */
10
+ async function load() {
11
+ const t = new Date(now());
12
+ const day = t.toISOString().slice(0, 10);
13
+ const month = day.slice(0, 7);
14
+ const raw = await o.store.get(key);
15
+ const s = isRecord(raw) ? raw : {};
16
+ const holds = {};
17
+ if (isRecord(s.holds)) {
18
+ for (const [id, h] of Object.entries(s.holds).slice(0, MAX_HOLDS)) {
19
+ if (Array.isArray(h) && isNonNegative(h[0]) && isNonNegative(h[1]) && h[1] > now())
20
+ holds[id] = [h[0], h[1]];
21
+ }
22
+ }
23
+ return {
24
+ day, month, holds,
25
+ dayUsd: s.day === day && isNonNegative(s.dayUsd) ? s.dayUsd : 0,
26
+ monthUsd: s.month === month && isNonNegative(s.monthUsd) ? s.monthUsd : 0,
27
+ };
28
+ }
29
+ const held = (l) => Object.values(l.holds).reduce((a, h) => a + h[0], 0);
30
+ const save = (l) => o.store.set(key, l);
31
+ return {
32
+ allow: (info) => exclusive(async () => {
33
+ try {
34
+ const l = await load();
35
+ const pending = held(l) + hold;
36
+ if (o.dailyUsd !== undefined && l.dayUsd + pending > o.dailyUsd)
37
+ return false;
38
+ if (o.monthlyUsd !== undefined && l.monthUsd + pending > o.monthlyUsd)
39
+ return false;
40
+ if (Object.keys(l.holds).length >= MAX_HOLDS)
41
+ return false;
42
+ l.holds[info.id] = [hold, now() + ttl];
43
+ await save(l);
44
+ return true;
45
+ }
46
+ catch {
47
+ return false;
48
+ }
49
+ }),
50
+ record: (info) => exclusive(async () => {
51
+ const l = await load();
52
+ const taken = l.holds[info.id]?.[0] ?? hold;
53
+ delete l.holds[info.id];
54
+ // A cost nobody reported counts as the hold: better to over-count than to let a free ride through.
55
+ const cost = isNonNegative(info.costUsd) ? info.costUsd : taken;
56
+ l.dayUsd += cost;
57
+ l.monthUsd += cost;
58
+ await save(l);
59
+ }),
60
+ release: (info) => exclusive(async () => {
61
+ const l = await load();
62
+ if (!(info.id in l.holds))
63
+ return;
64
+ delete l.holds[info.id];
65
+ await save(l);
66
+ }),
67
+ report: () => exclusive(async () => {
68
+ const l = await load();
69
+ return { todayUsd: l.dayUsd, monthUsd: l.monthUsd, heldUsd: held(l), dailyUsd: o.dailyUsd, monthlyUsd: o.monthlyUsd };
70
+ }),
71
+ };
72
+ }
package/dist/store.d.ts CHANGED
@@ -1,3 +1,3 @@
1
- import type { KeyValueStore } from './types.js';
1
+ import type { AtomicStore } from './types.js';
2
2
  /** A store that lives in memory. The default; lost when the process ends. */
3
- export declare function memoryStore(): KeyValueStore;
3
+ export declare function memoryStore(now?: () => number): AtomicStore;
package/dist/store.js CHANGED
@@ -1,8 +1,24 @@
1
1
  /** A store that lives in memory. The default; lost when the process ends. */
2
- export function memoryStore() {
2
+ export function memoryStore(now = Date.now) {
3
3
  const data = new Map();
4
+ const live = (key) => {
5
+ const e = data.get(key);
6
+ if (!e)
7
+ return undefined;
8
+ if (e.exp <= now()) {
9
+ data.delete(key);
10
+ return undefined;
11
+ }
12
+ return e;
13
+ };
4
14
  return {
5
- get: async (key) => data.get(key) ?? null,
6
- set: async (key, value) => { data.set(key, value); },
15
+ get: async (key) => live(key)?.v ?? null,
16
+ set: async (key, value, opts) => { data.set(key, { v: value, exp: opts?.ttlMs ? now() + opts.ttlMs : Infinity }); },
17
+ incrBy: async (key, by, opts) => {
18
+ const e = live(key);
19
+ const total = (typeof e?.v === 'number' ? e.v : 0) + by;
20
+ data.set(key, { v: total, exp: e ? e.exp : now() + opts.ttlMs });
21
+ return total;
22
+ },
7
23
  };
8
24
  }
@@ -0,0 +1,10 @@
1
+ import type { KeyValueStore } from '../types.js';
2
+ /** What it needs from a Workers KV namespace binding. */
3
+ export interface KVNamespaceLike {
4
+ get(key: string, type: 'json'): Promise<unknown>;
5
+ put(key: string, value: string, opts?: {
6
+ expirationTtl?: number;
7
+ }): Promise<unknown>;
8
+ }
9
+ /** Workers KV as a Lodestone store. Good for the model catalogue and cached answers. Two limits to know: KV is eventually consistent, so what one location writes another sees after a short delay (the breaker learns a little slower), and one key can be written about once a second, so a model failing in a burst may lose some writes. Not for rate-limit or spend counters: use Upstash, or a Durable Object, for those. */
10
+ export declare function cloudflareKvStore(kv: KVNamespaceLike): KeyValueStore;
@@ -0,0 +1,11 @@
1
+ import { safeKey } from './safeKey.js';
2
+ /** Workers KV as a Lodestone store. Good for the model catalogue and cached answers. Two limits to know: KV is eventually consistent, so what one location writes another sees after a short delay (the breaker learns a little slower), and one key can be written about once a second, so a model failing in a burst may lose some writes. Not for rate-limit or spend counters: use Upstash, or a Durable Object, for those. */
3
+ export function cloudflareKvStore(kv) {
4
+ return {
5
+ async get(key) { return (await kv.get(await safeKey(key, 400), 'json')) ?? null; },
6
+ async set(key, value, opts) {
7
+ // KV refuses a lifetime under 60 seconds.
8
+ await kv.put(await safeKey(key, 400), JSON.stringify(value), opts?.ttlMs ? { expirationTtl: Math.max(60, Math.ceil(opts.ttlMs / 1000)) } : undefined);
9
+ },
10
+ };
11
+ }
@@ -0,0 +1,14 @@
1
+ import type { KeyValueStore } from '../types.js';
2
+ /** What it needs from a Deno KV database (`await Deno.openKv()`). */
3
+ export interface DenoKvLike {
4
+ get(key: unknown[]): Promise<{
5
+ value: unknown;
6
+ }>;
7
+ set(key: unknown[], value: unknown, opts?: {
8
+ expireIn?: number;
9
+ }): Promise<unknown>;
10
+ }
11
+ /** Deno KV (Deno Deploy's built-in database) as a Lodestone store. Values expire on their own. */
12
+ export declare function denoKvStore(kv: DenoKvLike, o?: {
13
+ prefix?: string;
14
+ }): KeyValueStore;
@@ -0,0 +1,8 @@
1
+ /** Deno KV (Deno Deploy's built-in database) as a Lodestone store. Values expire on their own. */
2
+ export function denoKvStore(kv, o = {}) {
3
+ const prefix = o.prefix ?? 'lodestone';
4
+ return {
5
+ async get(key) { return (await kv.get([prefix, key])).value ?? null; },
6
+ async set(key, value, opts) { await kv.set([prefix, key], value, opts?.ttlMs ? { expireIn: opts.ttlMs } : undefined); },
7
+ };
8
+ }
@@ -0,0 +1,9 @@
1
+ export { netlifyBlobsStore } from './netlify.js';
2
+ export type { NetlifyBlobsLike } from './netlify.js';
3
+ export { cloudflareKvStore } from './cloudflare.js';
4
+ export type { KVNamespaceLike } from './cloudflare.js';
5
+ export { upstashStore } from './upstash.js';
6
+ export type { UpstashOptions } from './upstash.js';
7
+ export { denoKvStore } from './deno.js';
8
+ export type { DenoKvLike } from './deno.js';
9
+ export { safeKey } from './safeKey.js';
@@ -0,0 +1,5 @@
1
+ export { netlifyBlobsStore } from './netlify.js';
2
+ export { cloudflareKvStore } from './cloudflare.js';
3
+ export { upstashStore } from './upstash.js';
4
+ export { denoKvStore } from './deno.js';
5
+ export { safeKey } from './safeKey.js';
@@ -0,0 +1,12 @@
1
+ import type { KeyValueStore } from '../types.js';
2
+ /** What it needs from a Netlify Blobs store (`getStore('name')` from @netlify/blobs). */
3
+ export interface NetlifyBlobsLike {
4
+ get(key: string, opts: {
5
+ type: 'json';
6
+ }): Promise<unknown>;
7
+ setJSON(key: string, value: unknown): Promise<unknown>;
8
+ }
9
+ /** Netlify Blobs as a Lodestone store. Blobs cannot expire values, so each is saved with the time it stops being good and ignored after that. Fine for the catalogue, cached answers and which models are failing. Not a good home for rate-limit or spend counters under load: Blobs has no atomic add, and writes from separate functions can overwrite each other. Use Upstash for those. */
10
+ export declare function netlifyBlobsStore(blobs: NetlifyBlobsLike, o?: {
11
+ now?: () => number;
12
+ }): KeyValueStore;
@@ -0,0 +1,9 @@
1
+ import { safeKey, unwrap, wrap } from './safeKey.js';
2
+ /** Netlify Blobs as a Lodestone store. Blobs cannot expire values, so each is saved with the time it stops being good and ignored after that. Fine for the catalogue, cached answers and which models are failing. Not a good home for rate-limit or spend counters under load: Blobs has no atomic add, and writes from separate functions can overwrite each other. Use Upstash for those. */
3
+ export function netlifyBlobsStore(blobs, o = {}) {
4
+ const now = o.now ?? Date.now;
5
+ return {
6
+ async get(key) { return unwrap(await blobs.get(await safeKey(key), { type: 'json' }), now()); },
7
+ async set(key, value, opts) { await blobs.setJSON(await safeKey(key), wrap(value, opts?.ttlMs, now())); },
8
+ };
9
+ }
@@ -0,0 +1,10 @@
1
+ /** A key every store accepts: letters, digits, dot, dash and underscore stay as they are, anything else becomes ~ and two hex digits
2
+ * (so the same key always gives the same result, and two different keys never give the same one). A key too long for a store is replaced by its hash. */
3
+ export declare function safeKey(key: string, max?: number): Promise<string>;
4
+ /** For stores with no expiry of their own: a value wrapped with the time it stops being good. A value written by something else is passed through. */
5
+ export declare const wrap: (v: unknown, ttlMs: number | undefined, now: number) => {
6
+ exp?: number | undefined;
7
+ __lodestone: number;
8
+ v: unknown;
9
+ };
10
+ export declare function unwrap(raw: unknown, now: number): unknown;
@@ -0,0 +1,20 @@
1
+ import { sha256Hex } from '../util.js';
2
+ /** A key every store accepts: letters, digits, dot, dash and underscore stay as they are, anything else becomes ~ and two hex digits
3
+ * (so the same key always gives the same result, and two different keys never give the same one). A key too long for a store is replaced by its hash. */
4
+ export async function safeKey(key, max = 200) {
5
+ let out = '';
6
+ for (const b of new TextEncoder().encode(key)) {
7
+ const c = String.fromCharCode(b);
8
+ out += /[A-Za-z0-9._-]/.test(c) ? c : `~${b.toString(16).padStart(2, '0')}`;
9
+ }
10
+ return out.length <= max ? out : `h-${await sha256Hex(key, 32)}`;
11
+ }
12
+ /** For stores with no expiry of their own: a value wrapped with the time it stops being good. A value written by something else is passed through. */
13
+ export const wrap = (v, ttlMs, now) => ({ __lodestone: 1, v, ...(ttlMs ? { exp: now + ttlMs } : {}) });
14
+ export function unwrap(raw, now) {
15
+ if (typeof raw === 'object' && raw !== null && raw.__lodestone === 1) {
16
+ const { v, exp } = raw;
17
+ return typeof exp === 'number' && exp <= now ? null : v ?? null;
18
+ }
19
+ return raw ?? null;
20
+ }
@@ -0,0 +1,14 @@
1
+ import type { AtomicStore } from '../types.js';
2
+ export interface UpstashOptions {
3
+ /** The REST URL of the database, https://....upstash.io (also what Vercel KV gives you as KV_REST_API_URL). */
4
+ url: string;
5
+ /** The REST token. Keep it in an environment variable. */
6
+ token: string;
7
+ /** Put in front of every key. Default lodestone: */
8
+ prefix?: string;
9
+ fetch?: typeof fetch;
10
+ timeoutMs?: number;
11
+ }
12
+ /** Upstash Redis (and Vercel KV, which is Upstash) as a Lodestone store, over its REST API with plain fetch: no client library, so it runs on Workers, Deno, Bun and Node.
13
+ * It can count atomically, so a rate limiter that uses it is exact across every function, which no get/set store can offer. */
14
+ export declare function upstashStore(o: UpstashOptions): AtomicStore;
@@ -0,0 +1,50 @@
1
+ /** Upstash Redis (and Vercel KV, which is Upstash) as a Lodestone store, over its REST API with plain fetch: no client library, so it runs on Workers, Deno, Bun and Node.
2
+ * It can count atomically, so a rate limiter that uses it is exact across every function, which no get/set store can offer. */
3
+ export function upstashStore(o) {
4
+ const base = new URL(o.url);
5
+ if (base.protocol !== 'https:')
6
+ throw new Error('upstashStore: the URL must be https');
7
+ const f = o.fetch ?? fetch;
8
+ const prefix = o.prefix ?? 'lodestone:';
9
+ async function send(path, body) {
10
+ const res = await f(new URL(path, base), {
11
+ method: 'POST',
12
+ headers: { Authorization: `Bearer ${o.token}`, 'Content-Type': 'application/json' },
13
+ body: JSON.stringify(body),
14
+ signal: AbortSignal.timeout(o.timeoutMs ?? 5_000),
15
+ });
16
+ if (!res.ok)
17
+ throw new Error(`upstash: ${res.status}`); // the body is not included: it can echo a key
18
+ return res.json();
19
+ }
20
+ const one = async (cmd) => {
21
+ const r = (await send('/', cmd));
22
+ if (r.error)
23
+ throw new Error('upstash: the command was refused');
24
+ return r.result;
25
+ };
26
+ return {
27
+ async get(key) {
28
+ const raw = await one(['GET', prefix + key]);
29
+ if (typeof raw !== 'string')
30
+ return null;
31
+ try {
32
+ return JSON.parse(raw);
33
+ }
34
+ catch {
35
+ return null;
36
+ }
37
+ },
38
+ async set(key, value, opts) {
39
+ await one(['SET', prefix + key, JSON.stringify(value), ...(opts?.ttlMs ? ['PX', String(Math.ceil(opts.ttlMs))] : [])]);
40
+ },
41
+ async incrBy(key, by, opts) {
42
+ const k = prefix + key;
43
+ const r = (await send('/pipeline', [['INCRBY', k, by], ['PEXPIRE', k, opts.ttlMs, 'NX']]));
44
+ const n = r[0]?.result;
45
+ if (r[0]?.error || typeof n !== 'number')
46
+ throw new Error('upstash: the counter was refused');
47
+ return n;
48
+ },
49
+ };
50
+ }
package/dist/types.d.ts CHANGED
@@ -42,9 +42,21 @@ export interface Provider {
42
42
  /** The models the provider offers right now, with price and what it does with prompts where it says. Lets Lodestone drop a model that has gone and find others. */
43
43
  listModels?(apiKey: string, signal?: AbortSignal): Promise<CatalogModel[] | null>;
44
44
  }
45
- /** Where Lodestone keeps the model catalogue and cached answers. Anything that can
46
- * get and set JSON by key will do: Netlify Blobs, Redis, a SQLite table, a Map. */
45
+ /** Where Lodestone keeps the model catalogue, which models are failing, saved answers and counters.
46
+ * Anything that can get and set JSON by key will do: Netlify Blobs, Cloudflare KV, Upstash, Deno KV,
47
+ * a SQLite table, a Map. Ready-made adapters are in stores/. */
47
48
  export interface KeyValueStore {
48
49
  get(key: string): Promise<unknown>;
49
- set(key: string, value: unknown): Promise<void>;
50
+ /** `ttlMs` says how long the value is worth keeping. A store that cannot expire values may ignore it. */
51
+ set(key: string, value: unknown, opts?: {
52
+ ttlMs?: number;
53
+ }): Promise<void>;
54
+ }
55
+ /** A store that can add to a counter in one step, so two functions counting at the same moment do not lose a count.
56
+ * The rate limiter uses it when the store has it (Upstash does, and the memory store), and otherwise reads and writes. */
57
+ export interface AtomicStore extends KeyValueStore {
58
+ /** Adds `by` to the counter at `key` (starting from 0), returns the new total. A counter without a value expires after `ttlMs`. */
59
+ incrBy(key: string, by: number, opts: {
60
+ ttlMs: number;
61
+ }): Promise<number>;
50
62
  }