@extrafemi-ai/lodestone 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +43 -14
- package/SECURITY.md +17 -0
- package/dist/catalog.js +11 -2
- package/dist/errors.d.ts +6 -0
- package/dist/errors.js +1 -2
- package/dist/explain.js +4 -0
- package/dist/health.js +19 -3
- package/dist/index.d.ts +10 -4
- package/dist/index.js +4 -1
- package/dist/limiter.d.ts +34 -0
- package/dist/limiter.js +35 -0
- package/dist/lodestone.d.ts +35 -12
- package/dist/lodestone.js +73 -16
- package/dist/providers/openaiCompatible.d.ts +2 -0
- package/dist/providers/openaiCompatible.js +7 -1
- package/dist/scorecard.d.ts +9 -3
- package/dist/scorecard.js +45 -2
- package/dist/spend.d.ts +33 -0
- package/dist/spend.js +72 -0
- package/dist/store.d.ts +2 -2
- package/dist/store.js +19 -3
- package/dist/stores/cloudflare.d.ts +10 -0
- package/dist/stores/cloudflare.js +11 -0
- package/dist/stores/deno.d.ts +14 -0
- package/dist/stores/deno.js +8 -0
- package/dist/stores/index.d.ts +9 -0
- package/dist/stores/index.js +5 -0
- package/dist/stores/netlify.d.ts +12 -0
- package/dist/stores/netlify.js +9 -0
- package/dist/stores/safeKey.d.ts +10 -0
- package/dist/stores/safeKey.js +20 -0
- package/dist/stores/upstash.d.ts +14 -0
- package/dist/stores/upstash.js +50 -0
- package/dist/types.d.ts +15 -3
- package/dist/util.d.ts +7 -0
- package/dist/util.js +17 -0
- package/docs/protecting-your-budget.md +48 -0
- package/docs/runtimes.md +19 -0
- package/docs/stores.md +45 -0
- package/examples/netlify-function.ts +10 -7
- package/package.json +5 -2
package/dist/lodestone.js
CHANGED
|
@@ -10,6 +10,7 @@ import { google } from './providers/google.js';
|
|
|
10
10
|
import { openrouter } from './providers/openrouter.js';
|
|
11
11
|
import { rankModels } from './scorecard.js';
|
|
12
12
|
import { memoryStore } from './store.js';
|
|
13
|
+
import { sha256Hex } from './util.js';
|
|
13
14
|
const hasImage = (messages) => messages.some((m) => Array.isArray(m.content) && m.content.some((p) => p.type === 'image'));
|
|
14
15
|
/** How many candidates are kept in the background for a tier. A request still tries at most three. */
|
|
15
16
|
const POOL = 8;
|
|
@@ -60,13 +61,19 @@ export function createLodestone(config = {}) {
|
|
|
60
61
|
/** A fallback provider's models for a tier: the app's own, pruned and topped up from the provider's catalogue. */
|
|
61
62
|
async function resolveFallback(fb, tier, strict) {
|
|
62
63
|
const cat = fbCatalogs.get(fb);
|
|
64
|
+
// A private call may only go where something says prompts are not kept or trained on: the app's own word, or the provider's list.
|
|
65
|
+
const vouched = strict && fb.privacy === 'no-training';
|
|
63
66
|
if (!cat)
|
|
64
|
-
return fb.models;
|
|
67
|
+
return strict && !vouched ? [] : fb.models;
|
|
65
68
|
try {
|
|
66
|
-
|
|
69
|
+
const listed = await cat.load();
|
|
70
|
+
if (strict && !vouched && !listed?.length)
|
|
71
|
+
return []; // no list, so nothing to check the models against
|
|
72
|
+
const p = strict && !vouched ? { ...policy, requireNoTraining: true } : policy;
|
|
73
|
+
return resolveProviderModels(fb.models, listed, { tier, policy: p, strict: strict && !vouched, fill: fb.discover?.fill, max: fb.discover?.max, now: now() });
|
|
67
74
|
}
|
|
68
75
|
catch {
|
|
69
|
-
return fb.models;
|
|
76
|
+
return strict && !vouched ? [] : fb.models;
|
|
70
77
|
}
|
|
71
78
|
}
|
|
72
79
|
/** The scorecards, or none: never throws and never holds a request up for long. */
|
|
@@ -108,15 +115,29 @@ export function createLodestone(config = {}) {
|
|
|
108
115
|
throw new LodestoneError('aborted');
|
|
109
116
|
const quality = opts.quality ?? profile?.quality;
|
|
110
117
|
const order = async (models) => (quality && models.length > 1 ? rankModels(models, await cards(), quality, { now }) : models);
|
|
111
|
-
|
|
118
|
+
// A private call is never read from or written to the cache: an answer made from a CV must not sit in a shared store.
|
|
119
|
+
const cacheSlot = opts.cacheKey && !strict ? `lodestone:cache:${opts.cacheKey}` : null;
|
|
120
|
+
const cacheTtl = opts.cacheTtlMs ?? 24 * 3_600_000;
|
|
121
|
+
const fingerprint = cacheSlot ? await sha256Hex(JSON.stringify([opts.system ?? '', messages, tier, opts.task ?? ''])) : '';
|
|
112
122
|
if (cacheSlot) {
|
|
113
123
|
try {
|
|
114
124
|
const hit = (await store.get(cacheSlot));
|
|
115
|
-
|
|
125
|
+
const mine = typeof hit?.text === 'string' && typeof hit.at === 'number' && hit.fp === fingerprint; // an answer for different input is a miss
|
|
126
|
+
if (mine && now() - hit.at < cacheTtl && (!opts.accept || opts.accept(hit.text)))
|
|
116
127
|
return { text: hit.text, provider: hit.provider, model: hit.model, cached: true };
|
|
117
128
|
}
|
|
118
129
|
catch { /* a store that cannot be read is a cache miss */ }
|
|
119
130
|
}
|
|
131
|
+
// Each caller has their own allowance, so one of them cannot empty the platform's budget for the rest.
|
|
132
|
+
let throttled = null;
|
|
133
|
+
if (config.rateLimit && opts.caller && !opts.apiKey && !opts.route?.length) {
|
|
134
|
+
const r = await config.rateLimit.limiter.take(opts.caller).catch(() => null);
|
|
135
|
+
if (r && !r.ok) {
|
|
136
|
+
if ((config.rateLimit.whenOver ?? 'reject') === 'reject')
|
|
137
|
+
throw new LodestoneError('throttled', { retryAfterMs: r.retryAfterMs });
|
|
138
|
+
throttled = { retryAfterMs: r.retryAfterMs };
|
|
139
|
+
}
|
|
140
|
+
}
|
|
120
141
|
const started = now();
|
|
121
142
|
const left = () => budget - (now() - started);
|
|
122
143
|
const orKey = opts.apiKey || config.openrouter?.apiKey;
|
|
@@ -147,11 +168,22 @@ export function createLodestone(config = {}) {
|
|
|
147
168
|
return keep;
|
|
148
169
|
};
|
|
149
170
|
/** May the platform's money be spent on this attempt? A caller's own key is their money, not ours. */
|
|
171
|
+
let held = null;
|
|
150
172
|
const mayUse = async (provider, model, phase, free = false) => {
|
|
151
|
-
if (callerKey || free
|
|
173
|
+
if (callerKey || free)
|
|
152
174
|
return true;
|
|
153
|
-
|
|
175
|
+
// A caller over their allowance (with 'free-only') costs the platform nothing.
|
|
176
|
+
if (throttled) {
|
|
177
|
+
config.onAttempt?.({ provider: provider.name, model, phase, outcome: 'skipped', ms: 0 });
|
|
178
|
+
return false;
|
|
179
|
+
}
|
|
180
|
+
if (!config.spend)
|
|
154
181
|
return true;
|
|
182
|
+
const info = { id: crypto.randomUUID(), provider: provider.name, model, phase };
|
|
183
|
+
if (await config.spend.allow(info)) {
|
|
184
|
+
held = info;
|
|
185
|
+
return true;
|
|
186
|
+
}
|
|
155
187
|
spendBlocked = true;
|
|
156
188
|
config.onAttempt?.({ provider: provider.name, model, phase, outcome: 'skipped', ms: 0 });
|
|
157
189
|
return false;
|
|
@@ -182,6 +214,15 @@ export function createLodestone(config = {}) {
|
|
|
182
214
|
else
|
|
183
215
|
out = { kind: 'fail', status: 599, body: e.message };
|
|
184
216
|
}
|
|
217
|
+
// An attempt that did not answer gives back the hold it took on the spend limit.
|
|
218
|
+
const hold = held;
|
|
219
|
+
if (out.kind !== 'ok' && hold && hold.provider === provider.name && hold.model === model) {
|
|
220
|
+
held = null;
|
|
221
|
+
try {
|
|
222
|
+
await config.spend?.release?.(hold);
|
|
223
|
+
}
|
|
224
|
+
catch { /* the hold expires by itself */ }
|
|
225
|
+
}
|
|
185
226
|
if (out.kind === 'aborted')
|
|
186
227
|
throw new LodestoneError('aborted'); // the person left: nothing more to try
|
|
187
228
|
const event = { provider: provider.name, model, phase, outcome: out.kind === 'ok' ? 'ok' : out.kind === 'timeout' ? 'timeout' : 'failed', status: out.kind === 'fail' ? out.status : undefined, ms: now() - at };
|
|
@@ -200,14 +241,17 @@ export function createLodestone(config = {}) {
|
|
|
200
241
|
const result = { text: reply.text, provider: provider.name, model, usage: reply.usage };
|
|
201
242
|
if (phase && !callerKey && !free && config.spend) {
|
|
202
243
|
const costUsd = reply.usage?.costUsd ?? (await estimateCost(model, reply.usage));
|
|
244
|
+
const hold = held;
|
|
245
|
+
const id = hold && hold.provider === provider.name && hold.model === model ? hold.id : crypto.randomUUID();
|
|
246
|
+
held = null;
|
|
203
247
|
// A ledger that fails must never fail an answer the person is waiting for.
|
|
204
248
|
try {
|
|
205
|
-
await config.spend.record({ provider: provider.name, model, phase, usage: reply.usage, costUsd });
|
|
249
|
+
await config.spend.record({ id, provider: provider.name, model, phase, usage: reply.usage, costUsd });
|
|
206
250
|
}
|
|
207
251
|
catch { /* not counted */ }
|
|
208
252
|
}
|
|
209
253
|
if (cacheSlot)
|
|
210
|
-
await store.set(cacheSlot, { ...result, at: now() }).catch(() => { });
|
|
254
|
+
await store.set(cacheSlot, { ...result, at: now(), fp: fingerprint }, { ttlMs: cacheTtl }).catch(() => { });
|
|
211
255
|
return result;
|
|
212
256
|
};
|
|
213
257
|
// An explicit route: exactly the steps the caller named, in order.
|
|
@@ -339,6 +383,8 @@ export function createLodestone(config = {}) {
|
|
|
339
383
|
}
|
|
340
384
|
// Nothing answered. Say why, using what the first useful failure told us.
|
|
341
385
|
const ownKey = Boolean(opts.apiKey);
|
|
386
|
+
if (throttled)
|
|
387
|
+
throw new LodestoneError('throttled', throttled);
|
|
342
388
|
if (spendBlocked)
|
|
343
389
|
throw new LodestoneError('budget');
|
|
344
390
|
if (!attempted)
|
|
@@ -360,16 +406,27 @@ export function createLodestone(config = {}) {
|
|
|
360
406
|
const rateLimit = status === 429 && failure ? readRateLimit(failure.body) : undefined;
|
|
361
407
|
return new LodestoneError(kind, { status, rateLimit, ownKey, needsCredit, providerMessage: failure?.message });
|
|
362
408
|
}
|
|
363
|
-
/** Like complete(), but the answer must be JSON
|
|
409
|
+
/** Like complete(), but the answer must be JSON, and pass `validate` or `schema` if given. A model whose answer
|
|
410
|
+
* isn't is skipped for the next. `schema` is anything with a `safeParse` method, such as a Zod schema; `data`
|
|
411
|
+
* is then what the schema returns, so its defaults and transforms apply. */
|
|
364
412
|
async function json(opts) {
|
|
365
|
-
const
|
|
413
|
+
const parse = (text) => {
|
|
366
414
|
const v = tryParseJson(text);
|
|
367
|
-
|
|
415
|
+
if (v === undefined)
|
|
416
|
+
return { ok: false };
|
|
417
|
+
if (opts.validate && !opts.validate(v))
|
|
418
|
+
return { ok: false };
|
|
419
|
+
if (opts.schema) {
|
|
420
|
+
const r = opts.schema.safeParse(v);
|
|
421
|
+
return r.success ? { ok: true, data: r.data } : { ok: false };
|
|
422
|
+
}
|
|
423
|
+
return { ok: true, data: v };
|
|
368
424
|
};
|
|
369
|
-
const { validate: _v, ...rest } = opts;
|
|
370
|
-
const accept = opts.accept ? (s) =>
|
|
425
|
+
const { validate: _v, schema: _s, ...rest } = opts;
|
|
426
|
+
const accept = opts.accept ? (s) => parse(s).ok && opts.accept(s) : (s) => parse(s).ok;
|
|
371
427
|
const result = await complete({ ...rest, accept });
|
|
372
|
-
|
|
428
|
+
const done = parse(result.text);
|
|
429
|
+
return { ...result, data: (done.ok ? done.data : tryParseJson(result.text)) };
|
|
373
430
|
}
|
|
374
431
|
/** Fetches and saves OpenRouter's current catalogue. Run it daily. */
|
|
375
432
|
const refreshCatalog = () => catalog.refresh();
|
|
@@ -405,7 +462,7 @@ export function createLodestone(config = {}) {
|
|
|
405
462
|
}
|
|
406
463
|
return { ok: shortTiers.length === 0, catalogSize: models.length, withdrawn, expiring, shortTiers, ...(Object.keys(providers).length ? { providers } : {}) };
|
|
407
464
|
}
|
|
408
|
-
/** An answer already saved under this key, without calling any model: so a caller can serve it without counting it as a request. */
|
|
465
|
+
/** An answer already saved under this key, without calling any model: so a caller can serve it without counting it as a request. It cannot see the input, so the key must stand for all of it; complete() checks the input itself and will not serve an answer made for something else. */
|
|
409
466
|
async function peek(cacheKey, ttlMs = 24 * 3_600_000) {
|
|
410
467
|
try {
|
|
411
468
|
const hit = (await store.get(`lodestone:cache:${cacheKey}`));
|
|
@@ -23,6 +23,8 @@ export declare function openaiCompatible(o: OpenAICompatibleOptions): Provider;
|
|
|
23
23
|
export declare const normalizeRequesty: (r: Record<string, unknown>) => CatalogModel | null;
|
|
24
24
|
/** LLM API lists chat, image, video, speech and embedding models together: only "chat" is kept. */
|
|
25
25
|
export declare const normalizeLlmapi: (r: Record<string, unknown>) => CatalogModel | null;
|
|
26
|
+
/** Groq lists speech, text-to-speech and safety-classifier models beside its chat models (seen 28 Oct 2026: whisper, orpheus, prompt-guard, safeguard). Those are not chat models, so they are left out. */
|
|
27
|
+
export declare const normalizeGroq: (r: Record<string, unknown>) => CatalogModel | null;
|
|
26
28
|
export interface PresetOptions {
|
|
27
29
|
fetch?: typeof fetch;
|
|
28
30
|
headers?: Record<string, string>;
|
|
@@ -77,12 +77,18 @@ export const normalizeLlmapi = (r) => {
|
|
|
77
77
|
...(str(r.family) ? { publisher: labOf(str(r.family)) } : {}),
|
|
78
78
|
};
|
|
79
79
|
};
|
|
80
|
+
/** Groq lists speech, text-to-speech and safety-classifier models beside its chat models (seen 28 Oct 2026: whisper, orpheus, prompt-guard, safeguard). Those are not chat models, so they are left out. */
|
|
81
|
+
export const normalizeGroq = (r) => {
|
|
82
|
+
if (typeof r.id !== 'string' || r.active === false || /whisper|orpheus|tts|guard|embed/i.test(r.id))
|
|
83
|
+
return null;
|
|
84
|
+
return { id: r.id, ...(num(r.context_window) !== undefined ? { context_length: num(r.context_window) } : {}) };
|
|
85
|
+
};
|
|
80
86
|
/** https://www.requesty.ai: a gateway to 800+ models, with failover between providers. `region: 'eu'` keeps prompts in the EU. */
|
|
81
87
|
export const requesty = (o = {}) => openaiCompatible({ name: 'requesty', baseUrl: o.region === 'eu' ? 'https://router.eu.requesty.ai/v1' : 'https://router.requesty.ai/v1', normalize: normalizeRequesty, ...o });
|
|
82
88
|
/** https://llmapi.ai: a gateway to 150+ chat models. */
|
|
83
89
|
export const llmapi = (o = {}) => openaiCompatible({ name: 'llmapi', baseUrl: 'https://api.llmapi.ai/v1', normalize: normalizeLlmapi, ...o });
|
|
84
90
|
/** https://groq.com: fast inference with a free developer tier. */
|
|
85
|
-
export const groq = (o = {}) => openaiCompatible({ name: 'groq', baseUrl: 'https://api.groq.com/openai/v1', ...o });
|
|
91
|
+
export const groq = (o = {}) => openaiCompatible({ name: 'groq', baseUrl: 'https://api.groq.com/openai/v1', normalize: normalizeGroq, ...o });
|
|
86
92
|
/** https://cerebras.ai: fast inference with a free tier. */
|
|
87
93
|
export const cerebras = (o = {}) => openaiCompatible({ name: 'cerebras', baseUrl: 'https://api.cerebras.ai/v1', ...o });
|
|
88
94
|
export const openai = (o = {}) => openaiCompatible({ name: 'openai', baseUrl: 'https://api.openai.com/v1', ...o });
|
package/dist/scorecard.d.ts
CHANGED
|
@@ -51,11 +51,17 @@ export declare class ScorecardError extends Error {
|
|
|
51
51
|
* ScorecardError listing what is wrong. A scorecard with a schema version this
|
|
52
52
|
* library doesn't know is refused, not guessed at. */
|
|
53
53
|
export declare function parseScorecard(raw: unknown): Scorecard;
|
|
54
|
-
|
|
55
|
-
export declare function fetchScorecard(url: string, opts?: {
|
|
54
|
+
export interface FetchScorecardOptions {
|
|
56
55
|
fetch?: typeof fetch;
|
|
57
56
|
timeoutMs?: number;
|
|
58
|
-
|
|
57
|
+
/** Refuse a body bigger than this. Default 1 MB. */
|
|
58
|
+
maxBytes?: number;
|
|
59
|
+
/** Only fetch from these hosts (the final host, after any redirect). Default: any host. */
|
|
60
|
+
allowedHosts?: string[];
|
|
61
|
+
}
|
|
62
|
+
/** Fetches and checks a scorecard from a URL. Throws if it can't be read, is too big, comes from a host you did not allow, or isn't valid.
|
|
63
|
+
* Only https is accepted (http only for localhost). A scorecard can only change the order models are tried in, but a body of any size from anywhere is still not worth reading. */
|
|
64
|
+
export declare function fetchScorecard(url: string, opts?: FetchScorecardOptions): Promise<Scorecard>;
|
|
59
65
|
/** An id without its variant suffix: anthropic/claude-x:free is the same model as anthropic/claude-x. */
|
|
60
66
|
export declare const baseModelId: (id: string) => string;
|
|
61
67
|
/** The score for a model on a query, or undefined if nobody measured it well enough.
|
package/dist/scorecard.js
CHANGED
|
@@ -12,6 +12,7 @@ export class ScorecardError extends Error {
|
|
|
12
12
|
this.name = 'ScorecardError';
|
|
13
13
|
}
|
|
14
14
|
}
|
|
15
|
+
const MAX_SCORES = 20_000;
|
|
15
16
|
const isStr = (v) => typeof v === 'string' && v.length > 0;
|
|
16
17
|
const isNum = (v) => typeof v === 'number' && Number.isFinite(v);
|
|
17
18
|
const isDay = (v) => typeof v === 'string' && /^\d{4}-\d{2}-\d{2}/.test(v) && !Number.isNaN(Date.parse(v));
|
|
@@ -31,6 +32,8 @@ export function parseScorecard(raw) {
|
|
|
31
32
|
problems.push('generatedAt is missing or not a date');
|
|
32
33
|
if (!Array.isArray(c.scores))
|
|
33
34
|
problems.push('scores is not a list');
|
|
35
|
+
else if (c.scores.length > MAX_SCORES)
|
|
36
|
+
problems.push(`scores has more than ${MAX_SCORES} entries`);
|
|
34
37
|
else
|
|
35
38
|
c.scores.forEach((s, i) => {
|
|
36
39
|
const at = `scores[${i}]`;
|
|
@@ -53,12 +56,52 @@ export function parseScorecard(raw) {
|
|
|
53
56
|
throw new ScorecardError(problems);
|
|
54
57
|
return c;
|
|
55
58
|
}
|
|
56
|
-
/** Fetches and checks a scorecard from a URL. Throws if it can't be read or isn't valid.
|
|
59
|
+
/** Fetches and checks a scorecard from a URL. Throws if it can't be read, is too big, comes from a host you did not allow, or isn't valid.
|
|
60
|
+
* Only https is accepted (http only for localhost). A scorecard can only change the order models are tried in, but a body of any size from anywhere is still not worth reading. */
|
|
57
61
|
export async function fetchScorecard(url, opts = {}) {
|
|
62
|
+
const u = new URL(url);
|
|
63
|
+
const local = u.hostname === 'localhost' || u.hostname === '127.0.0.1' || u.hostname === '[::1]';
|
|
64
|
+
if (u.protocol !== 'https:' && !(u.protocol === 'http:' && local))
|
|
65
|
+
throw new Error(`scorecard ${url}: only https is accepted`);
|
|
66
|
+
const allowed = (host) => !opts.allowedHosts || opts.allowedHosts.map((h) => h.toLowerCase()).includes(host.toLowerCase());
|
|
67
|
+
if (!allowed(u.hostname))
|
|
68
|
+
throw new Error(`scorecard ${url}: host not allowed`);
|
|
58
69
|
const res = await (opts.fetch ?? fetch)(url, { signal: AbortSignal.timeout(opts.timeoutMs ?? 8_000) });
|
|
59
70
|
if (!res.ok)
|
|
60
71
|
throw new Error(`scorecard ${url}: ${res.status}`);
|
|
61
|
-
|
|
72
|
+
if (res.url && !allowed(new URL(res.url).hostname))
|
|
73
|
+
throw new Error(`scorecard ${url}: redirected to a host that is not allowed`);
|
|
74
|
+
const max = opts.maxBytes ?? 1_000_000;
|
|
75
|
+
const declared = Number(res.headers.get('content-length'));
|
|
76
|
+
if (Number.isFinite(declared) && declared > max)
|
|
77
|
+
throw new Error(`scorecard ${url}: larger than ${max} bytes`);
|
|
78
|
+
let text = '';
|
|
79
|
+
if (res.body) {
|
|
80
|
+
const reader = res.body.getReader();
|
|
81
|
+
const chunks = [];
|
|
82
|
+
let size = 0;
|
|
83
|
+
for (;;) {
|
|
84
|
+
const { done, value } = await reader.read();
|
|
85
|
+
if (done)
|
|
86
|
+
break;
|
|
87
|
+
size += value.byteLength;
|
|
88
|
+
if (size > max) {
|
|
89
|
+
await reader.cancel().catch(() => { });
|
|
90
|
+
throw new Error(`scorecard ${url}: larger than ${max} bytes`);
|
|
91
|
+
}
|
|
92
|
+
chunks.push(value);
|
|
93
|
+
}
|
|
94
|
+
const all = new Uint8Array(size);
|
|
95
|
+
let at = 0;
|
|
96
|
+
for (const c of chunks) {
|
|
97
|
+
all.set(c, at);
|
|
98
|
+
at += c.byteLength;
|
|
99
|
+
}
|
|
100
|
+
text = new TextDecoder().decode(all);
|
|
101
|
+
}
|
|
102
|
+
else
|
|
103
|
+
text = await res.text();
|
|
104
|
+
return parseScorecard(JSON.parse(text));
|
|
62
105
|
}
|
|
63
106
|
/** An id without its variant suffix: anthropic/claude-x:free is the same model as anthropic/claude-x. */
|
|
64
107
|
export const baseModelId = (id) => id.split(':')[0].toLowerCase();
|
package/dist/spend.d.ts
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { SpendControl } from './lodestone.js';
|
|
2
|
+
import type { KeyValueStore } from './types.js';
|
|
3
|
+
/** A daily and a monthly spend limit that holds under load. A bare `allow()` then `record()` lets a burst of
|
|
4
|
+
* requests all pass the check before any of them has been counted. This takes a hold for each paid attempt
|
|
5
|
+
* when it is allowed, and counts the holds against the limit, so concurrent requests see each other. The
|
|
6
|
+
* hold is replaced by the real cost when the attempt answers, and handed back when it does not. A hold that
|
|
7
|
+
* is never settled (the function was killed) expires by itself.
|
|
8
|
+
*
|
|
9
|
+
* Exact inside one process. Between processes it is as good as the store: with a plain get/set store, two
|
|
10
|
+
* functions allowing at the same moment can each take a hold the other did not see, so size `holdUsd` and
|
|
11
|
+
* the limit with that in mind. If the store cannot be read, the attempt is refused (the money is yours). */
|
|
12
|
+
export interface SpendLimitOptions {
|
|
13
|
+
store: KeyValueStore;
|
|
14
|
+
dailyUsd?: number;
|
|
15
|
+
monthlyUsd?: number;
|
|
16
|
+
/** Held for each paid attempt until it settles: roughly the most one call can cost. Default $0.01. */
|
|
17
|
+
holdUsd?: number;
|
|
18
|
+
/** A hold that is never settled goes away after this. Default 2 minutes. */
|
|
19
|
+
holdTtlMs?: number;
|
|
20
|
+
/** Where the ledger is kept. Default lodestone:spend */
|
|
21
|
+
key?: string;
|
|
22
|
+
now?: () => number;
|
|
23
|
+
}
|
|
24
|
+
export interface SpendReport {
|
|
25
|
+
todayUsd: number;
|
|
26
|
+
monthUsd: number;
|
|
27
|
+
heldUsd: number;
|
|
28
|
+
dailyUsd?: number;
|
|
29
|
+
monthlyUsd?: number;
|
|
30
|
+
}
|
|
31
|
+
export declare function createSpendLimit(o: SpendLimitOptions): Required<SpendControl> & {
|
|
32
|
+
report(): Promise<SpendReport>;
|
|
33
|
+
};
|
package/dist/spend.js
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { createMutex, isNonNegative, isRecord } from './util.js';
|
|
2
|
+
const MAX_HOLDS = 500;
|
|
3
|
+
export function createSpendLimit(o) {
|
|
4
|
+
const now = o.now ?? Date.now;
|
|
5
|
+
const key = o.key ?? 'lodestone:spend';
|
|
6
|
+
const hold = o.holdUsd ?? 0.01;
|
|
7
|
+
const ttl = o.holdTtlMs ?? 120_000;
|
|
8
|
+
const exclusive = createMutex();
|
|
9
|
+
/** What is in the store, checked: a ledger somebody else wrote is not trusted to be the right shape. */
|
|
10
|
+
async function load() {
|
|
11
|
+
const t = new Date(now());
|
|
12
|
+
const day = t.toISOString().slice(0, 10);
|
|
13
|
+
const month = day.slice(0, 7);
|
|
14
|
+
const raw = await o.store.get(key);
|
|
15
|
+
const s = isRecord(raw) ? raw : {};
|
|
16
|
+
const holds = {};
|
|
17
|
+
if (isRecord(s.holds)) {
|
|
18
|
+
for (const [id, h] of Object.entries(s.holds).slice(0, MAX_HOLDS)) {
|
|
19
|
+
if (Array.isArray(h) && isNonNegative(h[0]) && isNonNegative(h[1]) && h[1] > now())
|
|
20
|
+
holds[id] = [h[0], h[1]];
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
return {
|
|
24
|
+
day, month, holds,
|
|
25
|
+
dayUsd: s.day === day && isNonNegative(s.dayUsd) ? s.dayUsd : 0,
|
|
26
|
+
monthUsd: s.month === month && isNonNegative(s.monthUsd) ? s.monthUsd : 0,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
const held = (l) => Object.values(l.holds).reduce((a, h) => a + h[0], 0);
|
|
30
|
+
const save = (l) => o.store.set(key, l);
|
|
31
|
+
return {
|
|
32
|
+
allow: (info) => exclusive(async () => {
|
|
33
|
+
try {
|
|
34
|
+
const l = await load();
|
|
35
|
+
const pending = held(l) + hold;
|
|
36
|
+
if (o.dailyUsd !== undefined && l.dayUsd + pending > o.dailyUsd)
|
|
37
|
+
return false;
|
|
38
|
+
if (o.monthlyUsd !== undefined && l.monthUsd + pending > o.monthlyUsd)
|
|
39
|
+
return false;
|
|
40
|
+
if (Object.keys(l.holds).length >= MAX_HOLDS)
|
|
41
|
+
return false;
|
|
42
|
+
l.holds[info.id] = [hold, now() + ttl];
|
|
43
|
+
await save(l);
|
|
44
|
+
return true;
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
}),
|
|
50
|
+
record: (info) => exclusive(async () => {
|
|
51
|
+
const l = await load();
|
|
52
|
+
const taken = l.holds[info.id]?.[0] ?? hold;
|
|
53
|
+
delete l.holds[info.id];
|
|
54
|
+
// A cost nobody reported counts as the hold: better to over-count than to let a free ride through.
|
|
55
|
+
const cost = isNonNegative(info.costUsd) ? info.costUsd : taken;
|
|
56
|
+
l.dayUsd += cost;
|
|
57
|
+
l.monthUsd += cost;
|
|
58
|
+
await save(l);
|
|
59
|
+
}),
|
|
60
|
+
release: (info) => exclusive(async () => {
|
|
61
|
+
const l = await load();
|
|
62
|
+
if (!(info.id in l.holds))
|
|
63
|
+
return;
|
|
64
|
+
delete l.holds[info.id];
|
|
65
|
+
await save(l);
|
|
66
|
+
}),
|
|
67
|
+
report: () => exclusive(async () => {
|
|
68
|
+
const l = await load();
|
|
69
|
+
return { todayUsd: l.dayUsd, monthUsd: l.monthUsd, heldUsd: held(l), dailyUsd: o.dailyUsd, monthlyUsd: o.monthlyUsd };
|
|
70
|
+
}),
|
|
71
|
+
};
|
|
72
|
+
}
|
package/dist/store.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { AtomicStore } from './types.js';
|
|
2
2
|
/** A store that lives in memory. The default; lost when the process ends. */
|
|
3
|
-
export declare function memoryStore():
|
|
3
|
+
export declare function memoryStore(now?: () => number): AtomicStore;
|
package/dist/store.js
CHANGED
|
@@ -1,8 +1,24 @@
|
|
|
1
1
|
/** A store that lives in memory. The default; lost when the process ends. */
|
|
2
|
-
export function memoryStore() {
|
|
2
|
+
export function memoryStore(now = Date.now) {
|
|
3
3
|
const data = new Map();
|
|
4
|
+
const live = (key) => {
|
|
5
|
+
const e = data.get(key);
|
|
6
|
+
if (!e)
|
|
7
|
+
return undefined;
|
|
8
|
+
if (e.exp <= now()) {
|
|
9
|
+
data.delete(key);
|
|
10
|
+
return undefined;
|
|
11
|
+
}
|
|
12
|
+
return e;
|
|
13
|
+
};
|
|
4
14
|
return {
|
|
5
|
-
get: async (key) =>
|
|
6
|
-
set: async (key, value) => { data.set(key, value); },
|
|
15
|
+
get: async (key) => live(key)?.v ?? null,
|
|
16
|
+
set: async (key, value, opts) => { data.set(key, { v: value, exp: opts?.ttlMs ? now() + opts.ttlMs : Infinity }); },
|
|
17
|
+
incrBy: async (key, by, opts) => {
|
|
18
|
+
const e = live(key);
|
|
19
|
+
const total = (typeof e?.v === 'number' ? e.v : 0) + by;
|
|
20
|
+
data.set(key, { v: total, exp: e ? e.exp : now() + opts.ttlMs });
|
|
21
|
+
return total;
|
|
22
|
+
},
|
|
7
23
|
};
|
|
8
24
|
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { KeyValueStore } from '../types.js';
|
|
2
|
+
/** What it needs from a Workers KV namespace binding. */
|
|
3
|
+
export interface KVNamespaceLike {
|
|
4
|
+
get(key: string, type: 'json'): Promise<unknown>;
|
|
5
|
+
put(key: string, value: string, opts?: {
|
|
6
|
+
expirationTtl?: number;
|
|
7
|
+
}): Promise<unknown>;
|
|
8
|
+
}
|
|
9
|
+
/** Workers KV as a Lodestone store. Good for the model catalogue and cached answers. Two limits to know: KV is eventually consistent, so what one location writes another sees after a short delay (the breaker learns a little slower), and one key can be written about once a second, so a model failing in a burst may lose some writes. Not for rate-limit or spend counters: use Upstash, or a Durable Object, for those. */
|
|
10
|
+
export declare function cloudflareKvStore(kv: KVNamespaceLike): KeyValueStore;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { safeKey } from './safeKey.js';
|
|
2
|
+
/** Workers KV as a Lodestone store. Good for the model catalogue and cached answers. Two limits to know: KV is eventually consistent, so what one location writes another sees after a short delay (the breaker learns a little slower), and one key can be written about once a second, so a model failing in a burst may lose some writes. Not for rate-limit or spend counters: use Upstash, or a Durable Object, for those. */
|
|
3
|
+
export function cloudflareKvStore(kv) {
|
|
4
|
+
return {
|
|
5
|
+
async get(key) { return (await kv.get(await safeKey(key, 400), 'json')) ?? null; },
|
|
6
|
+
async set(key, value, opts) {
|
|
7
|
+
// KV refuses a lifetime under 60 seconds.
|
|
8
|
+
await kv.put(await safeKey(key, 400), JSON.stringify(value), opts?.ttlMs ? { expirationTtl: Math.max(60, Math.ceil(opts.ttlMs / 1000)) } : undefined);
|
|
9
|
+
},
|
|
10
|
+
};
|
|
11
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { KeyValueStore } from '../types.js';
|
|
2
|
+
/** What it needs from a Deno KV database (`await Deno.openKv()`). */
|
|
3
|
+
export interface DenoKvLike {
|
|
4
|
+
get(key: unknown[]): Promise<{
|
|
5
|
+
value: unknown;
|
|
6
|
+
}>;
|
|
7
|
+
set(key: unknown[], value: unknown, opts?: {
|
|
8
|
+
expireIn?: number;
|
|
9
|
+
}): Promise<unknown>;
|
|
10
|
+
}
|
|
11
|
+
/** Deno KV (Deno Deploy's built-in database) as a Lodestone store. Values expire on their own. */
|
|
12
|
+
export declare function denoKvStore(kv: DenoKvLike, o?: {
|
|
13
|
+
prefix?: string;
|
|
14
|
+
}): KeyValueStore;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/** Deno KV (Deno Deploy's built-in database) as a Lodestone store. Values expire on their own. */
|
|
2
|
+
export function denoKvStore(kv, o = {}) {
|
|
3
|
+
const prefix = o.prefix ?? 'lodestone';
|
|
4
|
+
return {
|
|
5
|
+
async get(key) { return (await kv.get([prefix, key])).value ?? null; },
|
|
6
|
+
async set(key, value, opts) { await kv.set([prefix, key], value, opts?.ttlMs ? { expireIn: opts.ttlMs } : undefined); },
|
|
7
|
+
};
|
|
8
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export { netlifyBlobsStore } from './netlify.js';
|
|
2
|
+
export type { NetlifyBlobsLike } from './netlify.js';
|
|
3
|
+
export { cloudflareKvStore } from './cloudflare.js';
|
|
4
|
+
export type { KVNamespaceLike } from './cloudflare.js';
|
|
5
|
+
export { upstashStore } from './upstash.js';
|
|
6
|
+
export type { UpstashOptions } from './upstash.js';
|
|
7
|
+
export { denoKvStore } from './deno.js';
|
|
8
|
+
export type { DenoKvLike } from './deno.js';
|
|
9
|
+
export { safeKey } from './safeKey.js';
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { KeyValueStore } from '../types.js';
|
|
2
|
+
/** What it needs from a Netlify Blobs store (`getStore('name')` from @netlify/blobs). */
|
|
3
|
+
export interface NetlifyBlobsLike {
|
|
4
|
+
get(key: string, opts: {
|
|
5
|
+
type: 'json';
|
|
6
|
+
}): Promise<unknown>;
|
|
7
|
+
setJSON(key: string, value: unknown): Promise<unknown>;
|
|
8
|
+
}
|
|
9
|
+
/** Netlify Blobs as a Lodestone store. Blobs cannot expire values, so each is saved with the time it stops being good and ignored after that. Fine for the catalogue, cached answers and which models are failing. Not a good home for rate-limit or spend counters under load: Blobs has no atomic add, and writes from separate functions can overwrite each other. Use Upstash for those. */
|
|
10
|
+
export declare function netlifyBlobsStore(blobs: NetlifyBlobsLike, o?: {
|
|
11
|
+
now?: () => number;
|
|
12
|
+
}): KeyValueStore;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { safeKey, unwrap, wrap } from './safeKey.js';
|
|
2
|
+
/** Netlify Blobs as a Lodestone store. Blobs cannot expire values, so each is saved with the time it stops being good and ignored after that. Fine for the catalogue, cached answers and which models are failing. Not a good home for rate-limit or spend counters under load: Blobs has no atomic add, and writes from separate functions can overwrite each other. Use Upstash for those. */
|
|
3
|
+
export function netlifyBlobsStore(blobs, o = {}) {
|
|
4
|
+
const now = o.now ?? Date.now;
|
|
5
|
+
return {
|
|
6
|
+
async get(key) { return unwrap(await blobs.get(await safeKey(key), { type: 'json' }), now()); },
|
|
7
|
+
async set(key, value, opts) { await blobs.setJSON(await safeKey(key), wrap(value, opts?.ttlMs, now())); },
|
|
8
|
+
};
|
|
9
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/** A key every store accepts: letters, digits, dot, dash and underscore stay as they are, anything else becomes ~ and two hex digits
|
|
2
|
+
* (so the same key always gives the same result, and two different keys never give the same one). A key too long for a store is replaced by its hash. */
|
|
3
|
+
export declare function safeKey(key: string, max?: number): Promise<string>;
|
|
4
|
+
/** For stores with no expiry of their own: a value wrapped with the time it stops being good. A value written by something else is passed through. */
|
|
5
|
+
export declare const wrap: (v: unknown, ttlMs: number | undefined, now: number) => {
|
|
6
|
+
exp?: number | undefined;
|
|
7
|
+
__lodestone: number;
|
|
8
|
+
v: unknown;
|
|
9
|
+
};
|
|
10
|
+
export declare function unwrap(raw: unknown, now: number): unknown;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { sha256Hex } from '../util.js';
|
|
2
|
+
/** A key every store accepts: letters, digits, dot, dash and underscore stay as they are, anything else becomes ~ and two hex digits
|
|
3
|
+
* (so the same key always gives the same result, and two different keys never give the same one). A key too long for a store is replaced by its hash. */
|
|
4
|
+
export async function safeKey(key, max = 200) {
|
|
5
|
+
let out = '';
|
|
6
|
+
for (const b of new TextEncoder().encode(key)) {
|
|
7
|
+
const c = String.fromCharCode(b);
|
|
8
|
+
out += /[A-Za-z0-9._-]/.test(c) ? c : `~${b.toString(16).padStart(2, '0')}`;
|
|
9
|
+
}
|
|
10
|
+
return out.length <= max ? out : `h-${await sha256Hex(key, 32)}`;
|
|
11
|
+
}
|
|
12
|
+
/** For stores with no expiry of their own: a value wrapped with the time it stops being good. A value written by something else is passed through. */
|
|
13
|
+
export const wrap = (v, ttlMs, now) => ({ __lodestone: 1, v, ...(ttlMs ? { exp: now + ttlMs } : {}) });
|
|
14
|
+
export function unwrap(raw, now) {
|
|
15
|
+
if (typeof raw === 'object' && raw !== null && raw.__lodestone === 1) {
|
|
16
|
+
const { v, exp } = raw;
|
|
17
|
+
return typeof exp === 'number' && exp <= now ? null : v ?? null;
|
|
18
|
+
}
|
|
19
|
+
return raw ?? null;
|
|
20
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { AtomicStore } from '../types.js';
|
|
2
|
+
export interface UpstashOptions {
|
|
3
|
+
/** The REST URL of the database, https://....upstash.io (also what Vercel KV gives you as KV_REST_API_URL). */
|
|
4
|
+
url: string;
|
|
5
|
+
/** The REST token. Keep it in an environment variable. */
|
|
6
|
+
token: string;
|
|
7
|
+
/** Put in front of every key. Default lodestone: */
|
|
8
|
+
prefix?: string;
|
|
9
|
+
fetch?: typeof fetch;
|
|
10
|
+
timeoutMs?: number;
|
|
11
|
+
}
|
|
12
|
+
/** Upstash Redis (and Vercel KV, which is Upstash) as a Lodestone store, over its REST API with plain fetch: no client library, so it runs on Workers, Deno, Bun and Node.
|
|
13
|
+
* It can count atomically, so a rate limiter that uses it is exact across every function, which no get/set store can offer. */
|
|
14
|
+
export declare function upstashStore(o: UpstashOptions): AtomicStore;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/** Upstash Redis (and Vercel KV, which is Upstash) as a Lodestone store, over its REST API with plain fetch: no client library, so it runs on Workers, Deno, Bun and Node.
|
|
2
|
+
* It can count atomically, so a rate limiter that uses it is exact across every function, which no get/set store can offer. */
|
|
3
|
+
export function upstashStore(o) {
|
|
4
|
+
const base = new URL(o.url);
|
|
5
|
+
if (base.protocol !== 'https:')
|
|
6
|
+
throw new Error('upstashStore: the URL must be https');
|
|
7
|
+
const f = o.fetch ?? fetch;
|
|
8
|
+
const prefix = o.prefix ?? 'lodestone:';
|
|
9
|
+
async function send(path, body) {
|
|
10
|
+
const res = await f(new URL(path, base), {
|
|
11
|
+
method: 'POST',
|
|
12
|
+
headers: { Authorization: `Bearer ${o.token}`, 'Content-Type': 'application/json' },
|
|
13
|
+
body: JSON.stringify(body),
|
|
14
|
+
signal: AbortSignal.timeout(o.timeoutMs ?? 5_000),
|
|
15
|
+
});
|
|
16
|
+
if (!res.ok)
|
|
17
|
+
throw new Error(`upstash: ${res.status}`); // the body is not included: it can echo a key
|
|
18
|
+
return res.json();
|
|
19
|
+
}
|
|
20
|
+
const one = async (cmd) => {
|
|
21
|
+
const r = (await send('/', cmd));
|
|
22
|
+
if (r.error)
|
|
23
|
+
throw new Error('upstash: the command was refused');
|
|
24
|
+
return r.result;
|
|
25
|
+
};
|
|
26
|
+
return {
|
|
27
|
+
async get(key) {
|
|
28
|
+
const raw = await one(['GET', prefix + key]);
|
|
29
|
+
if (typeof raw !== 'string')
|
|
30
|
+
return null;
|
|
31
|
+
try {
|
|
32
|
+
return JSON.parse(raw);
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
async set(key, value, opts) {
|
|
39
|
+
await one(['SET', prefix + key, JSON.stringify(value), ...(opts?.ttlMs ? ['PX', String(Math.ceil(opts.ttlMs))] : [])]);
|
|
40
|
+
},
|
|
41
|
+
async incrBy(key, by, opts) {
|
|
42
|
+
const k = prefix + key;
|
|
43
|
+
const r = (await send('/pipeline', [['INCRBY', k, by], ['PEXPIRE', k, opts.ttlMs, 'NX']]));
|
|
44
|
+
const n = r[0]?.result;
|
|
45
|
+
if (r[0]?.error || typeof n !== 'number')
|
|
46
|
+
throw new Error('upstash: the counter was refused');
|
|
47
|
+
return n;
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
}
|