@convex-dev/ai-budget 0.0.2-alpha.16 → 0.0.2-alpha.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -3
- package/dist/client/dashboard.js +12 -2
- package/dist/client/index.js +89 -59
- package/dist/component/lib.d.ts +3 -3
- package/dist/component/lib.js +101 -28
- package/package.json +1 -1
- package/src/client/dashboard.ts +12 -2
- package/src/client/index.ts +92 -54
- package/src/component/lib.test.ts +95 -11
- package/src/component/lib.ts +104 -30
package/README.md
CHANGED
|
@@ -390,6 +390,13 @@ reserve-then-settle admission check runs per bucket. Uncapped buckets never seri
|
|
|
390
390
|
adding tags you don't cap is free at admission; their totals still accrue for
|
|
391
391
|
reporting.
|
|
392
392
|
|
|
393
|
+
> **Choosing what to cap.** A cap makes its bucket's admissions atomic by
|
|
394
|
+
> reserving on that bucket's single row, so capping a dimension that *all* traffic
|
|
395
|
+
> shares (one `env`/`action` every request carries) serializes those admissions on
|
|
396
|
+
> one document under load. Prefer capping **naturally-sharded** dimensions — per
|
|
397
|
+
> `user`, per `customer` — and use the [global cap](#global-cap) for a
|
|
398
|
+
> deployment-wide ceiling. Uncapped high-traffic tags are always free.
|
|
399
|
+
|
|
393
400
|
### Tags — budgeting by any dimension
|
|
394
401
|
|
|
395
402
|
```ts
|
|
@@ -488,9 +495,12 @@ ai.global.status(ctx) // { limits, spentTodayNanos, spentTotalNanos, … }
|
|
|
488
495
|
ai.global.bump(ctx, { dailyNanos?, lifetimeNanos? })
|
|
489
496
|
```
|
|
490
497
|
|
|
491
|
-
A killswitch across everything. Backed by a sharded counter for
|
|
492
|
-
it's enforced **approximately**
|
|
493
|
-
|
|
498
|
+
A best-effort killswitch across everything. Backed by a sharded counter for
|
|
499
|
+
throughput, so it's enforced **approximately** — under burst it can overshoot the
|
|
500
|
+
cap by a bounded amount, and it excludes in-flight (not-yet-settled) spend. It is
|
|
501
|
+
**not** a to-the-dollar ceiling: for an exact limit use a per-bucket cap (those
|
|
502
|
+
reserve atomically); reach for the global cap when you want a deployment-wide
|
|
503
|
+
"stop everything" switch.
|
|
494
504
|
|
|
495
505
|
### Model policy
|
|
496
506
|
|
|
@@ -552,6 +562,12 @@ ai.global.setRetention(ctx, { retentionMs }) // default 1h; 0 disables
|
|
|
552
562
|
Full request rows (prompts + responses) are swept after the window to bound the
|
|
553
563
|
audit table. **Spend history survives** — it lives in separate durable rollups.
|
|
554
564
|
|
|
565
|
+
> **Prompt/response content is stored** on the request row until the window sweeps
|
|
566
|
+
> it (default 1h), so it's visible in the request log and to admin reads until
|
|
567
|
+
> then. If your prompts carry PII you don't want retained, shorten `retentionMs`
|
|
568
|
+
> (or set it low enough that content lives only as long as you need replay). Spend
|
|
569
|
+
> rollups never contain content, so trimming retention doesn't cost you history.
|
|
570
|
+
|
|
555
571
|
---
|
|
556
572
|
|
|
557
573
|
## Admin dashboard
|
package/dist/client/dashboard.js
CHANGED
|
@@ -97,8 +97,18 @@ async function renderBuckets() {
|
|
|
97
97
|
const dims = ["user", "action"];
|
|
98
98
|
const state = { dimension: window.__dim ?? "" };
|
|
99
99
|
const rows = await get("/buckets", state.dimension ? { dimension: state.dimension } : {});
|
|
100
|
-
|
|
101
|
-
|
|
100
|
+
// Within ONE dimension each request bills exactly one bucket, so summing is a
|
|
101
|
+
// true total. Across dimensions a request bills several buckets (user + action
|
|
102
|
+
// + tags), so summing multi-counts — use the deployment-wide spend instead.
|
|
103
|
+
let totalText;
|
|
104
|
+
if (state.dimension) {
|
|
105
|
+
const grand = rows.reduce((s, b) => s + b.totalSpendNanos, 0);
|
|
106
|
+
totalText = rows.length + " buckets · " + usd(grand) + " total (" + state.dimension + ")";
|
|
107
|
+
} else {
|
|
108
|
+
const g = await get("/global", {});
|
|
109
|
+
totalText = rows.length + " buckets · " + usd(g.spentTotalNanos ?? 0) + " spent (deployment)";
|
|
110
|
+
}
|
|
111
|
+
document.getElementById("total").textContent = totalText;
|
|
102
112
|
const dimSet = [...new Set(rows.map((b) => b.dimension).concat(dims))];
|
|
103
113
|
const sel = el("select", { value: state.dimension, style: "width:140px",
|
|
104
114
|
onchange: (e) => { window.__dim = e.target.value; render(); } },
|
package/dist/client/index.js
CHANGED
|
@@ -39,8 +39,10 @@ function toTokenCount(x) {
|
|
|
39
39
|
}
|
|
40
40
|
function extractUsage(usage) {
|
|
41
41
|
return {
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
// Cover AI SDK camelCase (v5/v7) AND raw OpenAI-compatible snake_case, so a
|
|
43
|
+
// `meter` caller passing a raw provider `usage` object still gets counts.
|
|
44
|
+
promptTokens: toTokenCount(usage?.inputTokens ?? usage?.promptTokens ?? usage?.prompt_tokens),
|
|
45
|
+
completionTokens: toTokenCount(usage?.outputTokens ?? usage?.completionTokens ?? usage?.completion_tokens),
|
|
44
46
|
// cached prompt tokens. The Convex gateway reports these at
|
|
45
47
|
// `usage.inputTokenDetails.cacheReadTokens`; the other paths cover AI SDK v5
|
|
46
48
|
// (`cachedInputTokens`) and raw OpenAI-compatible shapes.
|
|
@@ -240,35 +242,42 @@ export class AIBudget {
|
|
|
240
242
|
}
|
|
241
243
|
const { requestId, warnings, notices } = started;
|
|
242
244
|
const start = Date.now();
|
|
245
|
+
// Run the provider call. ONLY a failure of the call itself settles as an
|
|
246
|
+
// error (no charge expected).
|
|
247
|
+
let out;
|
|
243
248
|
try {
|
|
244
|
-
|
|
245
|
-
const { costNanos } = await this.settle(ctx, {
|
|
246
|
-
requestId,
|
|
247
|
-
responseText: out.text,
|
|
248
|
-
usage: out.usage,
|
|
249
|
-
promptTokens: out.promptTokens,
|
|
250
|
-
completionTokens: out.completionTokens,
|
|
251
|
-
cachedTokens: out.cachedTokens,
|
|
252
|
-
serverToolUses: out.serverToolUses,
|
|
253
|
-
costNanos: out.costNanos,
|
|
254
|
-
latencyMs: Date.now() - start,
|
|
255
|
-
});
|
|
256
|
-
// Re-derive the recorded usage for the return value.
|
|
257
|
-
const usage = out.promptTokens !== undefined ||
|
|
258
|
-
out.completionTokens !== undefined ||
|
|
259
|
-
out.cachedTokens !== undefined
|
|
260
|
-
? {
|
|
261
|
-
promptTokens: out.promptTokens ?? 0,
|
|
262
|
-
completionTokens: out.completionTokens ?? 0,
|
|
263
|
-
cachedTokens: out.cachedTokens ?? 0,
|
|
264
|
-
}
|
|
265
|
-
: extractUsage(out.usage);
|
|
266
|
-
return { text: out.text ?? "", requestId, costNanos, warnings, notices, ...usage };
|
|
249
|
+
out = await run();
|
|
267
250
|
}
|
|
268
251
|
catch (e) {
|
|
269
252
|
await this.settle(ctx, { requestId, error: String(e), latencyMs: Date.now() - start });
|
|
270
253
|
throw e;
|
|
271
254
|
}
|
|
255
|
+
// The call SUCCEEDED (the provider may have charged). Settle the real usage.
|
|
256
|
+
// If settlement itself fails here, do NOT fall into an error-settle that
|
|
257
|
+
// records zero — that would erase a real charge. Rethrow and leave the
|
|
258
|
+
// reservation for the reconciler; billing stays "unknown", never a false zero.
|
|
259
|
+
const { costNanos } = await this.settle(ctx, {
|
|
260
|
+
requestId,
|
|
261
|
+
responseText: out.text,
|
|
262
|
+
usage: out.usage,
|
|
263
|
+
promptTokens: out.promptTokens,
|
|
264
|
+
completionTokens: out.completionTokens,
|
|
265
|
+
cachedTokens: out.cachedTokens,
|
|
266
|
+
serverToolUses: out.serverToolUses,
|
|
267
|
+
costNanos: out.costNanos,
|
|
268
|
+
latencyMs: Date.now() - start,
|
|
269
|
+
});
|
|
270
|
+
// Re-derive the recorded usage for the return value.
|
|
271
|
+
const usage = out.promptTokens !== undefined ||
|
|
272
|
+
out.completionTokens !== undefined ||
|
|
273
|
+
out.cachedTokens !== undefined
|
|
274
|
+
? {
|
|
275
|
+
promptTokens: out.promptTokens ?? 0,
|
|
276
|
+
completionTokens: out.completionTokens ?? 0,
|
|
277
|
+
cachedTokens: out.cachedTokens ?? 0,
|
|
278
|
+
}
|
|
279
|
+
: extractUsage(out.usage);
|
|
280
|
+
return { text: out.text ?? "", requestId, costNanos, warnings, notices, ...usage };
|
|
272
281
|
}
|
|
273
282
|
/**
|
|
274
283
|
* One-shot chat through the AI Gateway with tracking + limits — sugar over
|
|
@@ -415,15 +424,10 @@ export class AIBudget {
|
|
|
415
424
|
wrapGenerate: async ({ doGenerate, params }) => {
|
|
416
425
|
const requestId = await begin(params);
|
|
417
426
|
const start = Date.now();
|
|
427
|
+
// Only a failure of the generation itself settles as an error.
|
|
428
|
+
let result;
|
|
418
429
|
try {
|
|
419
|
-
|
|
420
|
-
await finish(requestId, {
|
|
421
|
-
responseText: extractText(result),
|
|
422
|
-
...extractUsage(result.usage),
|
|
423
|
-
costNanos: extractGatewayCostNanos(result),
|
|
424
|
-
latencyMs: Date.now() - start,
|
|
425
|
-
});
|
|
426
|
-
return result;
|
|
430
|
+
result = await doGenerate();
|
|
427
431
|
}
|
|
428
432
|
catch (e) {
|
|
429
433
|
await finish(requestId, {
|
|
@@ -432,6 +436,15 @@ export class AIBudget {
|
|
|
432
436
|
});
|
|
433
437
|
throw e;
|
|
434
438
|
}
|
|
439
|
+
// Generation succeeded (provider may have charged). Settle the real
|
|
440
|
+
// usage; a failure here rethrows rather than recording a false zero.
|
|
441
|
+
await finish(requestId, {
|
|
442
|
+
responseText: extractText(result),
|
|
443
|
+
...extractUsage(result.usage),
|
|
444
|
+
costNanos: extractGatewayCostNanos(result),
|
|
445
|
+
latencyMs: Date.now() - start,
|
|
446
|
+
});
|
|
447
|
+
return result;
|
|
435
448
|
},
|
|
436
449
|
wrapStream: async ({ doStream, params }) => {
|
|
437
450
|
const requestId = await begin(params);
|
|
@@ -446,21 +459,22 @@ export class AIBudget {
|
|
|
446
459
|
// chunk, or a cancel — is safe: the first wins, the rest no-op.
|
|
447
460
|
// Without this an errored or abandoned stream would never settle and
|
|
448
461
|
// its real usage would be lost (recorded as free by the reconciler).
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
})
|
|
461
|
-
|
|
462
|
+
// Settle at most once, memoizing the PROMISE so the finish chunk, an
|
|
463
|
+
// error chunk, and flush all await the same settlement instead of
|
|
464
|
+
// racing, dropping it (the old `void settle()`), or flipping a
|
|
465
|
+
// "settled" flag before the mutation actually committed. A stream
|
|
466
|
+
// that is cancelled/never fully consumed won't deliver finish or
|
|
467
|
+
// flush; the reconciler's reservation expiry is the backstop there.
|
|
468
|
+
let settlement;
|
|
469
|
+
const settle = (error) => (settlement ??= finish(requestId, {
|
|
470
|
+
responseText: text,
|
|
471
|
+
error,
|
|
472
|
+
...extractUsage(usage),
|
|
473
|
+
costNanos: extractGatewayCostNanos({ providerMetadata }),
|
|
474
|
+
latencyMs: Date.now() - start,
|
|
475
|
+
}));
|
|
462
476
|
const tapped = result.stream.pipeThrough(new TransformStream({
|
|
463
|
-
transform(chunk, controller) {
|
|
477
|
+
async transform(chunk, controller) {
|
|
464
478
|
if (chunk?.type === "text-delta") {
|
|
465
479
|
text += chunk.delta ?? chunk.textDelta ?? "";
|
|
466
480
|
}
|
|
@@ -468,9 +482,11 @@ export class AIBudget {
|
|
|
468
482
|
usage = chunk.usage;
|
|
469
483
|
providerMetadata = chunk.providerMetadata ?? providerMetadata;
|
|
470
484
|
}
|
|
471
|
-
if (chunk?.type === "error")
|
|
472
|
-
void settle(String(chunk.error));
|
|
473
485
|
controller.enqueue(chunk);
|
|
486
|
+
// Settle after forwarding the terminal error chunk, and AWAIT
|
|
487
|
+
// it so a failed settle surfaces instead of being dropped.
|
|
488
|
+
if (chunk?.type === "error")
|
|
489
|
+
await settle(String(chunk.error));
|
|
474
490
|
},
|
|
475
491
|
async flush() {
|
|
476
492
|
await settle();
|
|
@@ -658,16 +674,26 @@ export class AIBudget {
|
|
|
658
674
|
return { ok: false, token: "" };
|
|
659
675
|
const url = new URL(request.url);
|
|
660
676
|
const bearer = (request.headers.get("authorization") ?? "").replace(/^Bearer\s+/i, "");
|
|
661
|
-
// `?token=` is accepted
|
|
662
|
-
//
|
|
663
|
-
// JSON API
|
|
664
|
-
|
|
677
|
+
// `?token=` is accepted ONLY for the initial page navigation (a browser GET
|
|
678
|
+
// can't set headers); the page strips it from the URL on load and calls the
|
|
679
|
+
// JSON API with the bearer header. Restrict it to that GET page route so a
|
|
680
|
+
// token can't be smuggled in a query string on API/mutation calls (where it
|
|
681
|
+
// would also land in access logs). Compared in constant time.
|
|
682
|
+
const sub = url.pathname.slice(prefix.length) || "/";
|
|
683
|
+
const isPageNav = request.method === "GET" && !sub.startsWith("/api");
|
|
684
|
+
const provided = bearer || (isPageNav ? url.searchParams.get("token") ?? "" : "");
|
|
665
685
|
return { ok: timingSafeEqual(provided, token), token };
|
|
666
686
|
};
|
|
667
687
|
const json = (data, status = 200) => new Response(JSON.stringify(data ?? null), {
|
|
668
688
|
status,
|
|
669
|
-
|
|
689
|
+
// Never let a shared cache/proxy retain budget data or the token-bearing
|
|
690
|
+
// page — these responses are per-viewer and sensitive.
|
|
691
|
+
headers: { "content-type": "application/json", "cache-control": "no-store" },
|
|
670
692
|
});
|
|
693
|
+
// JSON.stringify does NOT escape `<`, so a value containing `</script>`
|
|
694
|
+
// would close the inline <script> and break out. Escape `<` (and the JS line
|
|
695
|
+
// separators) before embedding in HTML.
|
|
696
|
+
const jsonForScript = (x) => JSON.stringify(x).replace(/[<\u2028\u2029]/g, (ch) => "\\u" + ch.charCodeAt(0).toString(16).padStart(4, "0"));
|
|
671
697
|
const handle = async (ctx, request) => {
|
|
672
698
|
const url = new URL(request.url);
|
|
673
699
|
const sub = url.pathname.slice(prefix.length) || "/";
|
|
@@ -724,12 +750,16 @@ export class AIBudget {
|
|
|
724
750
|
return json({ error: "not found" }, 404);
|
|
725
751
|
}
|
|
726
752
|
}
|
|
727
|
-
// Inject as JSON literals (function replacers so `$` in the
|
|
728
|
-
// treated as a replacement pattern
|
|
729
|
-
//
|
|
730
|
-
const html = DASHBOARD_HTML.replace(/__API_BASE__/g, () =>
|
|
753
|
+
// Inject as script-safe JSON literals (function replacers so `$` in the
|
|
754
|
+
// value isn't treated as a replacement pattern; `jsonForScript` escapes
|
|
755
|
+
// `<` so a token containing `</script>` can't break out of the inline JS).
|
|
756
|
+
const html = DASHBOARD_HTML.replace(/__API_BASE__/g, () => jsonForScript(`${prefix}/api`)).replace(/__TOKEN__/g, () => jsonForScript(token));
|
|
757
|
+
// The page embeds the bearer token — never let a shared cache retain it.
|
|
731
758
|
return new Response(html, {
|
|
732
|
-
headers: {
|
|
759
|
+
headers: {
|
|
760
|
+
"content-type": "text/html; charset=utf-8",
|
|
761
|
+
"cache-control": "no-store",
|
|
762
|
+
},
|
|
733
763
|
});
|
|
734
764
|
};
|
|
735
765
|
const handler = httpActionGeneric(handle);
|
package/dist/component/lib.d.ts
CHANGED
|
@@ -377,9 +377,9 @@ export declare const getGlobalStatus: import("convex/server").RegisteredQuery<"p
|
|
|
377
377
|
defaultWarnAtPct: number | null;
|
|
378
378
|
}>>;
|
|
379
379
|
export declare const setGlobalLimits: import("convex/server").RegisteredMutation<"public", {
|
|
380
|
-
dailySpendLimitNanos?: number | undefined;
|
|
381
|
-
lifetimeSpendLimitNanos?: number | undefined;
|
|
382
|
-
enforcement?: "hard" | "soft" | undefined;
|
|
380
|
+
dailySpendLimitNanos?: number | null | undefined;
|
|
381
|
+
lifetimeSpendLimitNanos?: number | null | undefined;
|
|
382
|
+
enforcement?: "hard" | "soft" | null | undefined;
|
|
383
383
|
}, Promise<null>>;
|
|
384
384
|
export declare const bumpGlobal: import("convex/server").RegisteredMutation<"public", {
|
|
385
385
|
dailyNanos?: number | undefined;
|
package/dist/component/lib.js
CHANGED
|
@@ -12,6 +12,30 @@ import { ShardedCounter } from "@convex-dev/sharded-counter";
|
|
|
12
12
|
// currency code alongside these amounts and convert here, at the one boundary.
|
|
13
13
|
const NANOS_PER_DOLLAR = 1e9;
|
|
14
14
|
const fmtUsd = (nanos) => `$${(nanos / NANOS_PER_DOLLAR).toFixed(4)}`;
|
|
15
|
+
// Convex's `v.number()` accepts NaN and ±Infinity. Those are poison here: a NaN
|
|
16
|
+
// cap or count silently defeats every `used > cap` comparison (NaN > x is
|
|
17
|
+
// false), so an unvalidated NaN would make admission fail OPEN and admit
|
|
18
|
+
// unlimited spend; +Infinity in totals is just as corrupting. Validate every
|
|
19
|
+
// externally-supplied accounting amount at the mutation boundary.
|
|
20
|
+
function assertAmount(n, name, { signed = false } = {}) {
|
|
21
|
+
if (n === undefined)
|
|
22
|
+
return;
|
|
23
|
+
if (!Number.isFinite(n) || !Number.isSafeInteger(n)) {
|
|
24
|
+
throw new Error(`${name} must be a finite safe integer (got ${n})`);
|
|
25
|
+
}
|
|
26
|
+
if (!signed && n < 0)
|
|
27
|
+
throw new Error(`${name} must be nonnegative (got ${n})`);
|
|
28
|
+
}
|
|
29
|
+
function assertFraction(n, name) {
|
|
30
|
+
if (n === undefined)
|
|
31
|
+
return;
|
|
32
|
+
if (!Number.isFinite(n) || n < 0 || n > 1) {
|
|
33
|
+
throw new Error(`${name} must be a number in [0, 1] (got ${n})`);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
// Coerce a caller/provider-supplied count to a finite nonnegative integer,
|
|
37
|
+
// mapping NaN/Infinity/garbage to 0 rather than poisoning downstream totals.
|
|
38
|
+
const safeCount = (n) => Number.isFinite(n) ? Math.max(0, Math.floor(n)) : 0;
|
|
15
39
|
// Built-in attribution dimensions. `user` and `action` are always populated
|
|
16
40
|
// from a request's userId/actionName; apps can add any other dimensions
|
|
17
41
|
// (team, project, customer, env, …) as tags. These two names are reserved —
|
|
@@ -525,8 +549,14 @@ export const startRequest = mutation({
|
|
|
525
549
|
notices.push(...globalEval.notices);
|
|
526
550
|
}
|
|
527
551
|
// Consume only after every admission check succeeds, in the same
|
|
528
|
-
// transaction as the reservations and request insert.
|
|
529
|
-
//
|
|
552
|
+
// transaction as the reservations and request insert. `throws: true` is
|
|
553
|
+
// deliberate, not a rough edge: we already `.check`ed every bucket above in
|
|
554
|
+
// this same serializable transaction, so a `.limit` here cannot fail on a
|
|
555
|
+
// bucket that passed check — and if it somehow did (or a later bucket did),
|
|
556
|
+
// throwing rolls back the WHOLE transaction, including the rate we already
|
|
557
|
+
// consumed on earlier buckets. Converting this to a graceful `{allowed:false}`
|
|
558
|
+
// return would COMMIT the partial consumption and leak rate capacity, so keep
|
|
559
|
+
// the throw.
|
|
530
560
|
for (const b of buckets) {
|
|
531
561
|
if (b.requestsPerMinute !== undefined) {
|
|
532
562
|
await requestRateLimiter.limit(ctx, "requests", {
|
|
@@ -612,23 +642,28 @@ export const finishRequest = mutation({
|
|
|
612
642
|
returns: v.object({ costNanos: v.number() }),
|
|
613
643
|
handler: async (ctx, args) => {
|
|
614
644
|
const request = await ctx.db.get(args.requestId);
|
|
645
|
+
// The request may be gone — retention purged it, or the owning bucket was
|
|
646
|
+
// deleted. A late/duplicate webhook must be an idempotent no-op, not a 500
|
|
647
|
+
// (the caller can't do anything useful with the error, and it triggers retries).
|
|
615
648
|
if (!request)
|
|
616
|
-
|
|
649
|
+
return { costNanos: 0 };
|
|
617
650
|
// Expiry releases capacity, but is not evidence that the provider charged
|
|
618
651
|
// nothing. Accept one final result even after expiry; duplicates stay no-ops.
|
|
619
652
|
if (request.status !== "pending" && !request.reservationExpired) {
|
|
620
653
|
return { costNanos: request.costNanos ?? 0 };
|
|
621
654
|
}
|
|
622
|
-
//
|
|
623
|
-
//
|
|
624
|
-
|
|
625
|
-
const
|
|
626
|
-
const
|
|
655
|
+
// Coerce caller/provider-supplied token counts to finite nonnegative
|
|
656
|
+
// integers: negatives would refund below a cap, and NaN/Infinity (which
|
|
657
|
+
// v.number() allows) would poison every downstream total and cap check.
|
|
658
|
+
const promptTokens = safeCount(args.promptTokens);
|
|
659
|
+
const completionTokens = safeCount(args.completionTokens);
|
|
660
|
+
const cachedTokens = Math.min(promptTokens, safeCount(args.cachedTokens));
|
|
627
661
|
// Prefer an authoritative gateway cost when supplied (it already includes
|
|
628
662
|
// tool fees); otherwise price from tokens — discounting the cached
|
|
629
|
-
// (prompt-cache-read) slice — plus any server-tool per-call fees.
|
|
663
|
+
// (prompt-cache-read) slice — plus any server-tool per-call fees. Require it
|
|
664
|
+
// finite: +Infinity passes a bare `>= 0` and would corrupt the totals.
|
|
630
665
|
let costNanos;
|
|
631
|
-
if (args.costNanos !== undefined && args.costNanos >= 0) {
|
|
666
|
+
if (args.costNanos !== undefined && Number.isFinite(args.costNanos) && args.costNanos >= 0) {
|
|
632
667
|
costNanos = Math.round(args.costNanos);
|
|
633
668
|
}
|
|
634
669
|
else {
|
|
@@ -937,10 +972,15 @@ export const setBucketLimits = mutation({
|
|
|
937
972
|
},
|
|
938
973
|
returns: v.null(),
|
|
939
974
|
handler: async (ctx, args) => {
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
975
|
+
assertAmount(args.requestsPerMinute, "requestsPerMinute");
|
|
976
|
+
assertAmount(args.maxConcurrent, "maxConcurrent");
|
|
977
|
+
assertAmount(args.dailySpendLimitNanos, "dailySpendLimitNanos");
|
|
978
|
+
assertAmount(args.monthlySpendLimitNanos, "monthlySpendLimitNanos");
|
|
979
|
+
assertAmount(args.lifetimeSpendLimitNanos, "lifetimeSpendLimitNanos");
|
|
980
|
+
assertAmount(args.dailyTokenLimit, "dailyTokenLimit");
|
|
981
|
+
assertAmount(args.monthlyTokenLimit, "monthlyTokenLimit");
|
|
982
|
+
assertAmount(args.lifetimeTokenLimit, "lifetimeTokenLimit");
|
|
983
|
+
assertFraction(args.warnAtPct, "warnAtPct");
|
|
944
984
|
const bucket = await getOrCreateBucket(ctx, args.dimension, args.value);
|
|
945
985
|
const { dimension: _d, value: _v, ...limits } = args;
|
|
946
986
|
await ctx.db.patch(bucket._id, limits);
|
|
@@ -960,6 +1000,9 @@ export const bumpBucket = mutation({
|
|
|
960
1000
|
args: { dimension: v.string(), value: v.string(), ...vBumpArgs },
|
|
961
1001
|
returns: v.null(),
|
|
962
1002
|
handler: async (ctx, { dimension, value, dailyNanos, monthlyNanos, lifetimeNanos }) => {
|
|
1003
|
+
assertAmount(dailyNanos, "dailyNanos");
|
|
1004
|
+
assertAmount(monthlyNanos, "monthlyNanos");
|
|
1005
|
+
assertAmount(lifetimeNanos, "lifetimeNanos");
|
|
963
1006
|
const bucket = await getOrCreateBucket(ctx, dimension, value);
|
|
964
1007
|
const today = dayStamp();
|
|
965
1008
|
const month = monthStamp();
|
|
@@ -989,6 +1032,8 @@ export const adjustBucket = mutation({
|
|
|
989
1032
|
},
|
|
990
1033
|
returns: v.null(),
|
|
991
1034
|
handler: async (ctx, { dimension, value, deltaNanos, tokens, reason }) => {
|
|
1035
|
+
assertAmount(deltaNanos, "deltaNanos", { signed: true });
|
|
1036
|
+
assertAmount(tokens, "tokens", { signed: true });
|
|
992
1037
|
const b = await getOrCreateBucket(ctx, dimension, value);
|
|
993
1038
|
const today = dayStamp();
|
|
994
1039
|
const month = monthStamp();
|
|
@@ -1004,6 +1049,12 @@ export const adjustBucket = mutation({
|
|
|
1004
1049
|
tokensToday: Math.max(0, (dSame ? b.tokensToday ?? 0 : 0) + dt),
|
|
1005
1050
|
spendThisMonthNanos: Math.max(0, (mSame ? b.spendThisMonthNanos ?? 0 : 0) + deltaNanos),
|
|
1006
1051
|
tokensThisMonth: Math.max(0, (mSame ? b.tokensThisMonth ?? 0 : 0) + dt),
|
|
1052
|
+
// Advancing the window here must also clear the OLD window's reserved
|
|
1053
|
+
// holds, or an in-flight request from the previous day/month would be
|
|
1054
|
+
// treated as reserving against the new window and its later release would
|
|
1055
|
+
// no longer match — stranding those reserved nanos/tokens.
|
|
1056
|
+
...(dSame ? {} : { reservedTodayNanos: 0, reservedTodayTokens: 0 }),
|
|
1057
|
+
...(mSame ? {} : { reservedMonthNanos: 0, reservedMonthTokens: 0 }),
|
|
1007
1058
|
});
|
|
1008
1059
|
await ctx.db.insert("adjustments", { dimension, value, deltaNanos, tokens: dt, reason });
|
|
1009
1060
|
await addUsage(ctx, dimension, value, "day", today, deltaNanos, dt, 0);
|
|
@@ -1064,6 +1115,17 @@ export const deleteBucket = mutation({
|
|
|
1064
1115
|
.withIndex("userId", (q) => q.eq("userId", value))
|
|
1065
1116
|
.take(DELETE_BATCH);
|
|
1066
1117
|
for (const r of rows) {
|
|
1118
|
+
// Before dropping the row, free or settle any hold it placed on OTHER
|
|
1119
|
+
// (shared) buckets — an action/customer bucket this user's request
|
|
1120
|
+
// reserved against. Otherwise deleting the only row that could release
|
|
1121
|
+
// that hold strands the shared bucket's reservation + pendingCount
|
|
1122
|
+
// forever, and a late finish would throw. A finished-but-unfolded row
|
|
1123
|
+
// is folded (the charge lands on the shared buckets); a still-pending
|
|
1124
|
+
// one just has its reservation released.
|
|
1125
|
+
if (r.settled === false)
|
|
1126
|
+
await foldOne(ctx, r);
|
|
1127
|
+
else if (r.status === "pending")
|
|
1128
|
+
await releaseReservation(ctx, r);
|
|
1067
1129
|
await deleteRequestTags(ctx, r._id);
|
|
1068
1130
|
await ctx.db.delete(r._id);
|
|
1069
1131
|
}
|
|
@@ -1139,18 +1201,29 @@ export const getGlobalStatus = query({
|
|
|
1139
1201
|
});
|
|
1140
1202
|
export const setGlobalLimits = mutation({
|
|
1141
1203
|
args: {
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1204
|
+
// Absent = leave unchanged; explicit null = clear that limit. (A bare
|
|
1205
|
+
// v.optional(number) that always rebuilt the full patch would let a
|
|
1206
|
+
// one-field edit — exactly what the dashboard sends — silently wipe the
|
|
1207
|
+
// other global controls by patching them to undefined.)
|
|
1208
|
+
dailySpendLimitNanos: v.optional(v.union(v.number(), v.null())),
|
|
1209
|
+
lifetimeSpendLimitNanos: v.optional(v.union(v.number(), v.null())),
|
|
1210
|
+
enforcement: v.optional(v.union(v.literal("hard"), v.literal("soft"), v.null())),
|
|
1145
1211
|
},
|
|
1146
1212
|
returns: v.null(),
|
|
1147
1213
|
handler: async (ctx, args) => {
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1214
|
+
if (typeof args.dailySpendLimitNanos === "number")
|
|
1215
|
+
assertAmount(args.dailySpendLimitNanos, "dailySpendLimitNanos");
|
|
1216
|
+
if (typeof args.lifetimeSpendLimitNanos === "number")
|
|
1217
|
+
assertAmount(args.lifetimeSpendLimitNanos, "lifetimeSpendLimitNanos");
|
|
1218
|
+
// The settings fields are "global"-prefixed; map the friendly arg names,
|
|
1219
|
+
// touching ONLY the keys the caller actually passed. null -> clear.
|
|
1220
|
+
const patch = {};
|
|
1221
|
+
if ("dailySpendLimitNanos" in args)
|
|
1222
|
+
patch.globalDailySpendLimitNanos = args.dailySpendLimitNanos ?? undefined;
|
|
1223
|
+
if ("lifetimeSpendLimitNanos" in args)
|
|
1224
|
+
patch.globalLifetimeSpendLimitNanos = args.lifetimeSpendLimitNanos ?? undefined;
|
|
1225
|
+
if ("enforcement" in args)
|
|
1226
|
+
patch.globalEnforcement = args.enforcement ?? undefined;
|
|
1154
1227
|
const existing = await getSettings(ctx);
|
|
1155
1228
|
if (existing) {
|
|
1156
1229
|
await ctx.db.patch(existing._id, patch);
|
|
@@ -1216,11 +1289,11 @@ export const setPrice = mutation({
|
|
|
1216
1289
|
handler: async (ctx, args) => {
|
|
1217
1290
|
// Negative prices would make costOf return a negative cost, which folds
|
|
1218
1291
|
// into totals as a spend *refund* — pushing a user back under their cap.
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1292
|
+
// NaN/Infinity (allowed by v.number()) are just as corrupting — a NaN rate
|
|
1293
|
+
// poisons every settled cost for the model — so require finite integers.
|
|
1294
|
+
assertAmount(args.inputNanosPerMTok, "inputNanosPerMTok");
|
|
1295
|
+
assertAmount(args.outputNanosPerMTok, "outputNanosPerMTok");
|
|
1296
|
+
assertAmount(args.cachedNanosPerMTok, "cachedNanosPerMTok");
|
|
1224
1297
|
const existing = await ctx.db
|
|
1225
1298
|
.query("prices")
|
|
1226
1299
|
.withIndex("model", (q) => q.eq("model", args.model))
|
package/package.json
CHANGED
package/src/client/dashboard.ts
CHANGED
|
@@ -97,8 +97,18 @@ async function renderBuckets() {
|
|
|
97
97
|
const dims = ["user", "action"];
|
|
98
98
|
const state = { dimension: window.__dim ?? "" };
|
|
99
99
|
const rows = await get("/buckets", state.dimension ? { dimension: state.dimension } : {});
|
|
100
|
-
|
|
101
|
-
|
|
100
|
+
// Within ONE dimension each request bills exactly one bucket, so summing is a
|
|
101
|
+
// true total. Across dimensions a request bills several buckets (user + action
|
|
102
|
+
// + tags), so summing multi-counts — use the deployment-wide spend instead.
|
|
103
|
+
let totalText;
|
|
104
|
+
if (state.dimension) {
|
|
105
|
+
const grand = rows.reduce((s, b) => s + b.totalSpendNanos, 0);
|
|
106
|
+
totalText = rows.length + " buckets · " + usd(grand) + " total (" + state.dimension + ")";
|
|
107
|
+
} else {
|
|
108
|
+
const g = await get("/global", {});
|
|
109
|
+
totalText = rows.length + " buckets · " + usd(g.spentTotalNanos ?? 0) + " spent (deployment)";
|
|
110
|
+
}
|
|
111
|
+
document.getElementById("total").textContent = totalText;
|
|
102
112
|
const dimSet = [...new Set(rows.map((b) => b.dimension).concat(dims))];
|
|
103
113
|
const sel = el("select", { value: state.dimension, style: "width:140px",
|
|
104
114
|
onchange: (e) => { window.__dim = e.target.value; render(); } },
|
package/src/client/index.ts
CHANGED
|
@@ -172,8 +172,14 @@ function extractUsage(usage: any): {
|
|
|
172
172
|
cachedTokens: number;
|
|
173
173
|
} {
|
|
174
174
|
return {
|
|
175
|
-
|
|
176
|
-
|
|
175
|
+
// Cover AI SDK camelCase (v5/v7) AND raw OpenAI-compatible snake_case, so a
|
|
176
|
+
// `meter` caller passing a raw provider `usage` object still gets counts.
|
|
177
|
+
promptTokens: toTokenCount(
|
|
178
|
+
usage?.inputTokens ?? usage?.promptTokens ?? usage?.prompt_tokens
|
|
179
|
+
),
|
|
180
|
+
completionTokens: toTokenCount(
|
|
181
|
+
usage?.outputTokens ?? usage?.completionTokens ?? usage?.completion_tokens
|
|
182
|
+
),
|
|
177
183
|
// cached prompt tokens. The Convex gateway reports these at
|
|
178
184
|
// `usage.inputTokenDetails.cacheReadTokens`; the other paths cover AI SDK v5
|
|
179
185
|
// (`cachedInputTokens`) and raw OpenAI-compatible shapes.
|
|
@@ -463,35 +469,42 @@ export class AIBudget {
|
|
|
463
469
|
}
|
|
464
470
|
const { requestId, warnings, notices } = started;
|
|
465
471
|
const start = Date.now();
|
|
472
|
+
// Run the provider call. ONLY a failure of the call itself settles as an
|
|
473
|
+
// error (no charge expected).
|
|
474
|
+
let out: Awaited<ReturnType<typeof run>>;
|
|
466
475
|
try {
|
|
467
|
-
|
|
468
|
-
const { costNanos } = await this.settle(ctx, {
|
|
469
|
-
requestId,
|
|
470
|
-
responseText: out.text,
|
|
471
|
-
usage: out.usage,
|
|
472
|
-
promptTokens: out.promptTokens,
|
|
473
|
-
completionTokens: out.completionTokens,
|
|
474
|
-
cachedTokens: out.cachedTokens,
|
|
475
|
-
serverToolUses: out.serverToolUses,
|
|
476
|
-
costNanos: out.costNanos,
|
|
477
|
-
latencyMs: Date.now() - start,
|
|
478
|
-
});
|
|
479
|
-
// Re-derive the recorded usage for the return value.
|
|
480
|
-
const usage =
|
|
481
|
-
out.promptTokens !== undefined ||
|
|
482
|
-
out.completionTokens !== undefined ||
|
|
483
|
-
out.cachedTokens !== undefined
|
|
484
|
-
? {
|
|
485
|
-
promptTokens: out.promptTokens ?? 0,
|
|
486
|
-
completionTokens: out.completionTokens ?? 0,
|
|
487
|
-
cachedTokens: out.cachedTokens ?? 0,
|
|
488
|
-
}
|
|
489
|
-
: extractUsage(out.usage);
|
|
490
|
-
return { text: out.text ?? "", requestId, costNanos, warnings, notices, ...usage };
|
|
476
|
+
out = await run();
|
|
491
477
|
} catch (e) {
|
|
492
478
|
await this.settle(ctx, { requestId, error: String(e), latencyMs: Date.now() - start });
|
|
493
479
|
throw e;
|
|
494
480
|
}
|
|
481
|
+
// The call SUCCEEDED (the provider may have charged). Settle the real usage.
|
|
482
|
+
// If settlement itself fails here, do NOT fall into an error-settle that
|
|
483
|
+
// records zero — that would erase a real charge. Rethrow and leave the
|
|
484
|
+
// reservation for the reconciler; billing stays "unknown", never a false zero.
|
|
485
|
+
const { costNanos } = await this.settle(ctx, {
|
|
486
|
+
requestId,
|
|
487
|
+
responseText: out.text,
|
|
488
|
+
usage: out.usage,
|
|
489
|
+
promptTokens: out.promptTokens,
|
|
490
|
+
completionTokens: out.completionTokens,
|
|
491
|
+
cachedTokens: out.cachedTokens,
|
|
492
|
+
serverToolUses: out.serverToolUses,
|
|
493
|
+
costNanos: out.costNanos,
|
|
494
|
+
latencyMs: Date.now() - start,
|
|
495
|
+
});
|
|
496
|
+
// Re-derive the recorded usage for the return value.
|
|
497
|
+
const usage =
|
|
498
|
+
out.promptTokens !== undefined ||
|
|
499
|
+
out.completionTokens !== undefined ||
|
|
500
|
+
out.cachedTokens !== undefined
|
|
501
|
+
? {
|
|
502
|
+
promptTokens: out.promptTokens ?? 0,
|
|
503
|
+
completionTokens: out.completionTokens ?? 0,
|
|
504
|
+
cachedTokens: out.cachedTokens ?? 0,
|
|
505
|
+
}
|
|
506
|
+
: extractUsage(out.usage);
|
|
507
|
+
return { text: out.text ?? "", requestId, costNanos, warnings, notices, ...usage };
|
|
495
508
|
}
|
|
496
509
|
|
|
497
510
|
/**
|
|
@@ -717,15 +730,10 @@ export class AIBudget {
|
|
|
717
730
|
wrapGenerate: async ({ doGenerate, params }: any) => {
|
|
718
731
|
const requestId = await begin(params);
|
|
719
732
|
const start = Date.now();
|
|
733
|
+
// Only a failure of the generation itself settles as an error.
|
|
734
|
+
let result: any;
|
|
720
735
|
try {
|
|
721
|
-
|
|
722
|
-
await finish(requestId, {
|
|
723
|
-
responseText: extractText(result),
|
|
724
|
-
...extractUsage(result.usage),
|
|
725
|
-
costNanos: extractGatewayCostNanos(result),
|
|
726
|
-
latencyMs: Date.now() - start,
|
|
727
|
-
});
|
|
728
|
-
return result;
|
|
736
|
+
result = await doGenerate();
|
|
729
737
|
} catch (e) {
|
|
730
738
|
await finish(requestId, {
|
|
731
739
|
error: String(e),
|
|
@@ -733,6 +741,15 @@ export class AIBudget {
|
|
|
733
741
|
});
|
|
734
742
|
throw e;
|
|
735
743
|
}
|
|
744
|
+
// Generation succeeded (provider may have charged). Settle the real
|
|
745
|
+
// usage; a failure here rethrows rather than recording a false zero.
|
|
746
|
+
await finish(requestId, {
|
|
747
|
+
responseText: extractText(result),
|
|
748
|
+
...extractUsage(result.usage),
|
|
749
|
+
costNanos: extractGatewayCostNanos(result),
|
|
750
|
+
latencyMs: Date.now() - start,
|
|
751
|
+
});
|
|
752
|
+
return result;
|
|
736
753
|
},
|
|
737
754
|
wrapStream: async ({ doStream, params }: any) => {
|
|
738
755
|
const requestId = await begin(params);
|
|
@@ -747,21 +764,24 @@ export class AIBudget {
|
|
|
747
764
|
// chunk, or a cancel — is safe: the first wins, the rest no-op.
|
|
748
765
|
// Without this an errored or abandoned stream would never settle and
|
|
749
766
|
// its real usage would be lost (recorded as free by the reconciler).
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
767
|
+
// Settle at most once, memoizing the PROMISE so the finish chunk, an
|
|
768
|
+
// error chunk, and flush all await the same settlement instead of
|
|
769
|
+
// racing, dropping it (the old `void settle()`), or flipping a
|
|
770
|
+
// "settled" flag before the mutation actually committed. A stream
|
|
771
|
+
// that is cancelled/never fully consumed won't deliver finish or
|
|
772
|
+
// flush; the reconciler's reservation expiry is the backstop there.
|
|
773
|
+
let settlement: Promise<{ costNanos: number }> | undefined;
|
|
774
|
+
const settle = (error?: string) =>
|
|
775
|
+
(settlement ??= finish(requestId, {
|
|
755
776
|
responseText: text,
|
|
756
777
|
error,
|
|
757
778
|
...extractUsage(usage),
|
|
758
779
|
costNanos: extractGatewayCostNanos({ providerMetadata }),
|
|
759
780
|
latencyMs: Date.now() - start,
|
|
760
|
-
});
|
|
761
|
-
};
|
|
781
|
+
}));
|
|
762
782
|
const tapped = result.stream.pipeThrough(
|
|
763
783
|
new TransformStream({
|
|
764
|
-
transform(chunk: any, controller) {
|
|
784
|
+
async transform(chunk: any, controller) {
|
|
765
785
|
if (chunk?.type === "text-delta") {
|
|
766
786
|
text += chunk.delta ?? chunk.textDelta ?? "";
|
|
767
787
|
}
|
|
@@ -769,8 +789,10 @@ export class AIBudget {
|
|
|
769
789
|
usage = chunk.usage;
|
|
770
790
|
providerMetadata = chunk.providerMetadata ?? providerMetadata;
|
|
771
791
|
}
|
|
772
|
-
if (chunk?.type === "error") void settle(String(chunk.error));
|
|
773
792
|
controller.enqueue(chunk);
|
|
793
|
+
// Settle after forwarding the terminal error chunk, and AWAIT
|
|
794
|
+
// it so a failed settle surfaces instead of being dropped.
|
|
795
|
+
if (chunk?.type === "error") await settle(String(chunk.error));
|
|
774
796
|
},
|
|
775
797
|
async flush() {
|
|
776
798
|
await settle();
|
|
@@ -1037,17 +1059,29 @@ export class AIBudget {
|
|
|
1037
1059
|
if (!token) return { ok: false, token: "" };
|
|
1038
1060
|
const url = new URL(request.url);
|
|
1039
1061
|
const bearer = (request.headers.get("authorization") ?? "").replace(/^Bearer\s+/i, "");
|
|
1040
|
-
// `?token=` is accepted
|
|
1041
|
-
//
|
|
1042
|
-
// JSON API
|
|
1043
|
-
|
|
1062
|
+
// `?token=` is accepted ONLY for the initial page navigation (a browser GET
|
|
1063
|
+
// can't set headers); the page strips it from the URL on load and calls the
|
|
1064
|
+
// JSON API with the bearer header. Restrict it to that GET page route so a
|
|
1065
|
+
// token can't be smuggled in a query string on API/mutation calls (where it
|
|
1066
|
+
// would also land in access logs). Compared in constant time.
|
|
1067
|
+
const sub = url.pathname.slice(prefix.length) || "/";
|
|
1068
|
+
const isPageNav = request.method === "GET" && !sub.startsWith("/api");
|
|
1069
|
+
const provided = bearer || (isPageNav ? url.searchParams.get("token") ?? "" : "");
|
|
1044
1070
|
return { ok: timingSafeEqual(provided, token), token };
|
|
1045
1071
|
};
|
|
1046
1072
|
const json = (data: unknown, status = 200) =>
|
|
1047
1073
|
new Response(JSON.stringify(data ?? null), {
|
|
1048
1074
|
status,
|
|
1049
|
-
|
|
1075
|
+
// Never let a shared cache/proxy retain budget data or the token-bearing
|
|
1076
|
+
// page — these responses are per-viewer and sensitive.
|
|
1077
|
+
headers: { "content-type": "application/json", "cache-control": "no-store" },
|
|
1050
1078
|
});
|
|
1079
|
+
// JSON.stringify does NOT escape `<`, so a value containing `</script>`
|
|
1080
|
+
// would close the inline <script> and break out. Escape `<` (and the JS line
|
|
1081
|
+
// separators) before embedding in HTML.
|
|
1082
|
+
const jsonForScript = (x: unknown) =>
|
|
1083
|
+
JSON.stringify(x).replace(/[<\u2028\u2029]/g, (ch) =>
|
|
1084
|
+
"\\u" + ch.charCodeAt(0).toString(16).padStart(4, "0"));
|
|
1051
1085
|
|
|
1052
1086
|
const handle = async (ctx: any, request: Request): Promise<Response> => {
|
|
1053
1087
|
const url = new URL(request.url);
|
|
@@ -1111,15 +1145,19 @@ export class AIBudget {
|
|
|
1111
1145
|
}
|
|
1112
1146
|
}
|
|
1113
1147
|
|
|
1114
|
-
// Inject as JSON literals (function replacers so `$` in the
|
|
1115
|
-
// treated as a replacement pattern
|
|
1116
|
-
//
|
|
1148
|
+
// Inject as script-safe JSON literals (function replacers so `$` in the
|
|
1149
|
+
// value isn't treated as a replacement pattern; `jsonForScript` escapes
|
|
1150
|
+
// `<` so a token containing `</script>` can't break out of the inline JS).
|
|
1117
1151
|
const html = DASHBOARD_HTML.replace(
|
|
1118
1152
|
/__API_BASE__/g,
|
|
1119
|
-
() =>
|
|
1120
|
-
).replace(/__TOKEN__/g, () =>
|
|
1153
|
+
() => jsonForScript(`${prefix}/api`)
|
|
1154
|
+
).replace(/__TOKEN__/g, () => jsonForScript(token));
|
|
1155
|
+
// The page embeds the bearer token — never let a shared cache retain it.
|
|
1121
1156
|
return new Response(html, {
|
|
1122
|
-
headers: {
|
|
1157
|
+
headers: {
|
|
1158
|
+
"content-type": "text/html; charset=utf-8",
|
|
1159
|
+
"cache-control": "no-store",
|
|
1160
|
+
},
|
|
1123
1161
|
});
|
|
1124
1162
|
};
|
|
1125
1163
|
|
|
@@ -332,8 +332,8 @@ describe("per-bucket rate limits", () => {
|
|
|
332
332
|
const t = initTest();
|
|
333
333
|
await setUserLimits(t, "u", { requestsPerMinute: 0 });
|
|
334
334
|
expect((await start(t, { userId: "u" })).code).toBe("rate_limit");
|
|
335
|
-
for (const requestsPerMinute of [-1, 0.5, Number.MAX_SAFE_INTEGER + 1]) {
|
|
336
|
-
await expect(setUserLimits(t, "u", { requestsPerMinute })).rejects.toThrow(
|
|
335
|
+
for (const requestsPerMinute of [-1, 0.5, Number.MAX_SAFE_INTEGER + 1, NaN, Infinity]) {
|
|
336
|
+
await expect(setUserLimits(t, "u", { requestsPerMinute })).rejects.toThrow(/requestsPerMinute/);
|
|
337
337
|
}
|
|
338
338
|
});
|
|
339
339
|
|
|
@@ -533,15 +533,17 @@ describe("model policy", () => {
|
|
|
533
533
|
});
|
|
534
534
|
|
|
535
535
|
describe("D-02 pricing validation", () => {
|
|
536
|
-
test("setPrice rejects negative rates", async () => {
|
|
537
|
-
const t = initTest();
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
536
|
+
test("setPrice rejects negative and non-finite rates", async () => {
|
|
537
|
+
const t = initTest();
|
|
538
|
+
for (const inputNanosPerMTok of [-1, NaN, Infinity, 0.5]) {
|
|
539
|
+
await expect(
|
|
540
|
+
t.mutation(api.lib.setPrice, {
|
|
541
|
+
model: "x/y",
|
|
542
|
+
inputNanosPerMTok,
|
|
543
|
+
outputNanosPerMTok: 5,
|
|
544
|
+
})
|
|
545
|
+
).rejects.toThrow(/inputNanosPerMTok/);
|
|
546
|
+
}
|
|
545
547
|
});
|
|
546
548
|
});
|
|
547
549
|
|
|
@@ -707,3 +709,85 @@ test("delayed folding attributes spend to completion day and leaves newer holds
|
|
|
707
709
|
expect(history[0].spendNanos).toBe(50);
|
|
708
710
|
} finally { vi.useRealTimers(); }
|
|
709
711
|
});
|
|
712
|
+
|
|
713
|
+
describe("v1 hardening", () => {
|
|
714
|
+
test("setGlobalLimits: a one-field update preserves the other global limits", async () => {
|
|
715
|
+
const t = initTest();
|
|
716
|
+
await t.mutation(api.lib.setGlobalLimits, {
|
|
717
|
+
dailySpendLimitNanos: 100,
|
|
718
|
+
lifetimeSpendLimitNanos: 500,
|
|
719
|
+
enforcement: "soft",
|
|
720
|
+
});
|
|
721
|
+
// Update ONLY the daily cap — must not wipe lifetime/enforcement.
|
|
722
|
+
await t.mutation(api.lib.setGlobalLimits, { dailySpendLimitNanos: 200 });
|
|
723
|
+
const g = await t.query(api.lib.getGlobalStatus, {});
|
|
724
|
+
expect(g.dailySpendLimitNanos).toBe(200);
|
|
725
|
+
expect(g.lifetimeSpendLimitNanos).toBe(500);
|
|
726
|
+
expect(g.enforcement).toBe("soft");
|
|
727
|
+
// Explicit null clears just that field.
|
|
728
|
+
await t.mutation(api.lib.setGlobalLimits, { lifetimeSpendLimitNanos: null });
|
|
729
|
+
const after = await t.query(api.lib.getGlobalStatus, {});
|
|
730
|
+
expect(after.lifetimeSpendLimitNanos).toBe(null);
|
|
731
|
+
expect(after.dailySpendLimitNanos).toBe(200);
|
|
732
|
+
});
|
|
733
|
+
|
|
734
|
+
test("finishRequest on a missing/deleted request is a graceful no-op", async () => {
|
|
735
|
+
const t = initTest();
|
|
736
|
+
const r = await start(t, { userId: "u" });
|
|
737
|
+
await t.mutation(api.lib.deleteBucket, { dimension: "user", value: "u" });
|
|
738
|
+
// The request row is gone; a late/duplicate webhook must not throw.
|
|
739
|
+
const out = await t.mutation(api.lib.finishRequest, {
|
|
740
|
+
requestId: r.requestId,
|
|
741
|
+
costNanos: 1_000_000,
|
|
742
|
+
});
|
|
743
|
+
expect(out.costNanos).toBe(0);
|
|
744
|
+
});
|
|
745
|
+
|
|
746
|
+
test("deleting a user releases holds it placed on a shared bucket", async () => {
|
|
747
|
+
const t = initTest();
|
|
748
|
+
// A shared action bucket with a cap, so requests reserve against it.
|
|
749
|
+
await t.mutation(api.lib.setBucketLimits, {
|
|
750
|
+
dimension: "action",
|
|
751
|
+
value: "shared",
|
|
752
|
+
lifetimeSpendLimitNanos: 1_000_000_000,
|
|
753
|
+
});
|
|
754
|
+
// A pending (unsettled) request from user "u" attributed to that action.
|
|
755
|
+
await start(t, { userId: "u", actionName: "shared" });
|
|
756
|
+
let action = await bucketOf(t, "action", "shared");
|
|
757
|
+
expect(action.reservedTotalNanos).toBeGreaterThan(0);
|
|
758
|
+
expect(action.pendingCount).toBe(1);
|
|
759
|
+
// Deleting the user must free the shared bucket's hold, not strand it.
|
|
760
|
+
await t.mutation(api.lib.deleteBucket, { dimension: "user", value: "u" });
|
|
761
|
+
action = await bucketOf(t, "action", "shared");
|
|
762
|
+
expect(action.reservedTotalNanos ?? 0).toBe(0);
|
|
763
|
+
expect(action.pendingCount ?? 0).toBe(0);
|
|
764
|
+
});
|
|
765
|
+
|
|
766
|
+
test("a NaN/Infinity cost or token count cannot poison bucket totals", async () => {
|
|
767
|
+
const t = initTest();
|
|
768
|
+
const r = await start(t, { userId: "u" });
|
|
769
|
+
await t.mutation(api.lib.finishRequest, {
|
|
770
|
+
requestId: r.requestId,
|
|
771
|
+
costNanos: NaN, // ignored (not finite) -> token pricing
|
|
772
|
+
promptTokens: NaN, // coerced to 0
|
|
773
|
+
completionTokens: 5,
|
|
774
|
+
});
|
|
775
|
+
vi.useFakeTimers();
|
|
776
|
+
await t.finishAllScheduledFunctions(vi.runAllTimers);
|
|
777
|
+
vi.useRealTimers();
|
|
778
|
+
const u = await userOf(t, "u");
|
|
779
|
+
expect(Number.isFinite(u.totalSpendNanos)).toBe(true);
|
|
780
|
+
expect(Number.isFinite(u.spendTodayNanos)).toBe(true);
|
|
781
|
+
expect(u.totalSpendNanos).toBeGreaterThanOrEqual(0);
|
|
782
|
+
});
|
|
783
|
+
|
|
784
|
+
test("a NaN limit is rejected rather than admitting unlimited spend", async () => {
|
|
785
|
+
const t = initTest();
|
|
786
|
+
await expect(
|
|
787
|
+
setUserLimits(t, "u", { dailySpendLimitNanos: NaN })
|
|
788
|
+
).rejects.toThrow(/dailySpendLimitNanos/);
|
|
789
|
+
await expect(
|
|
790
|
+
setUserLimits(t, "u", { dailySpendLimitNanos: Infinity })
|
|
791
|
+
).rejects.toThrow(/dailySpendLimitNanos/);
|
|
792
|
+
});
|
|
793
|
+
});
|
package/src/component/lib.ts
CHANGED
|
@@ -20,6 +20,33 @@ import { ShardedCounter } from "@convex-dev/sharded-counter";
|
|
|
20
20
|
const NANOS_PER_DOLLAR = 1e9;
|
|
21
21
|
const fmtUsd = (nanos: number) => `$${(nanos / NANOS_PER_DOLLAR).toFixed(4)}`;
|
|
22
22
|
|
|
23
|
+
// Convex's `v.number()` accepts NaN and ±Infinity. Those are poison here: a NaN
|
|
24
|
+
// cap or count silently defeats every `used > cap` comparison (NaN > x is
|
|
25
|
+
// false), so an unvalidated NaN would make admission fail OPEN and admit
|
|
26
|
+
// unlimited spend; +Infinity in totals is just as corrupting. Validate every
|
|
27
|
+
// externally-supplied accounting amount at the mutation boundary.
|
|
28
|
+
function assertAmount(
|
|
29
|
+
n: number | undefined,
|
|
30
|
+
name: string,
|
|
31
|
+
{ signed = false }: { signed?: boolean } = {}
|
|
32
|
+
) {
|
|
33
|
+
if (n === undefined) return;
|
|
34
|
+
if (!Number.isFinite(n) || !Number.isSafeInteger(n)) {
|
|
35
|
+
throw new Error(`${name} must be a finite safe integer (got ${n})`);
|
|
36
|
+
}
|
|
37
|
+
if (!signed && n < 0) throw new Error(`${name} must be nonnegative (got ${n})`);
|
|
38
|
+
}
|
|
39
|
+
function assertFraction(n: number | undefined, name: string) {
|
|
40
|
+
if (n === undefined) return;
|
|
41
|
+
if (!Number.isFinite(n) || n < 0 || n > 1) {
|
|
42
|
+
throw new Error(`${name} must be a number in [0, 1] (got ${n})`);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
// Coerce a caller/provider-supplied count to a finite nonnegative integer,
|
|
46
|
+
// mapping NaN/Infinity/garbage to 0 rather than poisoning downstream totals.
|
|
47
|
+
const safeCount = (n: number | undefined) =>
|
|
48
|
+
Number.isFinite(n) ? Math.max(0, Math.floor(n as number)) : 0;
|
|
49
|
+
|
|
23
50
|
// Built-in attribution dimensions. `user` and `action` are always populated
|
|
24
51
|
// from a request's userId/actionName; apps can add any other dimensions
|
|
25
52
|
// (team, project, customer, env, …) as tags. These two names are reserved —
|
|
@@ -683,8 +710,14 @@ export const startRequest = mutation({
|
|
|
683
710
|
}
|
|
684
711
|
|
|
685
712
|
// Consume only after every admission check succeeds, in the same
|
|
686
|
-
// transaction as the reservations and request insert.
|
|
687
|
-
//
|
|
713
|
+
// transaction as the reservations and request insert. `throws: true` is
|
|
714
|
+
// deliberate, not a rough edge: we already `.check`ed every bucket above in
|
|
715
|
+
// this same serializable transaction, so a `.limit` here cannot fail on a
|
|
716
|
+
// bucket that passed check — and if it somehow did (or a later bucket did),
|
|
717
|
+
// throwing rolls back the WHOLE transaction, including the rate we already
|
|
718
|
+
// consumed on earlier buckets. Converting this to a graceful `{allowed:false}`
|
|
719
|
+
// return would COMMIT the partial consumption and leak rate capacity, so keep
|
|
720
|
+
// the throw.
|
|
688
721
|
for (const b of buckets) {
|
|
689
722
|
if (b.requestsPerMinute !== undefined) {
|
|
690
723
|
await requestRateLimiter.limit(ctx, "requests", {
|
|
@@ -771,7 +804,10 @@ export const finishRequest = mutation({
|
|
|
771
804
|
returns: v.object({ costNanos: v.number() }),
|
|
772
805
|
handler: async (ctx, args) => {
|
|
773
806
|
const request = await ctx.db.get(args.requestId);
|
|
774
|
-
|
|
807
|
+
// The request may be gone — retention purged it, or the owning bucket was
|
|
808
|
+
// deleted. A late/duplicate webhook must be an idempotent no-op, not a 500
|
|
809
|
+
// (the caller can't do anything useful with the error, and it triggers retries).
|
|
810
|
+
if (!request) return { costNanos: 0 };
|
|
775
811
|
|
|
776
812
|
// Expiry releases capacity, but is not evidence that the provider charged
|
|
777
813
|
// nothing. Accept one final result even after expiry; duplicates stay no-ops.
|
|
@@ -779,16 +815,18 @@ export const finishRequest = mutation({
|
|
|
779
815
|
return { costNanos: request.costNanos ?? 0 };
|
|
780
816
|
}
|
|
781
817
|
|
|
782
|
-
//
|
|
783
|
-
//
|
|
784
|
-
|
|
785
|
-
const
|
|
786
|
-
const
|
|
818
|
+
// Coerce caller/provider-supplied token counts to finite nonnegative
|
|
819
|
+
// integers: negatives would refund below a cap, and NaN/Infinity (which
|
|
820
|
+
// v.number() allows) would poison every downstream total and cap check.
|
|
821
|
+
const promptTokens = safeCount(args.promptTokens);
|
|
822
|
+
const completionTokens = safeCount(args.completionTokens);
|
|
823
|
+
const cachedTokens = Math.min(promptTokens, safeCount(args.cachedTokens));
|
|
787
824
|
// Prefer an authoritative gateway cost when supplied (it already includes
|
|
788
825
|
// tool fees); otherwise price from tokens — discounting the cached
|
|
789
|
-
// (prompt-cache-read) slice — plus any server-tool per-call fees.
|
|
826
|
+
// (prompt-cache-read) slice — plus any server-tool per-call fees. Require it
|
|
827
|
+
// finite: +Infinity passes a bare `>= 0` and would corrupt the totals.
|
|
790
828
|
let costNanos: number;
|
|
791
|
-
if (args.costNanos !== undefined && args.costNanos >= 0) {
|
|
829
|
+
if (args.costNanos !== undefined && Number.isFinite(args.costNanos) && args.costNanos >= 0) {
|
|
792
830
|
costNanos = Math.round(args.costNanos);
|
|
793
831
|
} else {
|
|
794
832
|
const settings = await getSettings(ctx);
|
|
@@ -1115,10 +1153,15 @@ export const setBucketLimits = mutation({
|
|
|
1115
1153
|
},
|
|
1116
1154
|
returns: v.null(),
|
|
1117
1155
|
handler: async (ctx, args) => {
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1156
|
+
assertAmount(args.requestsPerMinute, "requestsPerMinute");
|
|
1157
|
+
assertAmount(args.maxConcurrent, "maxConcurrent");
|
|
1158
|
+
assertAmount(args.dailySpendLimitNanos, "dailySpendLimitNanos");
|
|
1159
|
+
assertAmount(args.monthlySpendLimitNanos, "monthlySpendLimitNanos");
|
|
1160
|
+
assertAmount(args.lifetimeSpendLimitNanos, "lifetimeSpendLimitNanos");
|
|
1161
|
+
assertAmount(args.dailyTokenLimit, "dailyTokenLimit");
|
|
1162
|
+
assertAmount(args.monthlyTokenLimit, "monthlyTokenLimit");
|
|
1163
|
+
assertAmount(args.lifetimeTokenLimit, "lifetimeTokenLimit");
|
|
1164
|
+
assertFraction(args.warnAtPct, "warnAtPct");
|
|
1122
1165
|
const bucket = await getOrCreateBucket(ctx, args.dimension, args.value);
|
|
1123
1166
|
const { dimension: _d, value: _v, ...limits } = args;
|
|
1124
1167
|
await ctx.db.patch(bucket._id, limits);
|
|
@@ -1140,6 +1183,9 @@ export const bumpBucket = mutation({
|
|
|
1140
1183
|
args: { dimension: v.string(), value: v.string(), ...vBumpArgs },
|
|
1141
1184
|
returns: v.null(),
|
|
1142
1185
|
handler: async (ctx, { dimension, value, dailyNanos, monthlyNanos, lifetimeNanos }) => {
|
|
1186
|
+
assertAmount(dailyNanos, "dailyNanos");
|
|
1187
|
+
assertAmount(monthlyNanos, "monthlyNanos");
|
|
1188
|
+
assertAmount(lifetimeNanos, "lifetimeNanos");
|
|
1143
1189
|
const bucket = await getOrCreateBucket(ctx, dimension, value);
|
|
1144
1190
|
const today = dayStamp();
|
|
1145
1191
|
const month = monthStamp();
|
|
@@ -1172,6 +1218,8 @@ export const adjustBucket = mutation({
|
|
|
1172
1218
|
},
|
|
1173
1219
|
returns: v.null(),
|
|
1174
1220
|
handler: async (ctx, { dimension, value, deltaNanos, tokens, reason }) => {
|
|
1221
|
+
assertAmount(deltaNanos, "deltaNanos", { signed: true });
|
|
1222
|
+
assertAmount(tokens, "tokens", { signed: true });
|
|
1175
1223
|
const b = await getOrCreateBucket(ctx, dimension, value);
|
|
1176
1224
|
const today = dayStamp();
|
|
1177
1225
|
const month = monthStamp();
|
|
@@ -1187,6 +1235,12 @@ export const adjustBucket = mutation({
|
|
|
1187
1235
|
tokensToday: Math.max(0, (dSame ? b.tokensToday ?? 0 : 0) + dt),
|
|
1188
1236
|
spendThisMonthNanos: Math.max(0, (mSame ? b.spendThisMonthNanos ?? 0 : 0) + deltaNanos),
|
|
1189
1237
|
tokensThisMonth: Math.max(0, (mSame ? b.tokensThisMonth ?? 0 : 0) + dt),
|
|
1238
|
+
// Advancing the window here must also clear the OLD window's reserved
|
|
1239
|
+
// holds, or an in-flight request from the previous day/month would be
|
|
1240
|
+
// treated as reserving against the new window and its later release would
|
|
1241
|
+
// no longer match — stranding those reserved nanos/tokens.
|
|
1242
|
+
...(dSame ? {} : { reservedTodayNanos: 0, reservedTodayTokens: 0 }),
|
|
1243
|
+
...(mSame ? {} : { reservedMonthNanos: 0, reservedMonthTokens: 0 }),
|
|
1190
1244
|
});
|
|
1191
1245
|
await ctx.db.insert("adjustments", { dimension, value, deltaNanos, tokens: dt, reason });
|
|
1192
1246
|
await addUsage(ctx, dimension, value, "day", today, deltaNanos, dt, 0);
|
|
@@ -1253,6 +1307,15 @@ export const deleteBucket = mutation({
|
|
|
1253
1307
|
.withIndex("userId", (q) => q.eq("userId", value))
|
|
1254
1308
|
.take(DELETE_BATCH);
|
|
1255
1309
|
for (const r of rows) {
|
|
1310
|
+
// Before dropping the row, free or settle any hold it placed on OTHER
|
|
1311
|
+
// (shared) buckets — an action/customer bucket this user's request
|
|
1312
|
+
// reserved against. Otherwise deleting the only row that could release
|
|
1313
|
+
// that hold strands the shared bucket's reservation + pendingCount
|
|
1314
|
+
// forever, and a late finish would throw. A finished-but-unfolded row
|
|
1315
|
+
// is folded (the charge lands on the shared buckets); a still-pending
|
|
1316
|
+
// one just has its reservation released.
|
|
1317
|
+
if (r.settled === false) await foldOne(ctx, r);
|
|
1318
|
+
else if (r.status === "pending") await releaseReservation(ctx, r);
|
|
1256
1319
|
await deleteRequestTags(ctx, r._id);
|
|
1257
1320
|
await ctx.db.delete(r._id);
|
|
1258
1321
|
}
|
|
@@ -1335,18 +1398,31 @@ export const getGlobalStatus = query({
|
|
|
1335
1398
|
|
|
1336
1399
|
export const setGlobalLimits = mutation({
|
|
1337
1400
|
args: {
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1401
|
+
// Absent = leave unchanged; explicit null = clear that limit. (A bare
|
|
1402
|
+
// v.optional(number) that always rebuilt the full patch would let a
|
|
1403
|
+
// one-field edit — exactly what the dashboard sends — silently wipe the
|
|
1404
|
+
// other global controls by patching them to undefined.)
|
|
1405
|
+
dailySpendLimitNanos: v.optional(v.union(v.number(), v.null())),
|
|
1406
|
+
lifetimeSpendLimitNanos: v.optional(v.union(v.number(), v.null())),
|
|
1407
|
+
enforcement: v.optional(
|
|
1408
|
+
v.union(v.literal("hard"), v.literal("soft"), v.null())
|
|
1409
|
+
),
|
|
1341
1410
|
},
|
|
1342
1411
|
returns: v.null(),
|
|
1343
1412
|
handler: async (ctx, args) => {
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1413
|
+
if (typeof args.dailySpendLimitNanos === "number")
|
|
1414
|
+
assertAmount(args.dailySpendLimitNanos, "dailySpendLimitNanos");
|
|
1415
|
+
if (typeof args.lifetimeSpendLimitNanos === "number")
|
|
1416
|
+
assertAmount(args.lifetimeSpendLimitNanos, "lifetimeSpendLimitNanos");
|
|
1417
|
+
// The settings fields are "global"-prefixed; map the friendly arg names,
|
|
1418
|
+
// touching ONLY the keys the caller actually passed. null -> clear.
|
|
1419
|
+
const patch: Record<string, unknown> = {};
|
|
1420
|
+
if ("dailySpendLimitNanos" in args)
|
|
1421
|
+
patch.globalDailySpendLimitNanos = args.dailySpendLimitNanos ?? undefined;
|
|
1422
|
+
if ("lifetimeSpendLimitNanos" in args)
|
|
1423
|
+
patch.globalLifetimeSpendLimitNanos = args.lifetimeSpendLimitNanos ?? undefined;
|
|
1424
|
+
if ("enforcement" in args)
|
|
1425
|
+
patch.globalEnforcement = args.enforcement ?? undefined;
|
|
1350
1426
|
const existing = await getSettings(ctx);
|
|
1351
1427
|
if (existing) {
|
|
1352
1428
|
await ctx.db.patch(existing._id, patch);
|
|
@@ -1415,13 +1491,11 @@ export const setPrice = mutation({
|
|
|
1415
1491
|
handler: async (ctx, args) => {
|
|
1416
1492
|
// Negative prices would make costOf return a negative cost, which folds
|
|
1417
1493
|
// into totals as a spend *refund* — pushing a user back under their cap.
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
)
|
|
1423
|
-
throw new Error("Prices must be non-negative");
|
|
1424
|
-
}
|
|
1494
|
+
// NaN/Infinity (allowed by v.number()) are just as corrupting — a NaN rate
|
|
1495
|
+
// poisons every settled cost for the model — so require finite integers.
|
|
1496
|
+
assertAmount(args.inputNanosPerMTok, "inputNanosPerMTok");
|
|
1497
|
+
assertAmount(args.outputNanosPerMTok, "outputNanosPerMTok");
|
|
1498
|
+
assertAmount(args.cachedNanosPerMTok, "cachedNanosPerMTok");
|
|
1425
1499
|
const existing = await ctx.db
|
|
1426
1500
|
.query("prices")
|
|
1427
1501
|
.withIndex("model", (q) => q.eq("model", args.model))
|