@convex-dev/ai-budget 0.0.2-alpha.15 → 0.0.2-alpha.17

package/README.md

@@ -23,7 +23,7 @@ full audit log you can replay later.
 - [Setup](#setup)
 - [Quickstart](#quickstart)
 - [Concepts](#concepts) — dimensions, nanodollars, reserve→settle
-- [Generating text](#generating-text) — `chat`, `languageModel`, replay
+- [Generating text](#generating-text) — `chat`, `languageModel`, `meter`, `decisions`, replay
 - [Budgets & limits](#budgets--limits) — set caps, bumps, credits, alerts
 - [Monitoring](#monitoring) — totals, spend history, the request log
 - [Deployment-wide controls](#deployment-wide-controls) — global cap, model policy, pricing, retention
@@ -251,6 +251,35 @@ await ai.meter(ctx,
   });
 ```
 
+### `ai.decisions` — structured decisions (Jev)
+
+Budget the gateway's Decisions endpoint ([Jev](https://docs.typesafe.ai)) — typed
+`choice` / `score` / `boolean` questions evaluated against a `state` — with the
+same limits, audit log, cost tracking, and tags as `ai.chat`:
+
+```ts
+const { answers, costNanos } = await ai.decisions(ctx, {
+  state: { ticket: "Customer cannot sign in" },
+  questions: {
+    priority: {
+      type: "choice",
+      instructions: "Choose the response priority",
+      criteria: { urgent: "Respond now", normal: "Respond today" },
+    },
+    needsReview: { type: "boolean", instructions: "Does a human need to review this?" },
+  },
+  tags: [{ dimension: "team", value: "support" }],
+});
+answers.priority.choice;        // "urgent" | "normal"
+answers.needsReview.probability;
+```
+
+Sugar over `ai.meter`, so cost is the gateway's authoritative amount. Model
+defaults to `defaultEvalModel` (`"typesafe/jev-1.13"`). Requires
+`@convex-dev/ai-sdk-provider` ≥ 0.2.1 and an `ai` version with
+`experimental_evaluate` — both imported lazily, so callers who don't use
+`decisions` are unaffected.
+
 ### `ai.begin` / `ai.settle` — long async jobs (video)
 
 A video job is submit → wait minutes → poll/webhook → done, spanning multiple
@@ -361,6 +390,13 @@ reserve-then-settle admission check runs per bucket. Uncapped buckets never seri
 adding tags you don't cap is free at admission; their totals still accrue for
 reporting.
 
+> **Choosing what to cap.** A cap makes its bucket's admissions atomic by
+> reserving on that bucket's single row, so capping a dimension that *all* traffic
+> shares (one `env`/`action` every request carries) serializes those admissions on
+> one document under load. Prefer capping **naturally-sharded** dimensions — per
+> `user`, per `customer` — and use the [global cap](#global-cap) for a
+> deployment-wide ceiling. Uncapped high-traffic tags are always free.
+
 ### Tags — budgeting by any dimension
 
 ```ts
@@ -459,9 +495,12 @@ ai.global.status(ctx)   // { limits, spentTodayNanos, spentTotalNanos, … }
 ai.global.bump(ctx, { dailyNanos?, lifetimeNanos? })
 ```
 
-A killswitch across everything. Backed by a sharded counter for throughput, so
-it's enforced **approximately** (bounded concurrency overshoot under burst).
-Per-bucket admission is atomic against each request's estimate.
+A best-effort killswitch across everything. Backed by a sharded counter for
+throughput, so it's enforced **approximately** — under burst it can overshoot the
+cap by a bounded amount, and it excludes in-flight (not-yet-settled) spend. It is
+**not** a to-the-dollar ceiling: for an exact limit use a per-bucket cap (those
+reserve atomically); reach for the global cap when you want a deployment-wide
+"stop everything" switch.
 
 ### Model policy
 
@@ -523,6 +562,12 @@ ai.global.setRetention(ctx, { retentionMs })   // default 1h; 0 disables
 Full request rows (prompts + responses) are swept after the window to bound the
 audit table. **Spend history survives** — it lives in separate durable rollups.
 
+> **Prompt/response content is stored** on the request row until the window sweeps
+> it (default 1h), so it's visible in the request log and to admin reads until
+> then. If your prompts carry PII you don't want retained, shorten `retentionMs`
+> (or set it low enough that content lives only as long as you need replay). Spend
+> rollups never contain content, so trimming retention doesn't cost you history.
+
 ---
 
 ## Admin dashboard

package/dist/client/dashboard.js

@@ -97,8 +97,18 @@ async function renderBuckets() {
   const dims = ["user", "action"];
   const state = { dimension: window.__dim ?? "" };
   const rows = await get("/buckets", state.dimension ? { dimension: state.dimension } : {});
-  const grand = rows.reduce((s, b) => s + b.totalSpendNanos, 0);
-  document.getElementById("total").textContent = rows.length + " buckets · " + usd(grand) + " total";
+  // Within ONE dimension each request bills exactly one bucket, so summing is a
+  // true total. Across dimensions a request bills several buckets (user + action
+  // + tags), so summing multi-counts — use the deployment-wide spend instead.
+  let totalText;
+  if (state.dimension) {
+    const grand = rows.reduce((s, b) => s + b.totalSpendNanos, 0);
+    totalText = rows.length + " buckets · " + usd(grand) + " total (" + state.dimension + ")";
+  } else {
+    const g = await get("/global", {});
+    totalText = rows.length + " buckets · " + usd(g.spentTotalNanos ?? 0) + " spent (deployment)";
+  }
+  document.getElementById("total").textContent = totalText;
   const dimSet = [...new Set(rows.map((b) => b.dimension).concat(dims))];
   const sel = el("select", { value: state.dimension, style: "width:140px",
     onchange: (e) => { window.__dim = e.target.value; render(); } },

package/dist/client/index.d.ts

@@ -39,6 +39,8 @@ export type SoftLimitInfo = BudgetEventInfo & {
 };
 export type AIBudgetOptions = {
     defaultModel?: string;
+    /** Default model for `decisions()` (the Decisions/"Jev" endpoint). */
+    defaultEvalModel?: string;
     /**
      * A *soft* limit was exceeded (request still allowed). Lets you surface budget
      * warnings even on the languageModel/Agent path where they can't be returned.
@@ -82,6 +84,16 @@ export type ChatResult = {
     /** Approaching-cap notices (empty unless a warnAtPct threshold was crossed). */
     notices: string[];
 };
+/** The tracked result of a `decisions()` call: budgeting metadata plus the
+ * structured answers from the Decisions ("Jev") endpoint. */
+export type DecisionResult = Omit<ChatResult, "text"> & {
+    /** Structured answers keyed by your question names (shape depends on each
+     * question type: `choice`, `score`, or `boolean`). */
+    answers: Record<string, any>;
+    /** The raw gateway response, including provider-specific fields (e.g.
+     * `confidence`) under `response.body`. */
+    response?: any;
+};
 /** Limits/controls settable on any budget bucket (user, action, or tag). */
 export type BucketLimits = {
     /** Token-bucket refill per minute and burst capacity; 0 blocks all requests. */
@@ -107,6 +119,7 @@ export type BumpArgs = {
 export declare class AIBudget {
     component: AIBudgetApi;
     defaultModel: string;
+    defaultEvalModel: string;
     private onSoftLimit?;
     private onThreshold?;
     private onLimitReached?;
@@ -208,6 +221,46 @@ export declare class AIBudget {
         /** Extra attribution dimensions to bill/limit (team, customer, env, …). */
         tags?: Tag[];
     }): Promise<ChatResult>;
+    /**
+     * Budget a structured decision through the AI Gateway's Decisions ("Jev")
+     * endpoint — sugar over `meter`. Evaluates typed `questions` (choice / score /
+     * boolean) about the `state` you provide, with the same reserve→settle
+     * limits, audit log, cost tracking, and per-tag attribution as `chat`. Call
+     * from an action. `userId` defaults to the authenticated caller.
+     *
+     * Requires `@convex-dev/ai-sdk-provider` >= 0.2.1 and an `ai` version that
+     * exposes `experimental_evaluate` (AI SDK 7's evaluation interface); both are
+     * imported lazily, so consumers who never call `decisions()` are unaffected.
+     *
+     *   const { answers } = await ai.decisions(ctx, {
+     *     state: { ticket: "Customer cannot sign in" },
+     *     questions: {
+     *       priority: { type: "choice", instructions: "...", criteria: { urgent: "...", normal: "..." } },
+     *       needsReview: { type: "boolean", instructions: "..." },
+     *     },
+     *   });
+     *   answers.priority.choice; // "urgent" | "normal"
+     */
+    decisions(ctx: RunMutationCtx, args: {
+        /** The evaluation model. Defaults to `defaultEvalModel` ("typesafe/jev-1.13"). */
+        model?: string;
+        /** Context the questions are evaluated against (a string or an object). */
+        state: unknown;
+        /** Typed questions (choice / score / boolean) keyed by name. */
+        questions: Record<string, unknown>;
+        /** Whom to bill. Defaults to the authenticated user (ctx.auth). */
+        userId?: string;
+        /** Attribute spend to this action name. Defaults to the calling action. */
+        action?: string;
+        /** Extra attribution dimensions to bill/limit (team, customer, env, …). */
+        tags?: Tag[];
+        /** Reserve this exact amount (nanodollars) up front — the decision cost
+         * isn't known before the call, so a hard cap is only exact with this. */
+        estimatedCostNanos?: number;
+        rerunOf?: string;
+        /** Cancel the underlying request. */
+        abortSignal?: AbortSignal;
+    }): Promise<DecisionResult>;
     /**
      * An AI SDK LanguageModel that enforces limits and records usage/cost for
      * `userId` on every call. Drop it into `generateText`, `streamText`, or the

package/dist/client/index.js

@@ -39,8 +39,10 @@ function toTokenCount(x) {
 }
 function extractUsage(usage) {
     return {
-        promptTokens: toTokenCount(usage?.inputTokens ?? usage?.promptTokens),
-        completionTokens: toTokenCount(usage?.outputTokens ?? usage?.completionTokens),
+        // Cover AI SDK camelCase (v5/v7) AND raw OpenAI-compatible snake_case, so a
+        // `meter` caller passing a raw provider `usage` object still gets counts.
+        promptTokens: toTokenCount(usage?.inputTokens ?? usage?.promptTokens ?? usage?.prompt_tokens),
+        completionTokens: toTokenCount(usage?.outputTokens ?? usage?.completionTokens ?? usage?.completion_tokens),
         // cached prompt tokens. The Convex gateway reports these at
         // `usage.inputTokenDetails.cacheReadTokens`; the other paths cover AI SDK v5
         // (`cachedInputTokens`) and raw OpenAI-compatible shapes.
@@ -115,12 +117,14 @@ function timingSafeEqual(a, b) {
 export class AIBudget {
     component;
     defaultModel;
+    defaultEvalModel;
     onSoftLimit;
     onThreshold;
     onLimitReached;
     constructor(component, options) {
         this.component = component;
         this.defaultModel = options?.defaultModel ?? "openai/gpt-4o-mini";
+        this.defaultEvalModel = options?.defaultEvalModel ?? "typesafe/jev-1.13";
         this.onSoftLimit = options?.onSoftLimit;
         this.onThreshold = options?.onThreshold;
         this.onLimitReached = options?.onLimitReached;
@@ -238,35 +242,42 @@ export class AIBudget {
         }
         const { requestId, warnings, notices } = started;
         const start = Date.now();
+        // Run the provider call. ONLY a failure of the call itself settles as an
+        // error (no charge expected).
+        let out;
         try {
-            const out = await run();
-            const { costNanos } = await this.settle(ctx, {
-                requestId,
-                responseText: out.text,
-                usage: out.usage,
-                promptTokens: out.promptTokens,
-                completionTokens: out.completionTokens,
-                cachedTokens: out.cachedTokens,
-                serverToolUses: out.serverToolUses,
-                costNanos: out.costNanos,
-                latencyMs: Date.now() - start,
-            });
-            // Re-derive the recorded usage for the return value.
-            const usage = out.promptTokens !== undefined ||
-                out.completionTokens !== undefined ||
-                out.cachedTokens !== undefined
-                ? {
-                    promptTokens: out.promptTokens ?? 0,
-                    completionTokens: out.completionTokens ?? 0,
-                    cachedTokens: out.cachedTokens ?? 0,
-                }
-                : extractUsage(out.usage);
-            return { text: out.text ?? "", requestId, costNanos, warnings, notices, ...usage };
+            out = await run();
         }
         catch (e) {
             await this.settle(ctx, { requestId, error: String(e), latencyMs: Date.now() - start });
             throw e;
         }
+        // The call SUCCEEDED (the provider may have charged). Settle the real usage.
+        // If settlement itself fails here, do NOT fall into an error-settle that
+        // records zero — that would erase a real charge. Rethrow and leave the
+        // reservation for the reconciler; billing stays "unknown", never a false zero.
+        const { costNanos } = await this.settle(ctx, {
+            requestId,
+            responseText: out.text,
+            usage: out.usage,
+            promptTokens: out.promptTokens,
+            completionTokens: out.completionTokens,
+            cachedTokens: out.cachedTokens,
+            serverToolUses: out.serverToolUses,
+            costNanos: out.costNanos,
+            latencyMs: Date.now() - start,
+        });
+        // Re-derive the recorded usage for the return value.
+        const usage = out.promptTokens !== undefined ||
+            out.completionTokens !== undefined ||
+            out.cachedTokens !== undefined
+            ? {
+                promptTokens: out.promptTokens ?? 0,
+                completionTokens: out.completionTokens ?? 0,
+                cachedTokens: out.cachedTokens ?? 0,
+            }
+            : extractUsage(out.usage);
+        return { text: out.text ?? "", requestId, costNanos, warnings, notices, ...usage };
     }
     /**
      * One-shot chat through the AI Gateway with tracking + limits — sugar over
@@ -302,6 +313,72 @@ export class AIBudget {
             };
         });
     }
+    /**
+     * Budget a structured decision through the AI Gateway's Decisions ("Jev")
+     * endpoint — sugar over `meter`. Evaluates typed `questions` (choice / score /
+     * boolean) about the `state` you provide, with the same reserve→settle
+     * limits, audit log, cost tracking, and per-tag attribution as `chat`. Call
+     * from an action. `userId` defaults to the authenticated caller.
+     *
+     * Requires `@convex-dev/ai-sdk-provider` >= 0.2.1 and an `ai` version that
+     * exposes `experimental_evaluate` (AI SDK 7's evaluation interface); both are
+     * imported lazily, so consumers who never call `decisions()` are unaffected.
+     *
+     *   const { answers } = await ai.decisions(ctx, {
+     *     state: { ticket: "Customer cannot sign in" },
+     *     questions: {
+     *       priority: { type: "choice", instructions: "...", criteria: { urgent: "...", normal: "..." } },
+     *       needsReview: { type: "boolean", instructions: "..." },
+     *     },
+     *   });
+     *   answers.priority.choice; // "urgent" | "normal"
+     */
+    async decisions(ctx, args) {
+        const model = args.model ?? this.defaultEvalModel;
+        // `evaluate` is an experimental, version-gated export; import it lazily and
+        // untyped so consumers on an older `ai` (who never call this) aren't broken.
+        const evaluate = (await import("ai")).experimental_evaluate;
+        if (typeof evaluate !== "function") {
+            throw new Error("ai-budget: decisions() needs `experimental_evaluate` from the `ai` " +
+                "package (AI SDK 7's evaluation interface). Upgrade `ai` to a " +
+                "version that exports it.");
+        }
+        // Likewise, `evaluationModel` exists on @convex-dev/ai-sdk-provider >= 0.2.1.
+        const evaluationModel = convexGateway.evaluationModel;
+        if (typeof evaluationModel !== "function") {
+            throw new Error("ai-budget: decisions() needs `convexGateway.evaluationModel` from " +
+                "@convex-dev/ai-sdk-provider >= 0.2.1. Upgrade the provider.");
+        }
+        let decision;
+        const result = await this.meter(ctx, {
+            model,
+            // Store the structured request for audit/replay.
+            messages: [
+                {
+                    role: "user",
+                    content: JSON.stringify({ state: args.state, questions: args.questions }),
+                },
+            ],
+            userId: args.userId,
+            action: args.action,
+            tags: args.tags,
+            estimatedCostNanos: args.estimatedCostNanos,
+            rerunOf: args.rerunOf,
+        }, async () => {
+            decision = await evaluate({
+                model: evaluationModel(model),
+                state: args.state,
+                questions: args.questions,
+                ...(args.abortSignal ? { abortSignal: args.abortSignal } : {}),
+            });
+            return {
+                usage: decision?.usage,
+                costNanos: extractGatewayCostNanos(decision),
+            };
+        });
+        const { text: _text, ...tracking } = result;
+        return { ...tracking, answers: decision?.answers ?? {}, response: decision?.response };
+    }
     /**
      * An AI SDK LanguageModel that enforces limits and records usage/cost for
      * `userId` on every call. Drop it into `generateText`, `streamText`, or the
@@ -347,15 +424,10 @@ export class AIBudget {
                 wrapGenerate: async ({ doGenerate, params }) => {
                     const requestId = await begin(params);
                     const start = Date.now();
+                    // Only a failure of the generation itself settles as an error.
+                    let result;
                     try {
-                        const result = await doGenerate();
-                        await finish(requestId, {
-                            responseText: extractText(result),
-                            ...extractUsage(result.usage),
-                            costNanos: extractGatewayCostNanos(result),
-                            latencyMs: Date.now() - start,
-                        });
-                        return result;
+                        result = await doGenerate();
                     }
                     catch (e) {
                         await finish(requestId, {
@@ -364,6 +436,15 @@ export class AIBudget {
                         });
                         throw e;
                     }
+                    // Generation succeeded (provider may have charged). Settle the real
+                    // usage; a failure here rethrows rather than recording a false zero.
+                    await finish(requestId, {
+                        responseText: extractText(result),
+                        ...extractUsage(result.usage),
+                        costNanos: extractGatewayCostNanos(result),
+                        latencyMs: Date.now() - start,
+                    });
+                    return result;
                 },
                 wrapStream: async ({ doStream, params }) => {
                     const requestId = await begin(params);
@@ -378,21 +459,22 @@ export class AIBudget {
                         // chunk, or a cancel — is safe: the first wins, the rest no-op.
                         // Without this an errored or abandoned stream would never settle and
                         // its real usage would be lost (recorded as free by the reconciler).
-                        let settled = false;
-                        const settle = (error) => {
-                            if (settled)
-                                return;
-                            settled = true;
-                            return finish(requestId, {
-                                responseText: text,
-                                error,
-                                ...extractUsage(usage),
-                                costNanos: extractGatewayCostNanos({ providerMetadata }),
-                                latencyMs: Date.now() - start,
-                            });
-                        };
+                        // Settle at most once, memoizing the PROMISE so the finish chunk, an
+                        // error chunk, and flush all await the same settlement instead of
+                        // racing, dropping it (the old `void settle()`), or flipping a
+                        // "settled" flag before the mutation actually committed. A stream
+                        // that is cancelled/never fully consumed won't deliver finish or
+                        // flush; the reconciler's reservation expiry is the backstop there.
+                        let settlement;
+                        const settle = (error) => (settlement ??= finish(requestId, {
+                            responseText: text,
+                            error,
+                            ...extractUsage(usage),
+                            costNanos: extractGatewayCostNanos({ providerMetadata }),
+                            latencyMs: Date.now() - start,
+                        }));
                         const tapped = result.stream.pipeThrough(new TransformStream({
-                            transform(chunk, controller) {
+                            async transform(chunk, controller) {
                                 if (chunk?.type === "text-delta") {
                                     text += chunk.delta ?? chunk.textDelta ?? "";
                                 }
@@ -400,9 +482,11 @@ export class AIBudget {
                                     usage = chunk.usage;
                                     providerMetadata = chunk.providerMetadata ?? providerMetadata;
                                 }
-                                if (chunk?.type === "error")
-                                    void settle(String(chunk.error));
                                 controller.enqueue(chunk);
+                                // Settle after forwarding the terminal error chunk, and AWAIT
+                                // it so a failed settle surfaces instead of being dropped.
+                                if (chunk?.type === "error")
+                                    await settle(String(chunk.error));
                             },
                             async flush() {
                                 await settle();
@@ -590,16 +674,26 @@ export class AIBudget {
                 return { ok: false, token: "" };
             const url = new URL(request.url);
             const bearer = (request.headers.get("authorization") ?? "").replace(/^Bearer\s+/i, "");
-            // `?token=` is accepted only for the initial page navigation (a browser
-            // GET can't set headers); the page strips it from the URL on load and the
-            // JSON API is called with the bearer header. Compared in constant time.
-            const provided = bearer || url.searchParams.get("token") || "";
+            // `?token=` is accepted ONLY for the initial page navigation (a browser GET
+            // can't set headers); the page strips it from the URL on load and calls the
+            // JSON API with the bearer header. Restrict it to that GET page route so a
+            // token can't be smuggled in a query string on API/mutation calls (where it
+            // would also land in access logs). Compared in constant time.
+            const sub = url.pathname.slice(prefix.length) || "/";
+            const isPageNav = request.method === "GET" && !sub.startsWith("/api");
+            const provided = bearer || (isPageNav ? url.searchParams.get("token") ?? "" : "");
             return { ok: timingSafeEqual(provided, token), token };
         };
         const json = (data, status = 200) => new Response(JSON.stringify(data ?? null), {
             status,
-            headers: { "content-type": "application/json" },
+            // Never let a shared cache/proxy retain budget data or the token-bearing
+            // page — these responses are per-viewer and sensitive.
+            headers: { "content-type": "application/json", "cache-control": "no-store" },
         });
+        // JSON.stringify does NOT escape `<`, so a value containing `</script>`
+        // would close the inline <script> and break out. Escape `<` (and the JS line
+        // separators) before embedding in HTML.
+        const jsonForScript = (x) => JSON.stringify(x).replace(/[<\u2028\u2029]/g, (ch) => "\\u" + ch.charCodeAt(0).toString(16).padStart(4, "0"));
         const handle = async (ctx, request) => {
             const url = new URL(request.url);
             const sub = url.pathname.slice(prefix.length) || "/";
@@ -656,12 +750,16 @@ export class AIBudget {
                         return json({ error: "not found" }, 404);
                 }
             }
-            // Inject as JSON literals (function replacers so `$` in the value isn't
-            // treated as a replacement pattern). This keeps a token/prefix containing
-            // quotes, backslashes, or `</script>` from breaking out of the JS string.
-            const html = DASHBOARD_HTML.replace(/__API_BASE__/g, () => JSON.stringify(`${prefix}/api`)).replace(/__TOKEN__/g, () => JSON.stringify(token));
+            // Inject as script-safe JSON literals (function replacers so `$` in the
+            // value isn't treated as a replacement pattern; `jsonForScript` escapes
+            // `<` so a token containing `</script>` can't break out of the inline JS).
+            const html = DASHBOARD_HTML.replace(/__API_BASE__/g, () => jsonForScript(`${prefix}/api`)).replace(/__TOKEN__/g, () => jsonForScript(token));
+            // The page embeds the bearer token — never let a shared cache retain it.
             return new Response(html, {
-                headers: { "content-type": "text/html; charset=utf-8" },
+                headers: {
+                    "content-type": "text/html; charset=utf-8",
+                    "cache-control": "no-store",
+                },
             });
         };
         const handler = httpActionGeneric(handle);

package/dist/component/lib.d.ts

@@ -377,9 +377,9 @@ export declare const getGlobalStatus: import("convex/server").RegisteredQuery<"p
     defaultWarnAtPct: number | null;
 }>>;
 export declare const setGlobalLimits: import("convex/server").RegisteredMutation<"public", {
-    dailySpendLimitNanos?: number | undefined;
-    lifetimeSpendLimitNanos?: number | undefined;
-    enforcement?: "hard" | "soft" | undefined;
+    dailySpendLimitNanos?: number | null | undefined;
+    lifetimeSpendLimitNanos?: number | null | undefined;
+    enforcement?: "hard" | "soft" | null | undefined;
 }, Promise<null>>;
 export declare const bumpGlobal: import("convex/server").RegisteredMutation<"public", {
     dailyNanos?: number | undefined;