@ultimat3/ai 27.9.0 → 27.10.0

package/src/gateway.ts

@@ -16,7 +16,8 @@ import {
   stringField,
 } from '@ultimat3/core';
 import type { Money } from '@ultimat3/money';
-import type { BudgetKeys, BudgetLimits, BudgetStore } from './budget';
+import { billedOf } from './billed';
+import type { BudgetKeys, BudgetLimits, BudgetReservation, BudgetStore } from './budget';
 import {
   assertFiniteLimits,
   BudgetLedger,
@@ -28,6 +29,7 @@ import { AiProviderUnavailableError } from './errors';
 import { resolveModel } from './model-resolve';
 import type { ModelId } from './models';
 import type { GenerateRequest, GenerateResult, Provider, StreamChunk } from './provider';
+import type { SamplingWire } from './sampling';
 
 /**
  * Response cache. Structurally satisfied by `@ultimat3/cache`'s memo/LRU tiers; declared as
@@ -82,6 +84,8 @@ export interface ProviderGatewayInput {
 }
 
 export interface Gateway {
+  /** The wire formats of the providers serving `model` — what `llm.sampling.ignored` is read from. */
+  samplingWires?(model: ModelId): readonly SamplingWire[];
   /**
    * The model a call with none declared runs on — `providerGateway({ defaultModel })`. Read by
    * `llm()` and `agent()` after the declaration and its prompt, so the app's choice is the one
@@ -186,11 +190,23 @@ class GatewayImpl implements Gateway {
     try {
       result = await this.attempt(model, (provider) => provider.generate(resolved));
     } catch (error) {
-      // A call that never landed must not go on holding its reservation.
-      await ledger?.release(reservation);
+      // A call that never landed must not go on holding its reservation — and one that DID land
+      // and still failed (`billedOf`: the provider answered with something unusable) is recorded
+      // as what it cost. Releasing it told the ledger a billed call was free.
+      await settleFailure(ledger, reservation, error);
       throw error;
     }
-    await ledger?.record(result.usage, result.cost, reservation);
+    // The provider was PAID by now. Nothing below may turn that answer into a failure: a ledger
+    // that cannot book it gives the reservation back and says so, and the answer is returned.
+    try {
+      await ledger?.record(result.usage, result.costExact ?? result.cost, reservation);
+    } catch (error) {
+      await ledger?.release(reservation).catch(() => undefined);
+      logger.warn('ai.budget.record_failed', {
+        provider: result.provider ?? 'unknown',
+        error: renderThrowable(error),
+      });
+    }
     // A refusal is not an answer, so it is not cached. Storing one would keep serving a decision
     // the classifier might not make twice, long after the prompt that provoked it was fixed.
     if (result.stopReason !== 'refusal') await this.remember(cacheKey, result);
@@ -240,17 +256,33 @@ class GatewayImpl implements Gateway {
         }
         // Settled only once `record` has LANDED. Marking it first left a store that threw here
         // holding the reservation and half the record — the `finally` saw a settled stream.
-        await ledger?.record(chunk.result.usage, chunk.result.cost, reservation);
+        const { usage, cost, costExact } = chunk.result;
+        await ledger?.record(usage, costExact ?? cost, reservation);
         settled = true;
         yield { type: 'done', result: { ...chunk.result, provider: provider.name } };
       }
+    } catch (error) {
+      // Thrown after the provider reported usage: the bill is recorded, exactly as `generate` does.
+      // Only when nothing settled yet — a provider that throws AFTER its `done` chunk was already
+      // recorded, and releasing again would credit the ledger a second time.
+      if (!settled) {
+        settled = true;
+        await settleFailure(ledger, reservation, error);
+      }
+      throw error;
     } finally {
-      // A stream that threw, or that its consumer abandoned, never reached `done` — so nothing
-      // reconciled the reservation and it would hold the ceiling for the rest of the window.
+      // A stream its consumer abandoned never reached `done` — so nothing reconciled the
+      // reservation and it would hold the ceiling for the rest of the window.
       if (!settled) await ledger?.release(reservation);
     }
   }
 
+  samplingWires(model: ModelId): readonly SamplingWire[] {
+    return this.config.providers
+      .filter((p) => p.models.includes(model))
+      .flatMap((p) => (p.wire === undefined ? [] : [p.wire]));
+  }
+
   /** Whether any provider serving `model` would send a hinted request with a cache mark. */
   private marks(model: ModelId): boolean {
     return this.config.providers.some(
@@ -326,6 +358,17 @@ class GatewayImpl implements Gateway {
   }
 }
 
+/** A failed call's reservation: reconciled against its bill when the failure carries one, else given back. */
+async function settleFailure(
+  ledger: BudgetLedger | undefined,
+  reservation: BudgetReservation | undefined,
+  error: unknown,
+): Promise<void> {
+  const billed = billedOf(error);
+  if (billed === undefined) return ledger?.release(reservation);
+  await ledger?.record(billed.usage, billed.cost, reservation);
+}
+
 /**
  * The request's completion ceiling, screened at the one seam every model call in an app passes
  * through — `llm()`, `agent()`, an eval, a judge and a hand-built `generate()` alike.
@@ -423,5 +466,8 @@ export function promptCacheKey(request: GenerateRequest): string {
     thinking: request.thinking ?? 'adaptive',
     tools: request.tools ?? [],
     stop: request.stopSequences ?? [],
+    // Only when asked for: an absent knob must leave every key written before it existed as it was.
+    ...(request.temperature === undefined ? {} : { temperature: request.temperature }),
+    ...(request.seed === undefined ? {} : { seed: request.seed }),
   });
 }

package/src/in-process-tool.ts

@@ -0,0 +1,77 @@
+// Single responsibility: `inProcessTool()` — a tool that exists only for an agent in this
+// process. A name, a description, an input schema and an `execute`; nothing an MCP surface, an
+// HTTP route, `openapi.json` or the manifest ever sees.
+//
+// Not a ninth primitive and not an action: it is a function the agent's own action may call, as
+// code in that action's `handle` would be. The AGENT is the operation with a policy; this runs as
+// `ctx.actor` inside it, and anything it reaches that needs its own authorization is an `action()`
+// it calls — whose policy then decides, as always.
+//
+// It exists because the other way to give an agent a tool was to declare an action
+// `mcp: { expose: true }` — the same flag that serves it to every external MCP client. A tool that
+// must never be an MCP tool had to claim it was one.
+
+import { InputInvalidError } from '@ultimat3/action';
+import type { Actor } from '@ultimat3/core';
+import type { InferOutput, StandardSchemaV1 } from '@ultimat3/schema';
+import { formatIssues, toWireSchema, validateAsync } from '@ultimat3/schema';
+import { AiRequestInvalidError } from './errors';
+import { RESPOND } from './respond';
+import type { ProjectableAction } from './tools';
+
+export interface InProcessToolDef<TInput extends StandardSchemaV1> {
+  /** What the model calls it. Letters, digits, `_` and `-`, at most 64 — what every provider takes. */
+  readonly name: string;
+  /** What the model reads to decide when to call it. Required: a tool with none is never chosen well. */
+  readonly description: string;
+  /** Parses what the model sent before `execute` sees it; projected as the tool's input schema. */
+  readonly input: TInput;
+  /**
+   * Runs as the request's own actor — never one the model named. Throw an `UltimateError` to refuse:
+   * its code, cause and fix are what the model reads (`discloseErrors`).
+   */
+  execute(args: { readonly input: InferOutput<TInput>; readonly actor: Actor }): unknown;
+  /** `false` withholds this tool's refusals from the model as a projected tool's are. Default `true`. */
+  readonly discloseErrors?: boolean;
+}
+
+const NAME = /^[A-Za-z0-9_-]{1,64}$/;
+
+/** A tool for `agent({ tools })` that is this process's alone. Declared once, at module scope. */
+export function inProcessTool<TInput extends StandardSchemaV1>(
+  def: InProcessToolDef<TInput>,
+): ProjectableAction {
+  if (!NAME.test(def.name) || def.name === RESPOND) {
+    throw new AiRequestInvalidError({
+      detail: `inProcessTool() was given the name ${JSON.stringify(def.name)}; a tool name is 1 to 64 letters, digits, "_" or "-", and "${RESPOND}" is the answer tool's`,
+      fix: "inProcessTool({ name: 'kbRead', … })   # letters, digits, _ and - only",
+    });
+  }
+  if (def.description.trim() === '') {
+    throw new AiRequestInvalidError({
+      detail: `inProcessTool("${def.name}") has an empty description, and the description is all the model has to decide when to call it`,
+      fix: `inProcessTool({ name: '${def.name}', description: 'Read one document by id.', … })`,
+    });
+  }
+  return {
+    name: def.name,
+    description: def.description,
+    // A schema the wire cannot express throws HERE, at declaration — as `respondFor`'s does.
+    inputJsonSchema: toWireSchema(def.input),
+    inProcess: true,
+    ...(def.discloseErrors === undefined ? {} : { discloseErrors: def.discloseErrors }),
+    async run({ input, actor }) {
+      const parsed = await validateAsync(def.input, input);
+      if (parsed.issues !== undefined) {
+        // The line only: the structured issues are `@ultimat3/action`'s own shape, for a form.
+        throw new InputInvalidError(
+          def.name,
+          formatIssues(parsed.issues).join('; '),
+          undefined,
+          'tool',
+        );
+      }
+      return def.execute({ input: parsed.value, actor });
+    },
+  };
+}

package/src/index.ts

@@ -5,14 +5,37 @@
 export type { Infer } from '@ultimat3/schema';
 export { t } from '@ultimat3/schema';
 export { agent } from './agent';
+export type {
+  AgentAction,
+  AgentContinuation,
+  AgentRun,
+  AgentTranscript,
+} from './agent-continue';
+export {
+  MAX_TRANSCRIPT_CHARS,
+  TRANSCRIPT_META,
+  TRANSCRIPT_VERSION,
+  transcriptOf,
+} from './agent-continue';
 export type { AgentBudget, AgentDef, AgentTurn, AgentVarsArgs } from './agent-def';
-export { AgentDeadlineError, AgentToolLoopError } from './agent-errors';
+export {
+  AgentDeadlineError,
+  AgentToolLoopError,
+  AgentTranscriptInvalidError,
+} from './agent-errors';
 export type { AgentBudgetFact, AgentFact } from './agent-facts';
 export { describeAgents, resetAgents } from './agent-facts';
 export type { AgentJobOptions } from './agent-job';
 export { agentJob } from './agent-job';
 export type { AgentSystem } from './agent-system';
 export type { AgentRunUsage } from './agent-usage';
+export {
+  AiProviderUnexpectedError,
+  LlmToolArgumentsError,
+  LlmTruncatedError,
+} from './answer-errors';
+export type { Billed } from './billed';
+export { BILLED_META, billedOf } from './billed';
 export type {
   BudgetKeys,
   BudgetLedger,
@@ -48,6 +71,7 @@ export {
 } from './content-blocks';
 export type { ContentRefusal } from './content-errors';
 export { AiContentUnsupportedError } from './content-errors';
+export { COST_SCALE_DIGITS, costScale, exactCostOf, roundUpCost } from './cost';
 export type { EchoProvider, EchoProviderInput } from './echo-provider';
 export { echoProvider } from './echo-provider';
 export type { Embedder, HashEmbedder, HashEmbedderInput } from './embeddings';
@@ -79,7 +103,6 @@ export {
   LlmOutputInvalidError,
   LlmRefusedError,
   LlmStreamInvalidError,
-  LlmTruncatedError,
   VectorDimMismatchError,
   VectorScopeWidenedError,
 } from './errors';
@@ -129,7 +152,9 @@ export type { HiveDef, HiveSplitArgs } from './hive';
 export { hive } from './hive';
 export { HiveEmptyError } from './hive-errors';
 export type { HiveMember, HiveMemberError, HiveOutput, HiveResult } from './hive-result';
-export type { LlmAction, LlmBudget, LlmDef, LlmVarsArgs } from './llm';
+export type { InProcessToolDef } from './in-process-tool';
+export { inProcessTool } from './in-process-tool';
+export type { LlmAction, LlmBudget, LlmCall, LlmDef, LlmVarsArgs } from './llm';
 export { llm } from './llm';
 export type { LlmCache, LlmScopeArgs, LlmSemanticCache } from './llm-cache';
 export type { LlmStreamChunk } from './llm-stream';
@@ -248,6 +273,7 @@ export {
   configureAi,
   semanticCacheFor,
 } from './runtime';
+export type { Sampling } from './sampling';
 export type { Scorer } from './scorers';
 export {
   contains,

package/src/llm-def.ts

@@ -0,0 +1,97 @@
+// Single responsibility: what an `llm()` DECLARATION is — its options, and the one event its
+// observer is handed. Types only; ./llm owns the call that reads them. `agent-def.ts`'s twin.
+
+import type { ActionMcp, ActionPolicy } from '@ultimat3/action';
+import type { Ctx } from '@ultimat3/core';
+import type { Money } from '@ultimat3/money';
+import type { InferOutput, StandardSchemaV1 } from '@ultimat3/schema';
+import type { AgentRunUsage } from './agent-usage';
+import type { LlmCache } from './llm-cache';
+import type { ModelId } from './models';
+import type { Prompt, PromptVars } from './prompt';
+import type { PromptCacheOption } from './prompt-cache';
+import type { StopReason, TokenUsage } from './provider';
+
+/** Per-call ceilings, checked before the provider is reached. Never truncates — refuses. */
+export interface LlmBudget {
+  /** Prompt tokens. */
+  readonly tokensIn?: number;
+  /** Worst-case price of one call, integer minor units. */
+  readonly costPerCall?: Money;
+}
+
+export interface LlmVarsArgs<TInput extends StandardSchemaV1> {
+  readonly input: InferOutput<TInput>;
+  readonly ctx: Ctx;
+}
+
+export interface LlmDef<
+  TInput extends StandardSchemaV1,
+  TOutput extends StandardSchemaV1,
+  V extends PromptVars,
+> {
+  /**
+   * Overrides the prompt artifact's own model. The prompt's is part of its content hash and
+   * travels with it; this one is the deployment decision, so it wins.
+   */
+  readonly model?: ModelId;
+  readonly input: TInput;
+  readonly output: TOutput;
+  readonly prompt: Prompt<V>;
+  /**
+   * Everything the prompt is allowed to see, derived from the parsed input. Required, and
+   * async, because the input is usually an id and the prompt needs the row behind it — this
+   * is the one declared place a model call loads data, so a reader can see what was sent.
+   */
+  vars(args: LlmVarsArgs<TInput>): V | Promise<V>;
+  readonly cache?: LlmCache<InferOutput<TInput>>;
+  /**
+   * PROVIDER prompt caching (`./prompt-cache`) — not `cache`, which is this package's own semantic
+   * cache. Default `'off'`: a one-shot call marked for caching pays the write premium (1.25x on
+   * Anthropic) for a prefix nothing may ever read back. `'auto'` is for a call made many times
+   * inside the cache's lifetime with one long `system`; the repair turn reads it back either way.
+   */
+  readonly promptCache?: PromptCacheOption<LlmVarsArgs<TInput>>;
+  readonly budget?: LlmBudget;
+  readonly policy: ActionPolicy;
+  readonly mcp?: ActionMcp;
+  /** Enforced completion ceiling. The model never sees it, so it can be cut off mid-answer. */
+  readonly maxTokens?: number;
+  /**
+   * `temperature` (0 to 2) and `seed` (a whole number), for a call that should answer the same way
+   * twice — a verifier, a classifier. Sent only to a model whose row takes them
+   * (`registerModel({ sampling })`); to any other they are NOT sent, because a reasoning model
+   * answers one with a 400 — the `llm.sampling.ignored` span attribute names what was left out.
+   * `seed` exists on the chat-completions wire only, and is best effort where it exists.
+   */
+  readonly temperature?: number;
+  readonly seed?: number;
+  /**
+   * Called once per PROVIDER call this invocation makes — the answer, and the repair turn when
+   * there is one — before the answer is read. What `agent()`'s `onTurn` is to a run: the usage and
+   * cost of each call, and on a repair call the issues the model was told to fix.
+   *
+   * Observation only. Awaited, and a throw from it FAILS the call, for the reason `onTurn` gives.
+   * Not called on a semantic-cache hit: no provider call was made, and nothing was billed.
+   */
+  onCall?(event: LlmCall): void | Promise<void>;
+}
+
+/** One completed provider call of an `llm()` invocation, as an observer sees it. Facts only. */
+export interface LlmCall {
+  /** 1 is the answer; 2 is the repair turn. A streamed call is always 1. */
+  readonly attempt: number;
+  readonly model: ModelId;
+  /** Which provider answered, as the gateway stamped it. */
+  readonly provider: string | undefined;
+  readonly stopReason: StopReason;
+  readonly usage: TokenUsage;
+  /** This call's cost alone, integer minor units, rounded up. */
+  readonly cost: Money;
+  /** This call's cost exactly (`Money` at a finer `scale`); absent when the provider stated none. */
+  readonly costExact?: Money;
+  /** On the repair turn: the schema issues of the first answer — what the model was told to fix. */
+  readonly repairOf?: readonly string[];
+  /** The invocation so far, this call included — priced once from the summed usage. */
+  readonly run: AgentRunUsage;
+}

package/src/llm.ts

@@ -21,38 +21,36 @@
  * same rules.
  */
 
-import type { Action, ActionMcp, ActionPolicy, InvokeOptions } from '@ultimat3/action';
+import type { Action, InvokeOptions } from '@ultimat3/action';
 import { action } from '@ultimat3/action';
 import type { Ctx, Span, SpanAttributes } from '@ultimat3/core';
 import { finiteCount, withSpan } from '@ultimat3/core';
-import type { Money } from '@ultimat3/money';
 import type { InferInput, InferOutput, StandardSchemaV1 } from '@ultimat3/schema';
 import { formatIssues, validateAsync } from '@ultimat3/schema';
+import { runTotal } from './agent-usage';
+import { LlmTruncatedError } from './answer-errors';
 import type { BudgetLimits } from './budget';
 import { budgetKeysFor, currentBudget, screenedMoney, withBudget } from './budget';
-import {
-  LlmOutputInvalidError,
-  LlmRefusedError,
-  LlmStreamInvalidError,
-  LlmTruncatedError,
-} from './errors';
+import { LlmOutputInvalidError, LlmRefusedError, LlmStreamInvalidError } from './errors';
 import type { Gateway } from './gateway';
-import type { LlmCache } from './llm-cache';
 import { openCache } from './llm-cache';
+import type { LlmBudget, LlmDef } from './llm-def';
 import type { LlmSink, LlmStreamChunk } from './llm-stream';
 import { currentLlmSink, llmStream, streamOneTurn, withLlmSink } from './llm-stream';
 import { modelForCall } from './model-resolve';
-import type { ModelId } from './models';
 import { moreCapableThan } from './models';
-import type { Prompt, PromptVars } from './prompt';
-import type { PromptCacheOption } from './prompt-cache';
+import type { PromptVars } from './prompt';
 import { promptCacheHint } from './prompt-cache';
 import type { AiMessage, GenerateRequest, GenerateResult } from './provider';
 import { isTruncated } from './provider';
 import { assertNoSecrets } from './redaction';
 import type { Respond } from './respond';
 import { parseJsonish, RESPOND, respondFor } from './respond';
-import { aiGateway, aiRedactor } from './runtime';
+import type { RunMeter } from './run-meter';
+import { runMeter } from './run-meter';
+import { aiGateway, aiRedactor, installedGateway } from './runtime';
+import type { Sampling } from './sampling';
+import { ignoredSampling, screenSampling } from './sampling';
 
 /** Two attempts total: the answer, then one repair turn. See `LlmOutputInvalidError`. */
 const ATTEMPTS = 2;
@@ -63,52 +61,7 @@ const ATTEMPTS = 2;
  */
 const DEFAULT_MAX_TOKENS = 4_096;
 
-/** Per-call ceilings, checked before the provider is reached. Never truncates — refuses. */
-export interface LlmBudget {
-  /** Prompt tokens. */
-  readonly tokensIn?: number;
-  /** Worst-case price of one call, integer minor units. */
-  readonly costPerCall?: Money;
-}
-
-export interface LlmVarsArgs<TInput extends StandardSchemaV1> {
-  readonly input: InferOutput<TInput>;
-  readonly ctx: Ctx;
-}
-
-export interface LlmDef<
-  TInput extends StandardSchemaV1,
-  TOutput extends StandardSchemaV1,
-  V extends PromptVars,
-> {
-  /**
-   * Overrides the prompt artifact's own model. The prompt's is part of its content hash and
-   * travels with it; this one is the deployment decision, so it wins.
-   */
-  readonly model?: ModelId;
-  readonly input: TInput;
-  readonly output: TOutput;
-  readonly prompt: Prompt<V>;
-  /**
-   * Everything the prompt is allowed to see, derived from the parsed input. Required, and
-   * async, because the input is usually an id and the prompt needs the row behind it — this
-   * is the one declared place a model call loads data, so a reader can see what was sent.
-   */
-  vars(args: LlmVarsArgs<TInput>): V | Promise<V>;
-  readonly cache?: LlmCache<InferOutput<TInput>>;
-  /**
-   * PROVIDER prompt caching (`./prompt-cache`) — not `cache`, which is this package's own semantic
-   * cache. Default `'off'`: a one-shot call marked for caching pays the write premium (1.25x on
-   * Anthropic) for a prefix nothing may ever read back. `'auto'` is for a call made many times
-   * inside the cache's lifetime with one long `system`; the repair turn reads it back either way.
-   */
-  readonly promptCache?: PromptCacheOption<LlmVarsArgs<TInput>>;
-  readonly budget?: LlmBudget;
-  readonly policy: ActionPolicy;
-  readonly mcp?: ActionMcp;
-  /** Enforced completion ceiling. The model never sees it, so it can be cut off mid-answer. */
-  readonly maxTokens?: number;
-}
+export type { LlmBudget, LlmCall, LlmDef, LlmVarsArgs } from './llm-def';
 
 /**
  * An `action` with one extra way to be called. Every projection an action has, it has — its MCP tool,
@@ -152,12 +105,13 @@ export function llm<
   // off for the life of the process. A floor of 1 because a completion ceiling of zero tokens is
   // a call that cannot answer.
   finiteCount('llm', 'maxTokens', def.maxTokens ?? DEFAULT_MAX_TOKENS, 1);
+  const sampling = screenSampling('llm', def);
   const built = action<TInput, TOutput>({
     input: def.input,
     output: def.output,
     policy: def.policy,
     ...(def.mcp === undefined ? {} : { mcp: def.mcp }),
-    handle: (args) => generate(def, respond, args),
+    handle: (args) => generate(def, respond, sampling, args),
   });
   return streamable(built);
 }
@@ -195,6 +149,7 @@ async function generate<
 >(
   def: LlmDef<TInput, TOutput, V>,
   respond: Respond,
+  sampling: Sampling,
   args: { readonly input: InferOutput<TInput>; readonly ctx: Ctx },
 ): Promise<InferOutput<TOutput>> {
   const { prompt } = def;
@@ -227,6 +182,11 @@ async function generate<
       // an audit asks evidence for, and a redactor that silently stopped matching looks identical
       // to one that had nothing to remove until this attribute separates them.
       'llm.redacted': redacted,
+      'llm.sampling.ignored': ignoredSampling(
+        model,
+        sampling,
+        installedGateway()?.samplingWires?.(model),
+      ).join(','),
     });
 
     // A cached answer is still data of unknown provenance, so it goes through the schema like
@@ -254,6 +214,7 @@ async function generate<
       maxTokens: def.maxTokens ?? DEFAULT_MAX_TOKENS,
       ...(prompt.effort === undefined ? {} : { effort: prompt.effort }),
       ...(prompt.thinking === undefined ? {} : { thinking: prompt.thinking }),
+      ...sampling,
       // The caller's own signal, forwarded to the transport exactly as `agent()` does — and
       // inherited by `streamedAnswer` from this same `base`. Without it a disconnected caller left
       // the provider call in flight, billed and unread, and the repair turn bought a SECOND one.
@@ -272,17 +233,22 @@ async function generate<
     );
     const sink = currentLlmSink();
 
-    return withBudget(ledger, async () => {
+    // Every provider call goes through the meter, so a failure AFTER one answered — a refusal, a
+    // truncation, a schema no repair satisfied — is thrown carrying what it was billed.
+    const meter = runMeter();
+    const answer = async (): Promise<InferOutput<TOutput>> => {
       if (sink !== undefined) {
-        const value = await streamedAnswer(def.output, name, gateway, base, sink, span);
+        const value = await streamedAnswer(def, name, gateway, base, sink, span, meter);
         await cache?.remember(value);
         return value;
       }
       let messages: readonly AiMessage[] = request.messages;
       let issues = 'no output';
+      let rejected: readonly string[] | undefined;
       for (let attempt = 1; attempt <= ATTEMPTS; attempt += 1) {
-        const result = await gateway.generate({ ...request, messages });
+        const result = await meter.call(gateway.generate({ ...request, messages }));
         span.setAttributes({ 'llm.attempts': attempt, ...answerAttributes(result) });
+        await observeCall(def, meter, result, attempt, rejected);
         // Branch on the stop reason BEFORE reading the answer. A refusal carries empty or partial
         // content, so parsing it first reports a schema disagreement — a cause that is wrong, a
         // fix that does not apply, and a repair turn spent buying the same refusal again.
@@ -298,25 +264,38 @@ async function generate<
             explanation: result.stopDetails?.explanation,
           });
         }
+        // Cut off, so there is no answer to read — and it is read BEFORE the schema for the reason
+        // the refusal is: a `respond` call stopped at the ceiling arrives with no arguments, both
+        // wires read that as `{}`, and `{}` SATISFIES an `output` whose keys are optional. The
+        // ceiling is the same on the next attempt, so the repair turn is a second truncation.
+        if (isTruncated(result.stopReason)) {
+          throw new LlmTruncatedError({ prompt: name, maxTokens: request.maxTokens });
+        }
         const parsed = await validateAsync(def.output, respond.read(result));
         if (parsed.issues === undefined) {
           await cache?.remember(parsed.value);
           return parsed.value;
         }
-        // A cut-off answer that also fails its schema is not a disagreement about the shape: the
-        // ceiling is the same on the next attempt, so the repair turn is a second truncation.
-        if (isTruncated(result.stopReason)) {
-          throw new LlmTruncatedError({ prompt: name, maxTokens: request.maxTokens });
-        }
-        issues = formatIssues(parsed.issues).join('; ');
+        rejected = formatIssues(parsed.issues);
+        issues = rejected.join('; ');
         const echo = assistantEcho(result);
         messages =
           echo === undefined
             ? [...messages, repair(issues)]
             : [...messages, { role: 'assistant', content: echo }, repair(issues)];
       }
-      throw new LlmOutputInvalidError({ prompt: name, attempts: ATTEMPTS, issues });
-    });
+      throw new LlmOutputInvalidError({
+        prompt: name,
+        attempts: ATTEMPTS,
+        issues,
+        ...(rejected === undefined ? {} : { list: rejected }),
+      });
+    };
+    return withBudget(ledger, () =>
+      answer().catch((error: unknown) => {
+        throw meter.fail(error);
+      }),
+    );
   });
 }
 
@@ -345,16 +324,23 @@ export function answerAttributes(result: GenerateResult): SpanAttributes {
  * carries empty or partial content and parsing it first reports a schema disagreement that is not
  * one. Truncation is the same call it is on the non-streaming path.
  */
-async function streamedAnswer<TOutput extends StandardSchemaV1>(
-  output: TOutput,
+async function streamedAnswer<
+  TInput extends StandardSchemaV1,
+  TOutput extends StandardSchemaV1,
+  V extends PromptVars,
+>(
+  def: LlmDef<TInput, TOutput, V>,
   name: string,
   gateway: Gateway,
   request: GenerateRequest,
   sink: LlmSink,
   span: Span,
+  meter: RunMeter,
 ): Promise<InferOutput<TOutput>> {
-  const result = await streamOneTurn(gateway, request, sink);
+  const { output } = def;
+  const result = await meter.call(streamOneTurn(gateway, request, sink));
   span.setAttributes({ 'llm.attempts': 1, ...answerAttributes(result) });
+  await observeCall(def, meter, result, 1, undefined);
   if (result.stopReason === 'refusal') {
     throw new LlmRefusedError({
       prompt: name,
@@ -380,6 +366,32 @@ async function streamedAnswer<TOutput extends StandardSchemaV1>(
   });
 }
 
+/** One provider call, reported to the declaration's `onCall`. Awaited; a throw fails the call. */
+async function observeCall<
+  TInput extends StandardSchemaV1,
+  TOutput extends StandardSchemaV1,
+  V extends PromptVars,
+>(
+  def: LlmDef<TInput, TOutput, V>,
+  meter: RunMeter,
+  result: GenerateResult,
+  attempt: number,
+  repairOf: readonly string[] | undefined,
+): Promise<void> {
+  if (def.onCall === undefined) return;
+  await def.onCall({
+    attempt,
+    model: result.model,
+    provider: result.provider,
+    stopReason: result.stopReason,
+    usage: result.usage,
+    cost: result.cost,
+    ...(result.costExact === undefined ? {} : { costExact: result.costExact }),
+    ...(repairOf === undefined ? {} : { repairOf }),
+    run: runTotal(result.model, undefined, meter.total()),
+  });
+}
+
 /** `undefined` for "does not fit". Wrapped so a legitimately falsy value is still a hit. */
 async function accept<TOutput extends StandardSchemaV1>(
   schema: TOutput,