@ultimat3/ai 27.8.1 → 27.10.0

package/src/gateway.ts

@@ -16,7 +16,8 @@ import {
   stringField,
 } from '@ultimat3/core';
 import type { Money } from '@ultimat3/money';
-import type { BudgetKeys, BudgetLimits, BudgetStore } from './budget';
+import { billedOf } from './billed';
+import type { BudgetKeys, BudgetLimits, BudgetReservation, BudgetStore } from './budget';
 import {
   assertFiniteLimits,
   BudgetLedger,
@@ -28,6 +29,7 @@ import { AiProviderUnavailableError } from './errors';
 import { resolveModel } from './model-resolve';
 import type { ModelId } from './models';
 import type { GenerateRequest, GenerateResult, Provider, StreamChunk } from './provider';
+import type { SamplingWire } from './sampling';
 
 /**
  * Response cache. Structurally satisfied by `@ultimat3/cache`'s memo/LRU tiers; declared as
@@ -82,6 +84,8 @@ export interface ProviderGatewayInput {
 }
 
 export interface Gateway {
+  /** The wire formats of the providers serving `model` — what `llm.sampling.ignored` is read from. */
+  samplingWires?(model: ModelId): readonly SamplingWire[];
   /**
    * The model a call with none declared runs on — `providerGateway({ defaultModel })`. Read by
    * `llm()` and `agent()` after the declaration and its prompt, so the app's choice is the one
@@ -180,17 +184,29 @@ class GatewayImpl implements Gateway {
     // The estimate is DEBITED here, not merely checked: three concurrent calls under one ledger
     // all read the same `spent()` otherwise, all pass, and all three record against a ceiling
     // only one of them fitted.
-    const reservation = await ledger?.reserve(estimateSpend(resolved));
+    const reservation = await ledger?.reserve(estimateSpend(resolved, this.marks(model)));
 
     let result: GenerateResult;
     try {
       result = await this.attempt(model, (provider) => provider.generate(resolved));
     } catch (error) {
-      // A call that never landed must not go on holding its reservation.
-      await ledger?.release(reservation);
+      // A call that never landed must not go on holding its reservation — and one that DID land
+      // and still failed (`billedOf`: the provider answered with something unusable) is recorded
+      // as what it cost. Releasing it told the ledger a billed call was free.
+      await settleFailure(ledger, reservation, error);
       throw error;
     }
-    await ledger?.record(result.usage, result.cost, reservation);
+    // The provider was PAID by now. Nothing below may turn that answer into a failure: a ledger
+    // that cannot book it gives the reservation back and says so, and the answer is returned.
+    try {
+      await ledger?.record(result.usage, result.costExact ?? result.cost, reservation);
+    } catch (error) {
+      await ledger?.release(reservation).catch(() => undefined);
+      logger.warn('ai.budget.record_failed', {
+        provider: result.provider ?? 'unknown',
+        error: renderThrowable(error),
+      });
+    }
     // A refusal is not an answer, so it is not cached. Storing one would keep serving a decision
     // the classifier might not make twice, long after the prompt that provoked it was fixed.
     if (result.stopReason !== 'refusal') await this.remember(cacheKey, result);
@@ -230,7 +246,7 @@ class GatewayImpl implements Gateway {
     const provider = this.providerFor(model);
 
     const ledger = currentBudget() ?? this.callLedger({});
-    const reservation = await ledger?.reserve(estimateSpend(resolved));
+    const reservation = await ledger?.reserve(estimateSpend(resolved, this.marks(model)));
     let settled = false;
     try {
       for await (const chunk of provider.stream(resolved)) {
@@ -240,17 +256,40 @@ class GatewayImpl implements Gateway {
         }
         // Settled only once `record` has LANDED. Marking it first left a store that threw here
         // holding the reservation and half the record — the `finally` saw a settled stream.
-        await ledger?.record(chunk.result.usage, chunk.result.cost, reservation);
+        const { usage, cost, costExact } = chunk.result;
+        await ledger?.record(usage, costExact ?? cost, reservation);
         settled = true;
         yield { type: 'done', result: { ...chunk.result, provider: provider.name } };
       }
+    } catch (error) {
+      // Thrown after the provider reported usage: the bill is recorded, exactly as `generate` does.
+      // Only when nothing settled yet — a provider that throws AFTER its `done` chunk was already
+      // recorded, and releasing again would credit the ledger a second time.
+      if (!settled) {
+        settled = true;
+        await settleFailure(ledger, reservation, error);
+      }
+      throw error;
     } finally {
-      // A stream that threw, or that its consumer abandoned, never reached `done` — so nothing
-      // reconciled the reservation and it would hold the ceiling for the rest of the window.
+      // A stream its consumer abandoned never reached `done` — so nothing reconciled the
+      // reservation and it would hold the ceiling for the rest of the window.
       if (!settled) await ledger?.release(reservation);
     }
   }
 
+  samplingWires(model: ModelId): readonly SamplingWire[] {
+    return this.config.providers
+      .filter((p) => p.models.includes(model))
+      .flatMap((p) => (p.wire === undefined ? [] : [p.wire]));
+  }
+
+  /** Whether any provider serving `model` would send a hinted request with a cache mark. */
+  private marks(model: ModelId): boolean {
+    return this.config.providers.some(
+      (p) => p.models.includes(model) && p.marksPromptCache?.(model) === true,
+    );
+  }
+
   private providerFor(model: ModelId): Provider {
     const provider = this.config.providers.find((p) => p.models.includes(model));
     if (provider === undefined) {
@@ -319,6 +358,17 @@ class GatewayImpl implements Gateway {
   }
 }
 
+/** A failed call's reservation: reconciled against its bill when the failure carries one, else given back. */
+async function settleFailure(
+  ledger: BudgetLedger | undefined,
+  reservation: BudgetReservation | undefined,
+  error: unknown,
+): Promise<void> {
+  const billed = billedOf(error);
+  if (billed === undefined) return ledger?.release(reservation);
+  await ledger?.record(billed.usage, billed.cost, reservation);
+}
+
 /**
  * The request's completion ceiling, screened at the one seam every model call in an app passes
  * through — `llm()`, `agent()`, an eval, a judge and a hand-built `generate()` alike.
@@ -416,5 +466,8 @@ export function promptCacheKey(request: GenerateRequest): string {
     thinking: request.thinking ?? 'adaptive',
     tools: request.tools ?? [],
     stop: request.stopSequences ?? [],
+    // Only when asked for: an absent knob must leave every key written before it existed as it was.
+    ...(request.temperature === undefined ? {} : { temperature: request.temperature }),
+    ...(request.seed === undefined ? {} : { seed: request.seed }),
   });
 }

package/src/in-process-tool.ts

@@ -0,0 +1,77 @@
+// Single responsibility: `inProcessTool()` — a tool that exists only for an agent in this
+// process. A name, a description, an input schema and an `execute`; nothing an MCP surface, an
+// HTTP route, `openapi.json` or the manifest ever sees.
+//
+// Not a ninth primitive and not an action: it is a function the agent's own action may call, as
+// code in that action's `handle` would be. The AGENT is the operation with a policy; this runs as
+// `ctx.actor` inside it, and anything it reaches that needs its own authorization is an `action()`
+// it calls — whose policy then decides, as always.
+//
+// It exists because the other way to give an agent a tool was to declare an action
+// `mcp: { expose: true }` — the same flag that serves it to every external MCP client. A tool that
+// must never be an MCP tool had to claim it was one.
+
+import { InputInvalidError } from '@ultimat3/action';
+import type { Actor } from '@ultimat3/core';
+import type { InferOutput, StandardSchemaV1 } from '@ultimat3/schema';
+import { formatIssues, toWireSchema, validateAsync } from '@ultimat3/schema';
+import { AiRequestInvalidError } from './errors';
+import { RESPOND } from './respond';
+import type { ProjectableAction } from './tools';
+
+export interface InProcessToolDef<TInput extends StandardSchemaV1> {
+  /** What the model calls it. Letters, digits, `_` and `-`, at most 64 — what every provider takes. */
+  readonly name: string;
+  /** What the model reads to decide when to call it. Required: a tool with none is never chosen well. */
+  readonly description: string;
+  /** Parses what the model sent before `execute` sees it; projected as the tool's input schema. */
+  readonly input: TInput;
+  /**
+   * Runs as the request's own actor — never one the model named. Throw an `UltimateError` to refuse:
+   * its code, cause and fix are what the model reads (`discloseErrors`).
+   */
+  execute(args: { readonly input: InferOutput<TInput>; readonly actor: Actor }): unknown;
+  /** `false` withholds this tool's refusals from the model as a projected tool's are. Default `true`. */
+  readonly discloseErrors?: boolean;
+}
+
+const NAME = /^[A-Za-z0-9_-]{1,64}$/;
+
+/** A tool for `agent({ tools })` that is this process's alone. Declared once, at module scope. */
+export function inProcessTool<TInput extends StandardSchemaV1>(
+  def: InProcessToolDef<TInput>,
+): ProjectableAction {
+  if (!NAME.test(def.name) || def.name === RESPOND) {
+    throw new AiRequestInvalidError({
+      detail: `inProcessTool() was given the name ${JSON.stringify(def.name)}; a tool name is 1 to 64 letters, digits, "_" or "-", and "${RESPOND}" is the answer tool's`,
+      fix: "inProcessTool({ name: 'kbRead', … })   # letters, digits, _ and - only",
+    });
+  }
+  if (def.description.trim() === '') {
+    throw new AiRequestInvalidError({
+      detail: `inProcessTool("${def.name}") has an empty description, and the description is all the model has to decide when to call it`,
+      fix: `inProcessTool({ name: '${def.name}', description: 'Read one document by id.', … })`,
+    });
+  }
+  return {
+    name: def.name,
+    description: def.description,
+    // A schema the wire cannot express throws HERE, at declaration — as `respondFor`'s does.
+    inputJsonSchema: toWireSchema(def.input),
+    inProcess: true,
+    ...(def.discloseErrors === undefined ? {} : { discloseErrors: def.discloseErrors }),
+    async run({ input, actor }) {
+      const parsed = await validateAsync(def.input, input);
+      if (parsed.issues !== undefined) {
+        // The line only: the structured issues are `@ultimat3/action`'s own shape, for a form.
+        throw new InputInvalidError(
+          def.name,
+          formatIssues(parsed.issues).join('; '),
+          undefined,
+          'tool',
+        );
+      }
+      return def.execute({ input: parsed.value, actor });
+    },
+  };
+}

package/src/index.ts

@@ -4,12 +4,38 @@
 /** Re-exported so an `llm` file needs one import, not two. Same object as schema's. */
 export type { Infer } from '@ultimat3/schema';
 export { t } from '@ultimat3/schema';
-export type { AgentBudget, AgentDef, AgentTurn, AgentVarsArgs } from './agent';
 export { agent } from './agent';
+export type {
+  AgentAction,
+  AgentContinuation,
+  AgentRun,
+  AgentTranscript,
+} from './agent-continue';
+export {
+  MAX_TRANSCRIPT_CHARS,
+  TRANSCRIPT_META,
+  TRANSCRIPT_VERSION,
+  transcriptOf,
+} from './agent-continue';
+export type { AgentBudget, AgentDef, AgentTurn, AgentVarsArgs } from './agent-def';
+export {
+  AgentDeadlineError,
+  AgentToolLoopError,
+  AgentTranscriptInvalidError,
+} from './agent-errors';
 export type { AgentBudgetFact, AgentFact } from './agent-facts';
 export { describeAgents, resetAgents } from './agent-facts';
 export type { AgentJobOptions } from './agent-job';
 export { agentJob } from './agent-job';
+export type { AgentSystem } from './agent-system';
+export type { AgentRunUsage } from './agent-usage';
+export {
+  AiProviderUnexpectedError,
+  LlmToolArgumentsError,
+  LlmTruncatedError,
+} from './answer-errors';
+export type { Billed } from './billed';
+export { BILLED_META, billedOf } from './billed';
 export type {
   BudgetKeys,
   BudgetLedger,
@@ -45,6 +71,7 @@ export {
 } from './content-blocks';
 export type { ContentRefusal } from './content-errors';
 export { AiContentUnsupportedError } from './content-errors';
+export { COST_SCALE_DIGITS, costScale, exactCostOf, roundUpCost } from './cost';
 export type { EchoProvider, EchoProviderInput } from './echo-provider';
 export { echoProvider } from './echo-provider';
 export type { Embedder, HashEmbedder, HashEmbedderInput } from './embeddings';
@@ -76,7 +103,6 @@ export {
   LlmOutputInvalidError,
   LlmRefusedError,
   LlmStreamInvalidError,
-  LlmTruncatedError,
   VectorDimMismatchError,
   VectorScopeWidenedError,
 } from './errors';
@@ -126,7 +152,9 @@ export type { HiveDef, HiveSplitArgs } from './hive';
 export { hive } from './hive';
 export { HiveEmptyError } from './hive-errors';
 export type { HiveMember, HiveMemberError, HiveOutput, HiveResult } from './hive-result';
-export type { LlmAction, LlmBudget, LlmDef, LlmVarsArgs } from './llm';
+export type { InProcessToolDef } from './in-process-tool';
+export { inProcessTool } from './in-process-tool';
+export type { LlmAction, LlmBudget, LlmCall, LlmDef, LlmVarsArgs } from './llm';
 export { llm } from './llm';
 export type { LlmCache, LlmScopeArgs, LlmSemanticCache } from './llm-cache';
 export type { LlmStreamChunk } from './llm-stream';
@@ -177,6 +205,21 @@ export {
   promptVersions,
   resetPrompts,
 } from './prompt';
+export type {
+  CacheWire,
+  PromptCacheHint,
+  PromptCacheOption,
+  PromptCacheRule,
+  SystemParts,
+  SystemPrompt,
+} from './prompt-cache';
+export {
+  OPENAI_CACHE_KEY_MAX,
+  PROMPT_CACHE_RULES,
+  promptCacheRule,
+  SESSION_KEY_MAX,
+  systemText,
+} from './prompt-cache';
 export { promptFences } from './prompt-fence';
 export type {
   AiContentBlock,
@@ -230,6 +273,7 @@ export {
   configureAi,
   semanticCacheFor,
 } from './runtime';
+export type { Sampling } from './sampling';
 export type { Scorer } from './scorers';
 export {
   contains,

package/src/llm-def.ts

@@ -0,0 +1,97 @@
+// Single responsibility: what an `llm()` DECLARATION is — its options, and the one event its
+// observer is handed. Types only; ./llm owns the call that reads them. `agent-def.ts`'s twin.
+
+import type { ActionMcp, ActionPolicy } from '@ultimat3/action';
+import type { Ctx } from '@ultimat3/core';
+import type { Money } from '@ultimat3/money';
+import type { InferOutput, StandardSchemaV1 } from '@ultimat3/schema';
+import type { AgentRunUsage } from './agent-usage';
+import type { LlmCache } from './llm-cache';
+import type { ModelId } from './models';
+import type { Prompt, PromptVars } from './prompt';
+import type { PromptCacheOption } from './prompt-cache';
+import type { StopReason, TokenUsage } from './provider';
+
+/** Per-call ceilings, checked before the provider is reached. Never truncates — refuses. */
+export interface LlmBudget {
+  /** Prompt tokens. */
+  readonly tokensIn?: number;
+  /** Worst-case price of one call, integer minor units. */
+  readonly costPerCall?: Money;
+}
+
+export interface LlmVarsArgs<TInput extends StandardSchemaV1> {
+  readonly input: InferOutput<TInput>;
+  readonly ctx: Ctx;
+}
+
+export interface LlmDef<
+  TInput extends StandardSchemaV1,
+  TOutput extends StandardSchemaV1,
+  V extends PromptVars,
+> {
+  /**
+   * Overrides the prompt artifact's own model. The prompt's is part of its content hash and
+   * travels with it; this one is the deployment decision, so it wins.
+   */
+  readonly model?: ModelId;
+  readonly input: TInput;
+  readonly output: TOutput;
+  readonly prompt: Prompt<V>;
+  /**
+   * Everything the prompt is allowed to see, derived from the parsed input. Required, and
+   * async, because the input is usually an id and the prompt needs the row behind it — this
+   * is the one declared place a model call loads data, so a reader can see what was sent.
+   */
+  vars(args: LlmVarsArgs<TInput>): V | Promise<V>;
+  readonly cache?: LlmCache<InferOutput<TInput>>;
+  /**
+   * PROVIDER prompt caching (`./prompt-cache`) — not `cache`, which is this package's own semantic
+   * cache. Default `'off'`: a one-shot call marked for caching pays the write premium (1.25x on
+   * Anthropic) for a prefix nothing may ever read back. `'auto'` is for a call made many times
+   * inside the cache's lifetime with one long `system`; the repair turn reads it back either way.
+   */
+  readonly promptCache?: PromptCacheOption<LlmVarsArgs<TInput>>;
+  readonly budget?: LlmBudget;
+  readonly policy: ActionPolicy;
+  readonly mcp?: ActionMcp;
+  /** Enforced completion ceiling. The model never sees it, so it can be cut off mid-answer. */
+  readonly maxTokens?: number;
+  /**
+   * `temperature` (0 to 2) and `seed` (a whole number), for a call that should answer the same way
+   * twice — a verifier, a classifier. Sent only to a model whose row takes them
+   * (`registerModel({ sampling })`); to any other they are NOT sent, because a reasoning model
+   * answers one with a 400 — the `llm.sampling.ignored` span attribute names what was left out.
+   * `seed` exists on the chat-completions wire only, and is best effort where it exists.
+   */
+  readonly temperature?: number;
+  readonly seed?: number;
+  /**
+   * Called once per PROVIDER call this invocation makes — the answer, and the repair turn when
+   * there is one — before the answer is read. What `agent()`'s `onTurn` is to a run: the usage and
+   * cost of each call, and on a repair call the issues the model was told to fix.
+   *
+   * Observation only. Awaited, and a throw from it FAILS the call, for the reason `onTurn` gives.
+   * Not called on a semantic-cache hit: no provider call was made, and nothing was billed.
+   */
+  onCall?(event: LlmCall): void | Promise<void>;
+}
+
+/** One completed provider call of an `llm()` invocation, as an observer sees it. Facts only. */
+export interface LlmCall {
+  /** 1 is the answer; 2 is the repair turn. A streamed call is always 1. */
+  readonly attempt: number;
+  readonly model: ModelId;
+  /** Which provider answered, as the gateway stamped it. */
+  readonly provider: string | undefined;
+  readonly stopReason: StopReason;
+  readonly usage: TokenUsage;
+  /** This call's cost alone, integer minor units, rounded up. */
+  readonly cost: Money;
+  /** This call's cost exactly (`Money` at a finer `scale`); absent when the provider stated none. */
+  readonly costExact?: Money;
+  /** On the repair turn: the schema issues of the first answer — what the model was told to fix. */
+  readonly repairOf?: readonly string[];
+  /** The invocation so far, this call included — priced once from the summed usage. */
+  readonly run: AgentRunUsage;
+}