@ultimat3/ai 27.8.1 → 27.10.0

package/src/agent-def.ts

@@ -0,0 +1,134 @@
+// Single responsibility: what an `agent()` DECLARATION is — its options, and the one event its
+// observer is handed. Types only; ./agent owns the loop that reads them.
+
+import type { ActionMcp, ActionPolicy } from '@ultimat3/action';
+import type { Ctx } from '@ultimat3/core';
+import type { Money } from '@ultimat3/money';
+import type { InferOutput, StandardSchemaV1 } from '@ultimat3/schema';
+import type { AgentSystem } from './agent-system';
+import type { AgentRunUsage } from './agent-usage';
+import type { LlmBudget } from './llm';
+import type { ModelId } from './models';
+import type { Prompt, PromptVars } from './prompt';
+import type { PromptCacheOption } from './prompt-cache';
+import type { StopReason, TokenUsage } from './provider';
+import type { AgentTool } from './tools';
+
+export interface AgentBudget extends LlmBudget {
+  /**
+   * Token ceiling for the WHOLE run, every turn counted. The one ceiling `llm()` does not need:
+   * a single call is bounded by `maxTokens`, a loop is bounded by nothing until this is set.
+   */
+  readonly tokensPerRun?: number;
+}
+
+export interface AgentVarsArgs<TInput extends StandardSchemaV1> {
+  readonly input: InferOutput<TInput>;
+  readonly ctx: Ctx;
+}
+
+export interface AgentDef<
+  TInput extends StandardSchemaV1,
+  TOutput extends StandardSchemaV1,
+  V extends PromptVars,
+> {
+  readonly model?: ModelId;
+  readonly input: TInput;
+  readonly output: TOutput;
+  readonly prompt: Prompt<V>;
+  /** Same contract as `llm()`: the one declared place a run loads data. */
+  vars(args: AgentVarsArgs<TInput>): V | Promise<V>;
+  /**
+   * System text loaded at RUN time, beside the prompt's static `system`: a knowledge-base index,
+   * today's date. Redacted like the prompt. Not part of the prompt's hash, exactly as `vars` is not.
+   */
+  system?(args: AgentVarsArgs<TInput>): AgentSystem | Promise<AgentSystem>;
+  /**
+   * Provider prompt caching. Default `'auto'`: every turn's request asks the provider to cache its
+   * prefix (the stable system block, then the transcript up to its tail), so turn N+1 reads turn
+   * N's prefix at the cache rate instead of paying input for it again. `'off'` sends no mark.
+   * `{ session }` is `'auto'` plus a key for hosts that route by one. `PROMPT_CACHE_RULES` decides
+   * what each wire is sent; a host that caches implicitly is sent nothing either way.
+   */
+  readonly promptCache?: PromptCacheOption<AgentVarsArgs<TInput>>;
+  /**
+   * What the model may call: an `action()` that is `mcp: { expose: true }` — the same predicate an
+   * external MCP client is filtered by, so both are offered the same actions — or an
+   * `inProcessTool()`, which only this process's agents ever see. Anything else is refused at
+   * declaration rather than dropped: a tool that reads as offered and silently is not is the worst
+   * of both.
+   */
+  readonly tools: readonly AgentTool[];
+  /**
+   * `false` withholds the refusals of EVERY in-process tool this agent runs, as a projected tool's
+   * are — for an agent whose transcript leaves the tenant it ran for. Default `true`; one tool
+   * opts out with `inProcessTool({ discloseErrors: false })`.
+   */
+  readonly discloseToolErrors?: boolean;
+  /** Hard ceiling on model turns. Reaching it is `X_AGENT_MAX_TURNS`, never a partial answer. */
+  readonly maxTurns?: number;
+  readonly maxToolResultChars?: number;
+  /**
+   * Consecutive turns that may make the SAME tool calls and read the SAME results. Reaching it is
+   * `X_AGENT_TOOL_LOOP`. Default 3.
+   */
+  readonly maxRepeatedToolCalls?: number;
+  /**
+   * Wall-clock ceiling for the whole run, in milliseconds. Passing it aborts the model call in
+   * flight and is `X_AGENT_DEADLINE`. No default: a timeout nobody declared is a lost answer.
+   */
+  readonly deadlineMs?: number;
+  readonly budget?: AgentBudget;
+  readonly policy: ActionPolicy;
+  readonly mcp?: ActionMcp;
+  /** Enforced completion ceiling PER TURN. The model never sees it. */
+  readonly maxTokens?: number;
+  /**
+   * `temperature` (0 to 2) and `seed` (a whole number), for a call that should answer the same way
+   * twice — a verifier, a classifier. Sent only to a model whose row takes them
+   * (`registerModel({ sampling })`); to any other they are NOT sent, because a reasoning model
+   * answers one with a 400 — the `llm.sampling.ignored` span attribute names what was left out.
+   * `seed` exists on the chat-completions wire only, and is best effort where it exists.
+   */
+  readonly temperature?: number;
+  readonly seed?: number;
+  /**
+   * Called once per completed model turn, before the answer or the tool calls are acted on. The
+   * one thing a multi-turn run could not do until 2026-08: a 90-second loop emitted nothing until
+   * it returned, so a progress indicator, a per-turn spend line and a transcript log all had to be
+   * hand-rolled outside the framework — which is exactly the loop `agent()` exists to replace.
+   *
+   * Observation only: it cannot steer the loop, cannot see the transcript and cannot reach the
+   * actor. A throw from it FAILS the run rather than being swallowed — it is the app's code on the
+   * run's own path, and an observer that silently stopped working would be indistinguishable from
+   * one that is fine. The same facts land on the span as an `agent.turn` event, so a run that
+   * declares no hook is still readable in a trace.
+   *
+   * NOT `.stream()`. Tokens on a screen is a different contract, and `llm()`'s streamed half is
+   * one turn deep by construction; this is per TURN.
+   */
+  onTurn?(event: AgentTurn): void | Promise<void>;
+}
+
+/** One completed model turn, as an observer sees it. Facts only — no transcript, no actor. */
+export interface AgentTurn {
+  /** 1-based, so `turn === maxTurns` is the last one this run will take. */
+  readonly turn: number;
+  readonly maxTurns: number;
+  readonly model: ModelId;
+  /** Tools this turn asked for, in the order the model emitted them. `respond` is not one. */
+  readonly toolCalls: readonly string[];
+  readonly stopReason: StopReason;
+  readonly usage: TokenUsage;
+  /** This turn's cost alone, integer minor units, rounded up — never the run's running total. */
+  readonly cost: Money;
+  /** This turn's cost exactly (`Money` at a finer `scale`); absent when the provider stated none. */
+  readonly costExact?: Money;
+  /**
+   * Set on a REPAIR turn: the schema issues of the previous turn's answer, which is what the
+   * model was told to fix before it produced this one. Absent on every other turn.
+   */
+  readonly repairOf?: readonly string[];
+  /** The running total, this turn included — on the last turn, the whole run. */
+  readonly run: AgentRunUsage;
+}

package/src/agent-errors.ts

@@ -0,0 +1,69 @@
+// The X_AGENT_* codes an UNATTENDED run needs, and the one a continuation's handle is refused by, apart from ./errors only because one file has
+// one job and that catalogue is at its ceiling — the split `hive-errors.ts` made. The codes, their
+// titles and the single `registerErrorCodes` call stay in ./errors.
+
+import { UltimateError } from '@ultimat3/core';
+
+/**
+ * The loop asked for the same tool calls and read the same results, turn after turn.
+ *
+ * Call AND result, never the call alone: polling one tool with one argument is legitimate while
+ * what comes back keeps changing, and it stops being progress the moment it does not. A turn that
+ * repeats both can only be followed by a turn that repeats both, so every further turn buys the
+ * same non-answer over a transcript that is one turn longer.
+ */
+export class AgentToolLoopError extends UltimateError {
+  constructor(input: {
+    agent: string;
+    /** Laps of the cycle. */
+    repeats: number;
+    /** Turns in one lap: 1 is the same turn over and over, 2 is A,B,A,B. */
+    cycle: number;
+    tools: readonly string[];
+  }) {
+    super({
+      code: 'X_AGENT_TOOL_LOOP',
+      cause:
+        input.cycle === 1
+          ? `agent "${input.agent}" made the same tool calls (${input.tools.join(', ')}) and read the same results ${input.repeats} turns in a row`
+          : `agent "${input.agent}" went round the same ${input.cycle} tool turns (${input.tools.join(', ')}), reading the same results, ${input.repeats} times in a row`,
+      fix: 'maxRepeatedToolCalls: 5   # on the agent, ONLY for a tool that is polled and answers identically while it waits — otherwise tell the template what to do when a tool result does not change, then bump its version',
+      meta: { agent: input.agent, repeats: input.repeats, cycle: input.cycle, tools: input.tools },
+    });
+  }
+}
+
+/**
+ * The run was still going when its declared `deadlineMs` elapsed. Distinct from `X_ABORTED` — the
+ * caller did not leave, the declaration ran out — and from `X_AGENT_MAX_TURNS`, because the fix is
+ * about wall clock: a slow tool or a slow model, not a loop with no exit.
+ */
+export class AgentDeadlineError extends UltimateError {
+  constructor(input: { agent: string; deadlineMs: number }) {
+    super({
+      code: 'X_AGENT_DEADLINE',
+      cause: `agent "${input.agent}" was still running after its ${input.deadlineMs}ms deadline; whatever was in flight (setup, a model call or a tool) was signalled to stop, and no further turn or tool was started`,
+      fix: 'deadlineMs: 300_000   # on the agent, only once a trace shows the run converging — otherwise lower maxTurns, or shorten the slow tool the agent.turn span events name',
+      meta: { agent: input.agent, deadlineMs: input.deadlineMs },
+    });
+  }
+}
+
+/**
+ * A transcript handle handed to `.continue()` did not pass its screen. The handle is data an app
+ * stored and read back, so it is refused BEFORE anything is loaded, reserved or sent: a transcript
+ * with a tool call nothing answers is a 400 from every provider, and one from another agent or
+ * another format would resume a conversation this declaration never had.
+ */
+export class AgentTranscriptInvalidError extends UltimateError {
+  constructor(input: { agent: string; reason: string; fix?: string }) {
+    super({
+      code: 'X_AGENT_TRANSCRIPT_INVALID',
+      cause: `agent "${input.agent}" cannot continue from the transcript it was given: ${input.reason}`,
+      fix:
+        input.fix ??
+        'agent.run(input, { ctx })   # start a new run and store ITS transcript — pass back exactly the value .run() or .continue() returned, from the same agent and prompt version',
+      meta: { agent: input.agent },
+    });
+  }
+}

package/src/agent-guards.ts

@@ -0,0 +1,114 @@
+// Single responsibility: the two guards that let an `agent()` run with nobody watching — the
+// repeated-turn detector and the per-run deadline. Apart from ./agent so the loop reads as a loop.
+
+import { fingerprint, withChildContext } from '@ultimat3/core';
+import { AgentDeadlineError, AgentToolLoopError } from './agent-errors';
+import type { LlmToolCall, LlmToolResult } from './tools';
+
+/**
+ * Consecutive identical turns an agent may take when the declaration omits a number. Three: one
+ * repeat is a model re-checking, two is a model that has not read the answer, and nothing a third
+ * identical turn returns can make the fourth differ.
+ */
+export const DEFAULT_MAX_REPEATED_TOOL_CALLS = 3;
+
+/** The longest cycle looked for: A,B,C,A,B,C. A longer one is `maxTurns`' to stop. */
+const MAX_CYCLE = 3;
+
+/**
+ * Called once per TOOL turn, with what the model asked for and what it was told. Throws
+ * `X_AGENT_TOOL_LOOP` when the most recent tool turns are one cycle of up to `MAX_CYCLE` turns
+ * gone round `limit` times — the same turn `limit` times, or A,B,A,B… — where "the same" is the
+ * calls, their arguments, the results AND whether each failed. A changed result is progress and
+ * breaks the cycle. A fingerprint, not the text: one result can be megabytes before truncation.
+ *
+ * A repair turn (a wrong-shaped answer; no tool ran) is not a tool turn, so it neither counts nor
+ * resets: the correction is about the answer's shape and tells the model nothing new about a tool.
+ */
+export function toolLoopGuard(
+  agent: string,
+  limit: number,
+): (calls: readonly LlmToolCall[], results: readonly LlmToolResult[]) => void {
+  const window = MAX_CYCLE * limit;
+  let recent: readonly { readonly shape: string; readonly tools: readonly string[] }[] = [];
+  return (calls, results) => {
+    const shape = fingerprint({
+      calls: calls.map((call) => [call.name, call.input]),
+      results: results.map((result) => [result.content, result.isError === true]),
+    });
+    recent = [...recent, { shape, tools: calls.map((call) => call.name) }].slice(-window);
+    for (let cycle = 1; cycle <= MAX_CYCLE; cycle += 1) {
+      const laps = recent.slice(-cycle * limit);
+      if (laps.length < cycle * limit) break;
+      const repeats = (turn: (typeof laps)[number], index: number): boolean =>
+        index < cycle || isSameTurn(turn.shape, laps[index - cycle]?.shape);
+      if (!laps.every(repeats)) continue;
+      const tools = [...new Set(laps.slice(0, cycle).flatMap((turn) => turn.tools))];
+      throw new AgentToolLoopError({ agent, repeats: limit, cycle, tools });
+    }
+  };
+}
+
+const isSameTurn = (one: string, other: string | undefined): boolean => one === other;
+
+export interface RunDeadline {
+  /** The caller's signal and the deadline as one — what every provider call is handed. */
+  readonly signal: AbortSignal;
+  /** Throws `X_AGENT_DEADLINE` once the deadline has passed. Called before each turn and batch. */
+  check(): void;
+  /**
+   * Await `work`, but no longer than the deadline, and report a failure the deadline caused AS the
+   * deadline: the aborted socket surfaces as a transport error, which would otherwise read as the
+   * provider's fault. Work that outlives the race is left to finish — a tool has no signal to hear.
+   */
+  guard<T>(work: Promise<T>): Promise<T>;
+  /**
+   * Run `fn` with the deadline on the ambient `ctx.signal`, so a tool — an `action()` reading its
+   * own `ctx.signal`, a `fetch(url, { signal })` inside it — hears the deadline and can stop. A
+   * tool that never reads the signal still runs to its own end after the run has been refused.
+   */
+  scope<T>(fn: () => T): T;
+}
+
+/** `ms` undefined is no deadline: the caller's signal, and two no-ops. */
+export function runDeadline(
+  agent: string,
+  ms: number | undefined,
+  caller: AbortSignal,
+): RunDeadline {
+  if (ms === undefined) {
+    return { signal: caller, check: () => {}, guard: (work) => work, scope: (fn) => fn() };
+  }
+  const timer = AbortSignal.timeout(ms);
+  const expired = (): AgentDeadlineError => new AgentDeadlineError({ agent, deadlineMs: ms });
+  // The caller leaving first stays `X_ABORTED`: it is the truer account of why the run stopped.
+  const byDeadline = (): boolean => timer.aborted && !caller.aborted;
+  return {
+    signal: AbortSignal.any([caller, timer]),
+    scope: (fn) => withChildContext({ signal: timer }, fn),
+    check: () => {
+      if (byDeadline()) throw expired();
+    },
+    guard: async (work) => {
+      if (byDeadline()) {
+        // Abandoned before it was awaited; its own rejection must not go unhandled.
+        work.catch(() => {});
+        throw expired();
+      }
+      let stop = (): void => {};
+      const timeout = new Promise<never>((_, reject) => {
+        stop = () => reject(expired());
+        timer.addEventListener('abort', stop, { once: true });
+      });
+      try {
+        return await Promise.race([work, timeout]);
+      } catch (error) {
+        work.catch(() => {});
+        if (byDeadline()) throw expired();
+        throw error;
+      } finally {
+        timer.removeEventListener('abort', stop);
+      }
+    },
+  };
+}

package/src/agent-system.ts

@@ -0,0 +1,37 @@
+// Single responsibility: the system prompt ONE agent run sends — the prompt's static text and what
+// the declaration's `system()` loaded, composed into the stable and per-run parts a cache keys on.
+
+import type { SystemPrompt } from './prompt-cache';
+
+/**
+ * What `system()` adds to the prompt's own `system`. `stable` joins it in the block every run
+ * shares — it must be BYTE-identical run to run (sorted, undated), or each run pays a full cache
+ * write for it. `run` is a second block after it, for whatever differs.
+ */
+export interface AgentSystem {
+  readonly stable?: string;
+  readonly run?: string;
+}
+
+/**
+ * The system prompt one run sends: the prompt's static text and `system()`'s stable part as ONE
+ * block (the half every run shares), the per-run part after it. A plain string when there is no
+ * per-run part — the shape every request had before the parts existed.
+ */
+export function systemOf(
+  fixed: string | undefined,
+  extra: AgentSystem,
+  redact: (text: string) => string,
+): SystemPrompt | undefined {
+  const stable = [fixed ?? '', extra.stable ?? '']
+    .filter((part) => part !== '')
+    .map(redact)
+    .join('\n\n');
+  const run = extra.run === undefined || extra.run === '' ? '' : redact(extra.run);
+  if (run !== '') return { stable, run };
+  return fixed === undefined && stable === '' ? undefined : stable;
+}
+
+/** Whether two composed prompts are the same text — what `llm.redacted` compares. */
+export const sameSystem = (a: SystemPrompt | undefined, b: SystemPrompt | undefined): boolean =>
+  JSON.stringify(a) === JSON.stringify(b);

package/src/agent-turn.ts

@@ -0,0 +1,54 @@
+// Single responsibility: what the loop does with ONE completed model turn before it acts on it —
+// report it, and refuse the two stops that are not an answer. Split from ./agent at its line
+// ceiling.
+
+import type { Span } from '@ultimat3/core';
+import type { StandardSchemaV1 } from '@ultimat3/schema';
+import type { AgentDef, AgentTurn } from './agent-def';
+import { LlmRefusedError } from './errors';
+import { moreCapableThan } from './models';
+import type { PromptVars } from './prompt';
+import type { GenerateRequest, GenerateResult } from './provider';
+
+/**
+ * One turn, reported twice: onto the span every run already opens, and to the declaration's own
+ * hook. The span event is free — nothing to install, and a trace is where a stalled run is
+ * actually diagnosed — and the hook is what a progress indicator or a per-turn spend line reads.
+ *
+ * Awaited, and not guarded: a throw from `onTurn` fails the run. It is the app's code on the run's
+ * path, and an observer that quietly stopped working reads exactly like one that is fine.
+ */
+export async function observeTurn<
+  TInput extends StandardSchemaV1,
+  TOutput extends StandardSchemaV1,
+  V extends PromptVars,
+>(def: AgentDef<TInput, TOutput, V>, span: Span, event: AgentTurn): Promise<void> {
+  span.addEvent('agent.turn', {
+    'agent.turn': event.turn,
+    'agent.turn.tool_calls': event.toolCalls.length,
+    'llm.stop': event.stopReason,
+    'llm.cost.minor': event.cost.minor,
+    'agent.turn.repair': event.repairOf !== undefined,
+    // What the provider's prompt cache did for this turn — the two numbers that say whether the
+    // stable prefix is really stable.
+    'llm.cache.read_tokens': event.usage.cacheReadTokens,
+    'llm.cache.write_tokens': event.usage.cacheWriteTokens,
+  });
+  await def.onTurn?.(event);
+}
+
+/** Refuse before the answer is read, for the reason `llm()` does: a refusal is a 200 with no answer. */
+export function assertAnswerable(
+  result: GenerateResult,
+  name: string,
+  request: GenerateRequest,
+): void {
+  if (result.stopReason !== 'refusal') return;
+  throw new LlmRefusedError({
+    prompt: name,
+    model: result.model,
+    alternative: moreCapableThan(result.model, request),
+    category: result.stopDetails?.category,
+    explanation: result.stopDetails?.explanation,
+  });
+}

package/src/agent-usage.ts

@@ -0,0 +1,63 @@
+// Single responsibility: a run's running token and money total, across its turns and its
+// continuations.
+
+import type { Money } from '@ultimat3/money';
+import type { Billed } from './billed';
+import { addMoney, costOf, exactCostOf, roundUpCost } from './cost';
+import type { ModelId } from './models';
+import type { GenerateResult, TokenUsage } from './provider';
+
+/** The run so far, this turn included. */
+export interface AgentRunUsage {
+  readonly usage: TokenUsage;
+  /** In the currency's minor units, rounded up ONCE over the whole run — never a sum of turns. */
+  readonly cost: Money;
+  /** The same figure exactly, as `Money` at a finer `scale` (`exactCostOf`). */
+  readonly costExact: Money;
+}
+
+const ZERO: TokenUsage = {
+  inputTokens: 0,
+  outputTokens: 0,
+  cacheReadTokens: 0,
+  cacheWriteTokens: 0,
+};
+
+const plus = (a: TokenUsage, b: TokenUsage): TokenUsage => ({
+  inputTokens: a.inputTokens + b.inputTokens,
+  outputTokens: a.outputTokens + b.outputTokens,
+  cacheReadTokens: a.cacheReadTokens + b.cacheReadTokens,
+  cacheWriteTokens: a.cacheWriteTokens + b.cacheWriteTokens,
+});
+
+/** `usage` on `model`, priced once: each turn's own cost is rounded up, and ten round-ups is not the price of the run. */
+const priced = (model: ModelId, usage: TokenUsage): AgentRunUsage => ({
+  usage,
+  cost: costOf(model, usage),
+  costExact: exactCostOf(model, usage),
+});
+
+/** The run's total after one more turn, re-priced from the summed usage. */
+export function addUsage(total: AgentRunUsage | undefined, result: GenerateResult): AgentRunUsage {
+  return priced(result.model, plus(total?.usage ?? ZERO, result.usage));
+}
+
+/**
+ * A run's total across its continuations: what the handle already held plus what this call used,
+ * priced once from the sum.
+ */
+export function runTotal(
+  model: ModelId,
+  prior: TokenUsage | undefined,
+  own: { readonly usage: TokenUsage } | undefined,
+  /** What runs underneath were billed (`RunMeter.below`): another model's price, so ADDED, never re-priced. */
+  below?: Billed,
+): AgentRunUsage {
+  const base = priced(model, plus(prior ?? ZERO, own?.usage ?? ZERO));
+  if (below === undefined || below.cost.currency !== base.cost.currency) return base;
+  return {
+    usage: plus(base.usage, below.usage),
+    cost: addMoney(base.cost, roundUpCost(below.cost)),
+    costExact: addMoney(base.costExact, below.cost),
+  };
+}