@ultimat3/ai 29.4.0 → 30.0.0

package/src/evals.ts

@@ -12,6 +12,8 @@
 // to an exact prompt rather than "whatever was in main that day".
 
 import { finiteCount } from '@ultimat3/core';
+import type { AgentEvalInput, CaseOutcome } from './eval-agent';
+import { agentPrompt, runAgentCase } from './eval-agent';
 import type { EvalBaseline, Regression } from './eval-baseline';
 import {
   baselinePath,
@@ -25,9 +27,11 @@ import { EvalBaselineMissingError, EvalThresholdError } from './eval-errors';
 import type { Gateway } from './gateway';
 import type { Prompt, PromptVars } from './prompt';
 import { describePrompts } from './prompt';
-import type { Scorer } from './scorers';
+import type { EvalRun, Scorer } from './scorers';
 import { clampScore } from './scorers';
 
+export type { AgentEvalCase, AgentEvalInput, EvaluatedAgent } from './eval-agent';
+
 /** One case: the variables to render with, plus what a good answer looks like. */
 export interface EvalCase<V extends PromptVars = PromptVars> {
   readonly name: string;
@@ -36,7 +40,8 @@ export interface EvalCase<V extends PromptVars = PromptVars> {
   readonly expected?: string;
 }
 
-export interface DefineEvalInput<V extends PromptVars = PromptVars> {
+/** The prompt arm: one rendered request per case, no tools — for a prompt `llm()` sends. */
+export interface PromptEvalInput<V extends PromptVars = PromptVars> {
   readonly name: string;
   readonly prompt: Prompt<V>;
   readonly cases: readonly EvalCase<V>[];
@@ -52,11 +57,24 @@ export interface DefineEvalInput<V extends PromptVars = PromptVars> {
   readonly maxTokens?: number;
 }
 
+/**
+ * Either arm. `agent` runs the whole loop — the agent's tools (as the eval's doubles) and turns —
+ * because a prompt that says "answer through the respond tool" measured in a request offering no
+ * tool describes something the agent never runs. Narrow with `'prompt' in input`.
+ */
+export type DefineEvalInput<V extends PromptVars = PromptVars> =
+  | PromptEvalInput<V>
+  | AgentEvalInput;
+
 export interface CaseResult {
   readonly case: string;
   readonly output: string;
   readonly score: number;
   readonly perScorer: Readonly<Record<string, number>>;
+  /** Agent arm: what the run did — its turns and the tools it called, in order. */
+  readonly run?: { readonly turns: number; readonly toolCalls: readonly string[] };
+  /** Agent arm: the code a failed run threw. The case scored 0; the suite went on. */
+  readonly failed?: string;
 }
 
 export interface EvalResult {
@@ -64,6 +82,8 @@ export interface EvalResult {
   readonly promptRef: string;
   /** The prompt content hash. A score means nothing without it. */
   readonly promptHash: string;
+  /** The agent the agent arm ran, by its registered name. Absent on the prompt arm. */
+  readonly agent?: string;
   readonly score: number;
   /** The recorded run mean, or `undefined` when this eval has never been recorded. */
   readonly baseline: number | undefined;
@@ -77,7 +97,10 @@ export interface EvalResult {
 
 export interface Eval<V extends PromptVars = PromptVars> {
   readonly name: string;
+  /** The prompt measured: the arm's own, or the one the agent runs on. */
   readonly prompt: Prompt<V>;
+  /** The agent arm's agent, by its registered name; `''` until registration names it. */
+  readonly agent?: string;
   readonly tolerance: number;
   /** The baseline file, as declared — what `x verify` reports when it cannot be read. */
   readonly baseline: string;
@@ -97,22 +120,36 @@ export interface EvalFact {
   readonly name: string;
   readonly prompt: string;
   readonly promptId: string;
+  /** The agent arm's agent; absent on the prompt arm. */
+  readonly agent?: string;
   readonly tolerance: number;
   readonly baseline: string;
 }
 
 const registry = new Map<string, Eval>();
 
-export function defineEval<V extends PromptVars>(input: DefineEvalInput<V>): Eval<V> {
-  const evaluation: Eval<V> = {
+/** One way to declare an eval, two arms: `prompt` (one request per case) or `agent` (one run). */
+export function defineEval<V extends PromptVars>(input: PromptEvalInput<V>): Eval<V>;
+export function defineEval<I>(input: AgentEvalInput<I>): Eval;
+export function defineEval(input: DefineEvalInput): Eval {
+  const prompt = 'prompt' in input ? input.prompt : agentPrompt(input.agent, input.name);
+  const evaluation: Eval = {
     name: input.name,
-    prompt: input.prompt,
+    prompt,
+    // A getter: an agent is named by `registerAction` at boot, after this ran at module scope.
+    ...('agent' in input
+      ? {
+          get agent() {
+            return input.agent.name;
+          },
+        }
+      : {}),
     tolerance: input.tolerance,
     baseline: input.baseline,
-    run: (gateway) => runEval(input, gateway),
-    assert: (gateway) => assertEval(input, gateway),
+    run: (gateway) => runEval(input, prompt, gateway),
+    assert: (gateway) => assertEval(input, prompt, gateway),
   };
-  registry.set(input.name, evaluation as Eval);
+  registry.set(input.name, evaluation);
   return evaluation;
 }
 
@@ -122,6 +159,7 @@ export function describeEvals(): readonly EvalFact[] {
       name: evaluation.name,
       prompt: evaluation.prompt.ref,
       promptId: evaluation.prompt.id,
+      ...(evaluation.agent === undefined ? {} : { agent: evaluation.agent }),
       tolerance: evaluation.tolerance,
       baseline: evaluation.baseline,
     }))
@@ -141,12 +179,13 @@ export function resetEvals(): void {
   registry.clear();
 }
 
-async function assertEval<V extends PromptVars>(
-  input: DefineEvalInput<V>,
+async function assertEval(
+  input: DefineEvalInput,
+  prompt: Prompt,
   gateway: Gateway,
 ): Promise<EvalResult> {
   const path = baselinePath(input.baseline, input.name);
-  const result = await runEval(input, gateway);
+  const result = await runEval(input, prompt, gateway);
 
   if (recordingBaselines()) {
     await writeBaseline(path, baselineFrom(result));
@@ -172,41 +211,25 @@ async function assertEval<V extends PromptVars>(
   return result;
 }
 
-async function runEval<V extends PromptVars>(
-  input: DefineEvalInput<V>,
+async function runEval(
+  input: DefineEvalInput,
+  prompt: Prompt,
   gateway: Gateway,
 ): Promise<EvalResult> {
   const cases: CaseResult[] = [];
-  for (const testCase of input.cases) {
-    const generated = await gateway.generate({
-      messages: [{ role: 'user' as const, content: input.prompt.render(testCase.vars) }],
-      // A ceiling of `NaN` is not a ceiling: it becomes the pre-flight estimate, passes every
-      // budget check and then poisons the ledger it was checked against. Refused here, per RUN,
-      // because `defineEval` takes it as a plain field and there is no earlier seam.
-      maxTokens: finiteCount('defineEval', 'maxTokens', input.maxTokens ?? 1_024, 1),
-      ...(input.prompt.system !== undefined ? { system: input.prompt.system } : {}),
-      ...(input.prompt.model !== undefined ? { model: input.prompt.model } : {}),
-      ...(input.prompt.effort !== undefined ? { effort: input.prompt.effort } : {}),
-      // `thinking` too, and not because it is tidy: `promptHash` covers it, so the baseline is
-      // filed under a hash describing a configuration the measurement did not use. `llm()` and
-      // `agent()` have always sent both.
-      ...(input.prompt.thinking !== undefined ? { thinking: input.prompt.thinking } : {}),
-    });
-    const perScorer: Record<string, number> = {};
-    for (const scorer of input.scorers) {
-      perScorer[scorer.name] = clampScore(
-        await scorer.score({
-          output: generated.text,
-          ...(testCase.expected !== undefined ? { expected: testCase.expected } : {}),
-        }),
-      );
-    }
-    cases.push({
-      case: testCase.name,
-      output: generated.text,
-      score: mean(Object.values(perScorer)),
-      perScorer,
-    });
+  const outcomes =
+    'prompt' in input
+      ? input.cases.map((testCase) => ({
+          testCase,
+          outcome: () => promptCase(input, testCase.vars, gateway),
+        }))
+      : input.cases.map((testCase) => ({
+          testCase,
+          outcome: () => runAgentCase(input, testCase, gateway),
+        }));
+  // Serial, as before: a case is a paid call (an agent case, several), and order is the report's.
+  for (const { testCase, outcome } of outcomes) {
+    cases.push(await scoreCase(input.scorers, testCase, await outcome()));
   }
 
   const score = mean(cases.map((c) => c.score));
@@ -223,8 +246,9 @@ async function runEval<V extends PromptVars>(
 
   return {
     name: input.name,
-    promptRef: input.prompt.ref,
-    promptHash: input.prompt.hash,
+    promptRef: prompt.ref,
+    promptHash: prompt.hash,
+    ...('agent' in input ? { agent: input.agent.name } : {}),
     score,
     baseline: recorded?.score,
     tolerance: input.tolerance,
@@ -234,6 +258,62 @@ async function runEval<V extends PromptVars>(
   };
 }
 
+/** The prompt arm's one request: the rendered prompt, no tools. */
+async function promptCase(
+  input: PromptEvalInput,
+  vars: PromptVars,
+  gateway: Gateway,
+): Promise<CaseOutcome> {
+  const generated = await gateway.generate({
+    messages: [{ role: 'user' as const, content: input.prompt.render(vars) }],
+    // A ceiling of `NaN` is not a ceiling: it becomes the pre-flight estimate, passes every
+    // budget check and then poisons the ledger it was checked against. Refused here, per RUN,
+    // because `defineEval` takes it as a plain field and there is no earlier seam.
+    maxTokens: finiteCount('defineEval', 'maxTokens', input.maxTokens ?? 1_024, 1),
+    ...(input.prompt.system !== undefined ? { system: input.prompt.system } : {}),
+    ...(input.prompt.model !== undefined ? { model: input.prompt.model } : {}),
+    ...(input.prompt.effort !== undefined ? { effort: input.prompt.effort } : {}),
+    // `thinking` too, and not because it is tidy: `promptHash` covers it, so the baseline is
+    // filed under a hash describing a configuration the measurement did not use. `llm()` and
+    // `agent()` have always sent both.
+    ...(input.prompt.thinking !== undefined ? { thinking: input.prompt.thinking } : {}),
+  });
+  return { ok: true, output: generated.text };
+}
+
+/** Every scorer over one outcome. A failed agent run scores 0 on every scorer, and says why. */
+async function scoreCase(
+  scorers: readonly Scorer[],
+  testCase: { readonly name: string; readonly expected?: string },
+  outcome: CaseOutcome,
+): Promise<CaseResult> {
+  const perScorer: Record<string, number> = {};
+  for (const scorer of scorers) {
+    perScorer[scorer.name] = outcome.ok
+      ? clampScore(
+          await scorer.score({
+            output: outcome.output,
+            ...(testCase.expected !== undefined ? { expected: testCase.expected } : {}),
+            ...(outcome.run === undefined ? {} : { run: outcome.run }),
+          }),
+        )
+      : 0;
+  }
+  return {
+    case: testCase.name,
+    output: outcome.ok ? outcome.output : '',
+    score: mean(Object.values(perScorer)),
+    perScorer,
+    ...(outcome.ok && outcome.run !== undefined ? { run: reported(outcome.run) } : {}),
+    ...(outcome.ok ? {} : { failed: outcome.code }),
+  };
+}
+
+const reported = (run: EvalRun): NonNullable<CaseResult['run']> => ({
+  turns: run.turns,
+  toolCalls: run.toolCalls.map((call) => call.name),
+});
+
 const caseScores = (cases: readonly CaseResult[]): Record<string, number> =>
   Object.fromEntries(cases.map((c) => [c.case, c.score]));
 

package/src/gateway-providers.ts

@@ -0,0 +1,215 @@
+// Single responsibility: which provider the gateway asks, how often, and when it stops asking one
+// that is down. The retry loop across providers, the backoff mapping, the retryable test, and one
+// core `breaker()` per provider: a provider down for an hour cost `attempts` calls plus backoff on
+// EVERY call; behind its breaker it costs `attempts × failures` per cooldown, then none.
+
+import type { Breaker, BreakerTicket, Random } from '@ultimat3/core';
+import {
+  backoffDelay,
+  breaker,
+  isRetryableStatus,
+  isUltimateError,
+  renderThrowable,
+  settleWith,
+  stringField,
+  tryUseContext,
+} from '@ultimat3/core';
+import { AiProviderUnavailableError } from './errors';
+import type { RetryPolicy } from './gateway';
+import type { ModelId } from './models';
+import type { GenerateResult, Provider } from './provider';
+
+/**
+ * `providerGateway({ breaker })`: the breaker each provider sits behind, by a function of the
+ * provider (one spelling: build it with core's `breaker()`); `'off'` declines. Absent, one per
+ * provider name at `DEFAULT_GATEWAY_BREAKER`.
+ */
+export type GatewayBreakerOption = ((provider: Provider) => Breaker) | 'off';
+
+/** Five consecutive exhausted passes open a provider for thirty seconds. */
+export const DEFAULT_GATEWAY_BREAKER = Object.freeze({ failures: 5, cooldownMs: 30_000 });
+
+/** The one code `attempt` may collect into. Every other coded refusal is the caller's answer. */
+const PROVIDER_UNAVAILABLE = 'X_AI_PROVIDER_UNAVAILABLE';
+
+/**
+ * Full jitter: a uniform pick from [0, exponential], capped BEFORE the roll — core's
+ * `backoffDelay` with the gateway's own field names mapped onto it, and nothing else.
+ * `gateway-backoff.test.ts` pins the refusal of a non-finite bound and the clamped negative base.
+ */
+export function backoffMs(policy: RetryPolicy, attempt: number, random?: Random): number {
+  return backoffDelay({
+    attempt,
+    base: policy.baseDelayMs,
+    max: policy.maxDelayMs,
+    curve: 'exponential',
+    jitter: 'full',
+    random,
+  });
+}
+
+/**
+ * A transport that never produced a response. Node spells these `ETIMEDOUT`/`ECONNRESET`; Bun's
+ * `fetch` spells a refused, dropped or unopenable connection `ConnectionRefused`,
+ * `ConnectionClosed` and `FailedToOpenSocket`.
+ */
+const RETRYABLE_TRANSPORT_CODES: ReadonlySet<string> = new Set([
+  'ETIMEDOUT',
+  'ECONNRESET',
+  'ConnectionRefused',
+  'ConnectionClosed',
+  'FailedToOpenSocket',
+]);
+
+/**
+ * Retryable = the request was well formed and the provider was momentarily unable. A 400 is
+ * never retried: the same body produces the same rejection and only burns the budget. The status
+ * half is core's `isRetryableStatus`; the `code` half has no equivalent in core (a socket that
+ * timed out never produced a status).
+ */
+export function isRetryable(error: unknown): boolean {
+  if (typeof error !== 'object' || error === null) return false;
+  // A `Provider` is the APP's object: `e.status` is a getter call and, on a `Proxy`, a trap. A
+  // value that fights being read cannot be SHOWN to be retryable, so it fails closed.
+  try {
+    const e = error as { status?: unknown; code?: unknown };
+    if (typeof e.status === 'number') return isRetryableStatus(e.status);
+    return typeof e.code === 'string' && RETRYABLE_TRANSPORT_CODES.has(e.code);
+  } catch {
+    return false;
+  }
+}
+
+/** One breaker per provider name, made on first use. `undefined` under `'off'`. */
+export function providerBreakers(
+  option: GatewayBreakerOption | undefined,
+): (provider: Provider) => Breaker | undefined {
+  if (option === 'off') return () => undefined;
+  if (option !== undefined) return option;
+  const gates = new Map<string, Breaker>();
+  return (provider) => {
+    let gate = gates.get(provider.name);
+    if (gate === undefined) {
+      gate = breaker({ name: `ai.${provider.name}`, ...DEFAULT_GATEWAY_BREAKER });
+      gates.set(provider.name, gate);
+    }
+    return gate;
+  };
+}
+
+/** A pass that ran out: a failure about the provider, unless the caller went away meanwhile. */
+const exhausted = (gate: Breaker | undefined, ticket: BreakerTicket | undefined): void => {
+  if (tryUseContext()?.signal.aborted === true) gate?.released(ticket);
+  else gate?.failed(ticket);
+};
+
+export interface AttemptInput {
+  readonly model: ModelId;
+  readonly candidates: readonly Provider[];
+  readonly retry: RetryPolicy;
+  readonly sleep: (ms: number) => Promise<void>;
+  readonly random: Random | undefined;
+  readonly gateOf: (provider: Provider) => Breaker | undefined;
+  readonly call: (provider: Provider) => Promise<GenerateResult>;
+}
+
+/**
+ * Try every provider that serves the model, retrying each on a retryable failure with full-jitter
+ * backoff, skipping one whose breaker is open — no call, no sleep. Fallback is across PROVIDERS
+ * serving one model, never across models: a silent model swap changes what answered and what it
+ * cost. The provider that answered is stamped onto the result.
+ */
+export async function attemptProviders(input: AttemptInput): Promise<GenerateResult> {
+  const { model, retry } = input;
+  const failures: string[] = [];
+  if (input.candidates.length === 0) {
+    throw new AiProviderUnavailableError({
+      model,
+      attempts: [`no provider serves ${model}`],
+      unserved: true,
+    });
+  }
+  for (const provider of input.candidates) {
+    const gate = input.gateOf(provider);
+    const ticket = gate?.admit();
+    if (gate !== undefined && ticket === undefined) {
+      failures.push(`${provider.name}: breaker open`);
+      continue;
+    }
+    for (let attempt = 1; attempt <= retry.attempts; attempt += 1) {
+      try {
+        const answer = { ...(await input.call(provider)), provider: provider.name };
+        gate?.succeeded(ticket);
+        return answer;
+      } catch (error) {
+        // A coded refusal that is NOT `X_AI_PROVIDER_UNAVAILABLE` reaches the caller verbatim: it
+        // was raised before the socket opened, and the same rejection waits on every provider. The
+        // provider did not fail, so its breaker hears an answer. `stringField`, never `error.code`:
+        // the value came from an app's `Provider`, and a property read on it can trap.
+        if (isUltimateError(error) && stringField(error, 'code') !== PROVIDER_UNAVAILABLE) {
+          // The breaker's own verdict: a caller's abort (`X_ABORTED`, `X_DRAINING`) gives the probe
+          // back and closes nothing; a local refusal is the provider answering.
+          if (gate !== undefined) settleWith(gate, gate.judge(error), ticket);
+          throw error;
+        }
+        // `renderThrowable`: this line becomes a `cause`, and a renderer that throws would replace
+        // the coded refusal with a `TypeError`. It bounds a provider's 1 MB body too.
+        failures.push(`${provider.name}#${attempt}: ${renderThrowable(error)}`);
+        if (!isRetryable(error)) {
+          // A 400 is the provider answering: it is up, and a 400 never opens its breaker.
+          gate?.succeeded(ticket);
+          break;
+        }
+        if (attempt === retry.attempts) {
+          exhausted(gate, ticket);
+          break;
+        }
+        await input.sleep(backoffMs(retry, attempt, input.random));
+      }
+    }
+  }
+  throw new AiProviderUnavailableError({ model, attempts: failures });
+}
+
+/**
+ * The provider a STREAM goes to: the first serving the model whose breaker admits it, with its
+ * ticket. A stream never retries (tokens already handed over cannot be replayed).
+ */
+export function admittedProvider(
+  model: ModelId,
+  providers: readonly Provider[],
+  gateOf: (provider: Provider) => Breaker | undefined,
+): { readonly provider: Provider; readonly ticket: BreakerTicket | undefined } {
+  const candidates = providers.filter((p) => p.models.includes(model));
+  if (candidates.length === 0) {
+    throw new AiProviderUnavailableError({
+      model,
+      attempts: providers.map((p) => `${p.name}: does not serve ${model}`),
+      unserved: true,
+    });
+  }
+  for (const provider of candidates) {
+    const gate = gateOf(provider);
+    if (gate === undefined) return { provider, ticket: undefined };
+    const ticket = gate.admit();
+    if (ticket !== undefined) return { provider, ticket };
+  }
+  throw new AiProviderUnavailableError({
+    model,
+    attempts: candidates.map((p) => `${p.name}: breaker open`),
+  });
+}
+
+/** A stream's first chunk or first throw is the provider's verdict; an abandoned one says nothing. */
+export function streamVerdict(
+  gate: Breaker | undefined,
+  ticket: BreakerTicket | undefined,
+  error: unknown,
+): void {
+  if (gate === undefined) return;
+  if (error === undefined) gate.succeeded(ticket);
+  else if (tryUseContext()?.signal.aborted === true) gate.released(ticket);
+  else if (isRetryable(error) || stringField(error, 'code') === PROVIDER_UNAVAILABLE) {
+    gate.failed(ticket);
+  } else settleWith(gate, gate.judge(error), ticket);
+}