@ultimat3/ai 1.0.0

package/src/evals.ts

@@ -0,0 +1,242 @@
+// Evals are a test type.
+//
+// Not a notebook, not a dashboard, not a weekly report — a `bun test` case that fails CI.
+// A prompt change that drops a score below its recorded baseline should break the build exactly
+// like a type error does, because the consequence is the same: shipping something wrong.
+//
+// The gate is the DROP, never the absolute score: an absolute floor fails every eval at once the
+// day the provider ships a slightly different model, which teaches everyone to lower thresholds
+// until they measure nothing. `tolerance` is how far a score may fall before it is a regression.
+//
+// Every result is filed against a prompt's content hash, so a score is always attributable
+// to an exact prompt rather than "whatever was in main that day".
+
+import { EvalBaselineMissingError, EvalThresholdError } from './errors';
+import type { EvalBaseline, Regression } from './eval-baseline';
+import {
+  baselinePath,
+  describeRegression,
+  readBaseline,
+  recordingBaselines,
+  regressionsAgainst,
+  writeBaseline,
+} from './eval-baseline';
+import type { Gateway } from './gateway';
+import type { Prompt, PromptVars } from './prompt';
+import { describePrompts } from './prompt';
+import type { Scorer } from './scorers';
+import { clampScore } from './scorers';
+
+/** One case: the variables to render with, plus what a good answer looks like. */
+export interface EvalCase<V extends PromptVars = PromptVars> {
+  readonly name: string;
+  readonly vars: V;
+  /** Reference answer, or the substring/JSON the scorers check for. */
+  readonly expected?: string;
+}
+
+export interface DefineEvalInput<V extends PromptVars = PromptVars> {
+  readonly name: string;
+  readonly prompt: Prompt<V>;
+  readonly cases: readonly EvalCase<V>[];
+  readonly scorers: readonly Scorer[];
+  /**
+   * The committed scores this run is compared against. Write it as
+   * `import.meta.resolve('./name.baseline.json')` — a cwd-relative path resolves to a different
+   * file depending on where the suite was started.
+   */
+  readonly baseline: string;
+  /** How far a score may fall before it is a regression. Explicit: widening it is a diff. */
+  readonly tolerance: number;
+  readonly maxTokens?: number;
+}
+
+export interface CaseResult {
+  readonly case: string;
+  readonly output: string;
+  readonly score: number;
+  readonly perScorer: Readonly<Record<string, number>>;
+}
+
+export interface EvalResult {
+  readonly name: string;
+  readonly promptRef: string;
+  /** The prompt content hash. A score means nothing without it. */
+  readonly promptHash: string;
+  readonly score: number;
+  /** The recorded run mean, or `undefined` when this eval has never been recorded. */
+  readonly baseline: number | undefined;
+  readonly tolerance: number;
+  /** Every score that fell further than `tolerance` — the run mean and each case. */
+  readonly regressions: readonly Regression[];
+  /** A baseline exists and nothing regressed. Anything else is a red gate. */
+  readonly passed: boolean;
+  readonly cases: readonly CaseResult[];
+}
+
+export interface Eval<V extends PromptVars = PromptVars> {
+  readonly name: string;
+  readonly prompt: Prompt<V>;
+  readonly tolerance: number;
+  /** The baseline file, as declared — what `x verify` reports when it cannot be read. */
+  readonly baseline: string;
+  /** Score every case against the baseline. Never throws on a regression — `assert` does that. */
+  run(gateway: Gateway): Promise<EvalResult>;
+  /**
+   * Run and throw `X_EVAL_THRESHOLD` on a regression. This is the line a test file calls, so a
+   * prompt edit that broke a case is a red test naming that case and both its scores.
+   *
+   * Under `ULTIMATE_EVAL_RECORD=1` it writes the baseline instead of gating on it.
+   */
+  assert(gateway: Gateway): Promise<EvalResult>;
+}
+
+/** What `x verify` reads to decide whether every prompt is evaluated. */
+export interface EvalFact {
+  readonly name: string;
+  readonly prompt: string;
+  readonly promptId: string;
+  readonly tolerance: number;
+  readonly baseline: string;
+}
+
+const registry = new Map<string, Eval>();
+
+export function defineEval<V extends PromptVars>(input: DefineEvalInput<V>): Eval<V> {
+  const evaluation: Eval<V> = {
+    name: input.name,
+    prompt: input.prompt,
+    tolerance: input.tolerance,
+    baseline: input.baseline,
+    run: (gateway) => runEval(input, gateway),
+    assert: (gateway) => assertEval(input, gateway),
+  };
+  registry.set(input.name, evaluation as Eval);
+  return evaluation;
+}
+
+export function describeEvals(): readonly EvalFact[] {
+  return [...registry.values()]
+    .map((evaluation) => ({
+      name: evaluation.name,
+      prompt: evaluation.prompt.ref,
+      promptId: evaluation.prompt.id,
+      tolerance: evaluation.tolerance,
+      baseline: evaluation.baseline,
+    }))
+    .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+}
+
+/**
+ * Every registered prompt no eval names, by ID rather than by version: old versions are retained
+ * so traces stay interpretable, and an eval on the current version evaluates that lineage.
+ */
+export function promptsWithoutEvals(): readonly Prompt[] {
+  const covered = new Set([...registry.values()].map((evaluation) => evaluation.prompt.id));
+  return describePrompts().filter((prompt) => !covered.has(prompt.id));
+}
+
+export function resetEvals(): void {
+  registry.clear();
+}
+
+async function assertEval<V extends PromptVars>(
+  input: DefineEvalInput<V>,
+  gateway: Gateway,
+): Promise<EvalResult> {
+  const path = baselinePath(input.baseline, input.name);
+  const result = await runEval(input, gateway);
+
+  if (recordingBaselines()) {
+    await writeBaseline(path, baselineFrom(result));
+    return { ...result, baseline: result.score, regressions: [], passed: true };
+  }
+  if (result.baseline === undefined) {
+    throw new EvalBaselineMissingError({
+      eval: input.name,
+      path,
+      reason: 'has never been recorded',
+    });
+  }
+  if (result.regressions.length > 0) {
+    throw new EvalThresholdError({
+      eval: result.name,
+      score: result.score,
+      baseline: result.baseline,
+      tolerance: result.tolerance,
+      promptVersion: `${result.promptRef} (${result.promptHash})`,
+      regressed: result.regressions.map(describeRegression),
+    });
+  }
+  return result;
+}
+
+async function runEval<V extends PromptVars>(
+  input: DefineEvalInput<V>,
+  gateway: Gateway,
+): Promise<EvalResult> {
+  const cases: CaseResult[] = [];
+  for (const testCase of input.cases) {
+    const generated = await gateway.generate({
+      messages: [{ role: 'user' as const, content: input.prompt.render(testCase.vars) }],
+      maxTokens: input.maxTokens ?? 1_024,
+      ...(input.prompt.system !== undefined ? { system: input.prompt.system } : {}),
+      ...(input.prompt.model !== undefined ? { model: input.prompt.model } : {}),
+      ...(input.prompt.effort !== undefined ? { effort: input.prompt.effort } : {}),
+    });
+    const perScorer: Record<string, number> = {};
+    for (const scorer of input.scorers) {
+      perScorer[scorer.name] = clampScore(
+        await scorer.score({
+          output: generated.text,
+          ...(testCase.expected !== undefined ? { expected: testCase.expected } : {}),
+        }),
+      );
+    }
+    cases.push({
+      case: testCase.name,
+      output: generated.text,
+      score: mean(Object.values(perScorer)),
+      perScorer,
+    });
+  }
+
+  const score = mean(cases.map((c) => c.score));
+  const recorded = await readBaseline(baselinePath(input.baseline, input.name));
+  const regressions =
+    recorded === undefined
+      ? []
+      : regressionsAgainst({
+          baseline: recorded,
+          score,
+          cases: caseScores(cases),
+          tolerance: input.tolerance,
+        });
+
+  return {
+    name: input.name,
+    promptRef: input.prompt.ref,
+    promptHash: input.prompt.hash,
+    score,
+    baseline: recorded?.score,
+    tolerance: input.tolerance,
+    regressions,
+    passed: recorded !== undefined && regressions.length === 0,
+    cases,
+  };
+}
+
+const caseScores = (cases: readonly CaseResult[]): Record<string, number> =>
+  Object.fromEntries(cases.map((c) => [c.case, c.score]));
+
+/** What a run records: the numbers, and the exact prompt that produced them. */
+export const baselineFrom = (result: EvalResult): EvalBaseline => ({
+  eval: result.name,
+  prompt: result.promptRef,
+  promptHash: result.promptHash,
+  score: result.score,
+  cases: caseScores(result.cases),
+});
+
+const mean = (values: readonly number[]): number =>
+  values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length;

package/src/gateway.ts

@@ -0,0 +1,203 @@
+// The LLM gateway: one entry point for every model call in an Ultimate app.
+//
+// Provider-agnostic (routes by model id), streaming, retrying with backoff on rate limits,
+// budgeted, and cost-accounted in integer minor units. Everything an app does with a model
+// goes through here, so budgets and accounting cannot be bypassed by a stray fetch.
+
+import type { Money } from '@ultimat3/money';
+import type { BudgetLimits, BudgetStore } from './budget';
+import { BudgetLedger, currentBudget, estimateSpend, withBudget } from './budget';
+import { AiProviderUnavailableError } from './errors';
+import type { ModelId } from './models';
+import { DEFAULT_MODEL } from './models';
+import type { GenerateRequest, GenerateResult, Provider, StreamChunk } from './provider';
+
+/**
+ * Response cache. Structurally satisfied by `@ultimat3/cache`'s memo/LRU tiers; declared as
+ * an interface so the gateway is testable with a `Map` and so caching stays optional.
+ */
+export interface GatewayCache {
+  get(key: string): Promise<string | undefined> | string | undefined;
+  set(key: string, value: string): Promise<void> | void;
+}
+
+export interface RetryPolicy {
+  /** Total attempts including the first. */
+  readonly attempts: number;
+  /** First backoff in ms; doubled per attempt with full jitter. */
+  readonly baseDelayMs: number;
+  readonly maxDelayMs: number;
+}
+
+export const DEFAULT_RETRY: RetryPolicy = { attempts: 3, baseDelayMs: 500, maxDelayMs: 8_000 };
+
+export interface CreateGatewayInput {
+  /** Tried in order for a given model. First provider that lists the model wins. */
+  readonly providers: readonly Provider[];
+  readonly budget?: BudgetLimits;
+  readonly budgetStore?: BudgetStore;
+  readonly cache?: GatewayCache;
+  readonly retry?: RetryPolicy;
+  readonly defaultModel?: ModelId;
+  /** Overridable so a test can run backoff without waiting. */
+  sleep?(ms: number): Promise<void>;
+}
+
+export interface Gateway {
+  generate(request: GenerateRequest): Promise<GenerateResult>;
+  stream(request: GenerateRequest): AsyncIterable<StreamChunk>;
+  /** Open a budget scope. Nested gateway calls inside `fn` share one ledger. */
+  scope<T>(input: { actorKey?: string; orgKey?: string }, fn: () => Promise<T>): Promise<T>;
+  /** Accumulated cost of the ambient scope, or zero outside one. */
+  spent(): Promise<Money>;
+}
+
+export function createGateway(input: CreateGatewayInput): Gateway {
+  return new GatewayImpl(input);
+}
+
+class GatewayImpl implements Gateway {
+  private readonly config: CreateGatewayInput;
+  private readonly retry: RetryPolicy;
+  private readonly sleep: (ms: number) => Promise<void>;
+
+  constructor(config: CreateGatewayInput) {
+    this.config = config;
+    this.retry = config.retry ?? DEFAULT_RETRY;
+    this.sleep = config.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+  }
+
+  scope<T>(input: { actorKey?: string; orgKey?: string }, fn: () => Promise<T>): Promise<T> {
+    const ledger = new BudgetLedger({
+      limits: this.config.budget ?? {},
+      ...(input.actorKey !== undefined ? { actorKey: input.actorKey } : {}),
+      ...(input.orgKey !== undefined ? { orgKey: input.orgKey } : {}),
+      ...(this.config.budgetStore !== undefined ? { store: this.config.budgetStore } : {}),
+    });
+    return withBudget(ledger, fn);
+  }
+
+  async spent(): Promise<Money> {
+    const ledger = currentBudget();
+    if (ledger === undefined) return { minor: 0, currency: 'USD' };
+    return (await ledger.report()).cost;
+  }
+
+  async generate(request: GenerateRequest): Promise<GenerateResult> {
+    const model = request.model ?? this.config.defaultModel ?? DEFAULT_MODEL;
+    const resolved: GenerateRequest = { ...request, model };
+
+    const cacheKey = cacheKeyFor(resolved);
+    const cached = await this.config.cache?.get(cacheKey);
+    if (cached !== undefined) {
+      // A cache hit costs nothing, so it is deliberately NOT debited from the budget.
+      return JSON.parse(cached) as GenerateResult;
+    }
+
+    // Reserve against the ESTIMATE before spending anything — tokens AND money, since a
+    // cheap-in-tokens call on an expensive model is still a cost cap the app declared.
+    // `record` below replaces the estimate with the provider's real counts.
+    const ledger = currentBudget();
+    await ledger?.reserve(estimateSpend(resolved));
+
+    const result = await this.attempt(model, (provider) => provider.generate(resolved));
+    await ledger?.record(result.usage, result.cost);
+    // A refusal is not an answer, so it is not cached. Storing one would keep serving a decision
+    // the classifier might not make twice, long after the prompt that provoked it was fixed.
+    if (result.stopReason !== 'refusal') {
+      await this.config.cache?.set(cacheKey, JSON.stringify(result));
+    }
+    return result;
+  }
+
+  async *stream(request: GenerateRequest): AsyncIterable<StreamChunk> {
+    const model = request.model ?? this.config.defaultModel ?? DEFAULT_MODEL;
+    const resolved: GenerateRequest = { ...request, model };
+    const ledger = currentBudget();
+    await ledger?.reserve(estimateSpend(resolved));
+
+    // A stream is not retried mid-flight: the consumer has already seen tokens, and
+    // replaying from the top would duplicate them. Only the handshake retries.
+    const provider = this.providerFor(model);
+    for await (const chunk of provider.stream(resolved)) {
+      if (chunk.type === 'done') await ledger?.record(chunk.result.usage, chunk.result.cost);
+      yield chunk;
+    }
+  }
+
+  private providerFor(model: ModelId): Provider {
+    const provider = this.config.providers.find((p) => p.models.includes(model));
+    if (provider === undefined) {
+      throw new AiProviderUnavailableError({
+        model,
+        attempts: this.config.providers.map((p) => `${p.name}: does not serve ${model}`),
+      });
+    }
+    return provider;
+  }
+
+  /**
+   * Try every provider that serves `model`, retrying each on a retryable failure with
+   * exponential backoff plus full jitter (jitter matters: synchronised retries from N
+   * workers reproduce the rate limit they are backing off from).
+   */
+  private async attempt<T>(model: ModelId, call: (provider: Provider) => Promise<T>): Promise<T> {
+    const candidates = this.config.providers.filter((p) => p.models.includes(model));
+    const failures: string[] = [];
+    if (candidates.length === 0) {
+      throw new AiProviderUnavailableError({ model, attempts: [`no provider serves ${model}`] });
+    }
+
+    for (const provider of candidates) {
+      for (let attempt = 1; attempt <= this.retry.attempts; attempt += 1) {
+        try {
+          return await call(provider);
+        } catch (error) {
+          failures.push(`${provider.name}#${attempt}: ${messageOf(error)}`);
+          if (!isRetryable(error) || attempt === this.retry.attempts) break;
+          await this.sleep(backoffMs(this.retry, attempt));
+        }
+      }
+    }
+    throw new AiProviderUnavailableError({ model, attempts: failures });
+  }
+}
+
+/** Full jitter: a uniform pick from [0, exponential], capped. */
+export function backoffMs(policy: RetryPolicy, attempt: number): number {
+  const ceiling = Math.min(policy.baseDelayMs * 2 ** (attempt - 1), policy.maxDelayMs);
+  return Math.floor(Math.random() * ceiling);
+}
+
+/**
+ * Retryable = the request was well formed and the provider was momentarily unable. A 400 is
+ * never retried: the same body produces the same rejection and only burns the budget.
+ */
+export function isRetryable(error: unknown): boolean {
+  if (typeof error !== 'object' || error === null) return false;
+  const e = error as { status?: unknown; code?: unknown };
+  if (typeof e.status === 'number') return e.status === 429 || e.status >= 500;
+  return e.code === 'ETIMEDOUT' || e.code === 'ECONNRESET';
+}
+
+function messageOf(error: unknown): string {
+  if (error instanceof Error) return error.message;
+  return String(error);
+}
+
+/**
+ * Cache key. Every field that changes the answer is in it — a key that ignores `effort` or
+ * `system` would serve one prompt's answer for another.
+ */
+export function cacheKeyFor(request: GenerateRequest): string {
+  return JSON.stringify({
+    model: request.model,
+    system: request.system ?? '',
+    messages: request.messages,
+    maxTokens: request.maxTokens,
+    effort: request.effort ?? 'high',
+    thinking: request.thinking ?? 'adaptive',
+    tools: request.tools?.map((t) => t.name) ?? [],
+    stop: request.stopSequences ?? [],
+  });
+}

package/src/index.ts

@@ -0,0 +1,187 @@
+// Public API of @ultimat3/ai. Explicit — a wildcard barrel would leak internals an app
+// could depend on, and the gateway's guarantees only hold if every call goes through it.
+
+/** Re-exported so an `llm` file needs one import, not two. Same object as schema's. */
+export type { Infer } from '@ultimat3/schema';
+export { t } from '@ultimat3/schema';
+export type {
+  BudgetLedgerInput,
+  BudgetLimits,
+  BudgetReport,
+  BudgetStore,
+  SpendEstimate,
+} from './budget';
+export {
+  BudgetLedger,
+  currentBudget,
+  estimateSpend,
+  MemoryBudgetStore,
+  withBudget,
+} from './budget';
+export type { Embedder, HashEmbedderInput } from './embeddings';
+export {
+  cosine,
+  embedBatched,
+  embedOne,
+  fnv1a,
+  HashEmbedder,
+  normalize,
+  tokenize,
+} from './embeddings';
+export type { AiErrorCode } from './errors';
+export {
+  AI_ERROR_CODES,
+  AI_ERROR_TITLES,
+  AiBudgetExceededError,
+  AiGatewayMissingError,
+  AiKeyMissingError,
+  AiPromptRenderError,
+  AiPromptVersionError,
+  AiProviderUnavailableError,
+  AiRequestInvalidError,
+  AiTransportError,
+  EmbedderDimMismatchError,
+  EvalBaselineInvalidError,
+  EvalBaselineMissingError,
+  EvalMissingError,
+  EvalRecordingError,
+  EvalThresholdError,
+  LlmOutputInvalidError,
+  LlmRefusedError,
+  LlmTruncatedError,
+  VectorDimMismatchError,
+  VectorScopeWidenedError,
+} from './errors';
+export type { EvalBaseline, Regression } from './eval-baseline';
+export {
+  baselinePath,
+  describeRegression,
+  OVERALL,
+  RECORD_ENV,
+  readBaseline,
+  recordingBaselines,
+  regressionsAgainst,
+  writeBaseline,
+} from './eval-baseline';
+export type {
+  CaseResult,
+  DefineEvalInput,
+  Eval,
+  EvalCase,
+  EvalFact,
+  EvalResult,
+} from './evals';
+export {
+  baselineFrom,
+  defineEval,
+  describeEvals,
+  promptsWithoutEvals,
+  resetEvals,
+} from './evals';
+export type { CreateGatewayInput, Gateway, GatewayCache, RetryPolicy } from './gateway';
+export { backoffMs, cacheKeyFor, createGateway, DEFAULT_RETRY, isRetryable } from './gateway';
+export type {
+  LlmBudget,
+  LlmCache,
+  LlmDef,
+  LlmSemanticCache,
+  LlmVarsArgs,
+} from './llm';
+export { llm } from './llm';
+export type { Effort, ModelId, ModelReasoning, ModelSpec, ThinkingMode } from './models';
+export { DEFAULT_MODEL, EFFORTS, MODEL_IDS, MODELS, reasoningBody } from './models';
+export type { PgVectorStoreInput } from './pg-vector';
+export { PgVectorStore } from './pg-vector';
+export type {
+  PgHybridArgs,
+  PgSearchArgs,
+  PgVectorRowInput,
+  PgVectorTable,
+} from './pg-vector-sql';
+export {
+  conditionsSql,
+  ddlSql,
+  deleteSql,
+  hybridSql,
+  searchSql,
+  textSql,
+  upsertSql,
+  vectorLiteral,
+} from './pg-vector-sql';
+export type { DefinePromptInput, Prompt, PromptVars } from './prompt';
+export {
+  contentHash,
+  definePrompt,
+  describePrompts,
+  getPrompt,
+  promptVersions,
+  resetPrompts,
+} from './prompt';
+export type {
+  AiMessage,
+  AnthropicProviderInput,
+  EchoProviderInput,
+  GenerateRequest,
+  GenerateResult,
+  Provider,
+  StopDetails,
+  StopReason,
+  StreamChunk,
+  TokenUsage,
+} from './provider';
+export {
+  AnthropicProvider,
+  costOf,
+  EchoProvider,
+  estimateCost,
+  estimateInputTokens,
+  estimateTextTokens,
+  estimateTokens,
+  parseMessage,
+  requiresStreaming,
+  STREAM_ONLY_MAX_TOKENS,
+  totalTokens,
+} from './provider';
+export type {
+  AssembledContext,
+  Chunk,
+  ChunkInput,
+  Reranker,
+  RetrieveInput,
+} from './rag';
+export { assembleContext, chunk, indexDocument, passthroughReranker, retrieve } from './rag';
+export type { RemoteEmbedderInput } from './remote-embedder';
+export { RemoteEmbedder } from './remote-embedder';
+export type { AiRuntimeInput } from './runtime';
+export { aiEmbedder, aiGateway, configureAi, resetAiRuntime, semanticCacheFor } from './runtime';
+export type { Scorer } from './scorers';
+export {
+  contains,
+  exact,
+  jsonSchemaValid,
+  jsonValid,
+  llmJudge,
+  numericTolerance,
+} from './scorers';
+export type {
+  JsonSchema,
+  LlmTool,
+  LlmToolCall,
+  LlmToolResult,
+  ProjectableAction,
+} from './tools';
+export { runLlmToolCall, toLlmTool, toLlmTools } from './tools';
+export type {
+  HybridSearchInput,
+  MemoryVectorStoreInput,
+  MetadataFilter,
+  SearchHit,
+  StoredRecord,
+  VectorRecord,
+  VectorStore,
+} from './vector';
+export { fuse, MemoryVectorStore } from './vector';
+export type { VectorScope } from './vector-scope';
+export { NO_TENANT, narrowScope, scopeAdmits, tenantOf, UNSCOPED } from './vector-scope';
+export type { StreamState } from './wire';
+export { parseStopDetails, ZERO_USAGE } from './wire';