language-models 2.1.3 → 2.4.0

package/src/policy.ts

@@ -0,0 +1,343 @@
+/**
+ * ModelPolicy - per-model resilience and tier policy data
+ *
+ * `language-models` owns model identity (alias resolution, capability lookup).
+ * Resilience policy (retry, circuit breaker, fallback chain, batch tier) is
+ * a per-model concern that belongs alongside the catalog data — but the
+ * runtime *machinery* that applies the policy lives in `ai-functions`.
+ *
+ * This module provides:
+ * - `ModelPolicy` MDXLD type (`$type: 'ModelPolicy'`)
+ * - `policyFor(alias)` - resolve an alias and return its derived policy
+ * - `derivePolicy(model, alias?)` - inference layer that turns OpenRouter raw
+ *   data into a policy by applying heuristics (newest frontier model is best,
+ *   price within a family is inversely correlated with capability, etc.)
+ *
+ * Source of truth: OpenRouter raw catalog (data/models.json) + heuristics.
+ * Strategy: runtime derivation, cached. A static snapshot generator could be
+ * added later (see `derivePolicy` — it's pure, so you can pre-compute it).
+ *
+ * @packageDocumentation
+ */
+
+import { resolve, get, list, type ModelInfo } from './models.js'
+import { ALIASES } from './aliases.js'
+
+// ============================================================================
+// MDXLD types
+// ============================================================================
+
+/**
+ * Error category taxonomy. Mirrors `ai-functions`'s `ErrorCategory` enum by
+ * string value — we don't import it (circular), but the strings line up.
+ */
+export type ErrorCategoryName =
+  | 'network'
+  | 'rate_limit'
+  | 'invalid_input'
+  | 'authentication'
+  | 'server'
+  | 'context_length'
+  | 'unknown'
+
+/**
+ * Retry classification: which error categories are retryable for this model,
+ * plus backoff parameters.
+ */
+export interface RetryPolicyData {
+  maxRetries: number
+  baseDelay: number
+  maxDelay: number
+  multiplier: number
+  jitter: number
+  /** Categories that should trigger a retry */
+  retryableCategories: ErrorCategoryName[]
+}
+
+/**
+ * Circuit-breaker policy data (per-model state keys come from the alias).
+ */
+export interface CircuitBreakerPolicyData {
+  failureThreshold: number
+  resetTimeout: number
+  successThreshold: number
+}
+
+/**
+ * Tiers a model is eligible for.
+ *
+ * - `immediate`: synchronous online inference (always available)
+ * - `flex`: faster-than-batch processing (~minutes, ~50% discount)
+ *   — only OpenAI/Bedrock currently
+ * - `batch`: batch API processing (~hours, ~50% discount)
+ *   — OpenAI/Anthropic/Google/Bedrock/Cloudflare
+ */
+export type BatchTier = 'immediate' | 'flex' | 'batch'
+
+/**
+ * Provider-specific HTTP status code → ErrorCategory mapping.
+ * Empty here by default — `ai-functions/retry.ts#classifyError` handles the
+ * common cases. Override per-model if a provider has unusual error codes.
+ */
+export type ErrorMapping = Record<number, ErrorCategoryName>
+
+/**
+ * MDXLD-shaped per-model resilience and tier policy.
+ *
+ * `$type: 'ModelPolicy'`, `$id` is the resolved model id (e.g.
+ * `'anthropic/claude-opus-4.5'`).
+ */
+export interface ModelPolicy {
+  $type: 'ModelPolicy'
+  $id: string
+  /** Provider slug (e.g. 'anthropic') */
+  provider: string
+  retry: RetryPolicyData
+  circuitBreaker: CircuitBreakerPolicyData
+  /** Ordered list of model ids to try after this one fails */
+  fallbackChain: string[]
+  /** Tiers this model is eligible for */
+  batchTier: BatchTier[]
+  /** Provider-specific HTTP code → category overrides */
+  errorMapping: ErrorMapping
+}
+
+// ============================================================================
+// Defaults & heuristics
+// ============================================================================
+
+/** Frontier labs — newer releases tend to be more capable. */
+const FRONTIER_PROVIDERS = new Set(['anthropic', 'openai', 'google'])
+
+/** Providers with batch APIs supported by ai-functions. */
+const BATCH_PROVIDERS = new Set(['anthropic', 'openai', 'google', 'amazon-bedrock', 'cloudflare'])
+
+/** Providers with flex (faster-than-batch) APIs. */
+const FLEX_PROVIDERS = new Set(['openai', 'amazon-bedrock'])
+
+/** Curated fallback seeds — picked one per frontier family. */
+const FRONTIER_FALLBACK: readonly string[] = [
+  'anthropic/claude-sonnet-4.5',
+  'anthropic/claude-opus-4.5',
+  'openai/gpt-4o',
+  'google/gemini-2.5-pro',
+]
+
+/** Default retry policy — matches `ai-functions` `RetryPolicy` defaults. */
+export const DEFAULT_RETRY: RetryPolicyData = {
+  maxRetries: 3,
+  baseDelay: 1000,
+  maxDelay: 30000,
+  multiplier: 2,
+  jitter: 0,
+  retryableCategories: ['network', 'rate_limit', 'server', 'unknown'],
+}
+
+/** Default circuit breaker — matches `ai-functions` defaults. */
+export const DEFAULT_CIRCUIT_BREAKER: CircuitBreakerPolicyData = {
+  failureThreshold: 5,
+  resetTimeout: 30000,
+  successThreshold: 1,
+}
+
+/**
+ * Default policy for an unknown model. Used when no catalog entry is found.
+ */
+export function defaultPolicy(modelId: string): ModelPolicy {
+  const provider = modelId.includes('/') ? modelId.split('/')[0]! : 'unknown'
+  return {
+    $type: 'ModelPolicy',
+    $id: modelId,
+    provider,
+    retry: { ...DEFAULT_RETRY },
+    circuitBreaker: { ...DEFAULT_CIRCUIT_BREAKER },
+    fallbackChain: [],
+    batchTier: ['immediate'],
+    errorMapping: {},
+  }
+}
+
+// ============================================================================
+// Derivation layer
+// ============================================================================
+
+/**
+ * Parse a price-per-token string to a number. Returns 0 on parse failure.
+ * Pricing comes through OpenRouter as a string (e.g. "0.000003").
+ */
+function parsePrice(p?: string): number {
+  if (!p) return 0
+  const n = Number(p)
+  return Number.isFinite(n) ? n : 0
+}
+
+/**
+ * Extract a "family" key from a model id for fallback grouping.
+ * 'anthropic/claude-opus-4.5' → 'anthropic/claude'
+ * 'openai/gpt-4o-mini' → 'openai/gpt'
+ */
+function familyKey(id: string): string {
+  const slash = id.indexOf('/')
+  if (slash < 0) return id
+  const provider = id.substring(0, slash)
+  const rest = id.substring(slash + 1).toLowerCase()
+  // Strip trailing version tags / size qualifiers
+  const family = rest
+    .replace(/[-_]?\d.*$/, '') // drop -4.5, -2.0, etc
+    .replace(/(opus|sonnet|haiku|mini|pro|flash|lite|maverick|instruct).*$/, '$1')
+    .replace(/-?(opus|sonnet|haiku|mini|pro|flash|lite|maverick|instruct)$/, '')
+  return `${provider}/${family || rest.split('-')[0]}`
+}
+
+/**
+ * Derive the fallback chain for a model.
+ *
+ * Heuristics:
+ * 1. Prefer same-family siblings (e.g. sonnet → opus → haiku within Claude)
+ *    ordered by `created` (newer first), then by price descending (more
+ *    expensive within a family is usually more capable).
+ * 2. Then fall back to frontier-lab seeds, skipping the model itself and
+ *    anything from the same family already included.
+ * 3. Cap at 4 entries to keep latency bounded.
+ */
+function deriveFallbackChain(model: ModelInfo, allModels: ModelInfo[]): string[] {
+  const chain: string[] = []
+  const seen = new Set<string>([model.id])
+  const fam = familyKey(model.id)
+
+  // Step 1: same-family siblings, sorted newest-first then by price desc.
+  const siblings = allModels
+    .filter((m) => m.id !== model.id && familyKey(m.id) === fam)
+    .map((m) => ({
+      m,
+      created: (m as ModelInfo & { created?: number }).created ?? 0,
+      price: parsePrice(m.pricing?.completion),
+    }))
+    .sort((a, b) => {
+      if (b.created !== a.created) return b.created - a.created
+      return b.price - a.price
+    })
+
+  for (const { m } of siblings.slice(0, 2)) {
+    if (!seen.has(m.id)) {
+      chain.push(m.id)
+      seen.add(m.id)
+    }
+  }
+
+  // Step 2: frontier seeds.
+  for (const seed of FRONTIER_FALLBACK) {
+    if (chain.length >= 4) break
+    if (seen.has(seed)) continue
+    if (familyKey(seed) === fam) continue
+    chain.push(seed)
+    seen.add(seed)
+  }
+
+  return chain
+}
+
+/**
+ * Derive batch-tier eligibility from provider capability.
+ */
+function deriveBatchTiers(provider: string): BatchTier[] {
+  const tiers: BatchTier[] = ['immediate']
+  if (FLEX_PROVIDERS.has(provider)) tiers.push('flex')
+  if (BATCH_PROVIDERS.has(provider)) tiers.push('batch')
+  return tiers
+}
+
+/**
+ * Derive the retry policy. Frontier providers get one extra attempt because
+ * their rate limits are typically more transient than long-tail providers.
+ */
+function deriveRetry(provider: string): RetryPolicyData {
+  if (FRONTIER_PROVIDERS.has(provider)) {
+    return { ...DEFAULT_RETRY, maxRetries: 4, jitter: 0.2 }
+  }
+  return { ...DEFAULT_RETRY }
+}
+
+/**
+ * Derive the circuit-breaker policy. Frontier providers get a higher
+ * failure threshold (more capacity) and a shorter reset timeout.
+ */
+function deriveCircuitBreaker(provider: string): CircuitBreakerPolicyData {
+  if (FRONTIER_PROVIDERS.has(provider)) {
+    return { failureThreshold: 8, resetTimeout: 20000, successThreshold: 1 }
+  }
+  return { ...DEFAULT_CIRCUIT_BREAKER }
+}
+
+/**
+ * Derivation layer — turn a `ModelInfo` (from OpenRouter raw data) into a
+ * `ModelPolicy` by applying heuristics.
+ *
+ * Pure function; safe to call at build time to pre-compute a static snapshot.
+ *
+ * @param model - The catalog entry for the model.
+ * @param allModels - The full catalog, used for sibling lookup. Optional;
+ *   defaults to `list()`.
+ */
+export function derivePolicy(model: ModelInfo, allModels?: ModelInfo[]): ModelPolicy {
+  const all = allModels ?? list()
+  const slash = model.id.indexOf('/')
+  const provider = slash > 0 ? model.id.substring(0, slash) : model.provider ?? 'unknown'
+
+  return {
+    $type: 'ModelPolicy',
+    $id: model.id,
+    provider,
+    retry: deriveRetry(provider),
+    circuitBreaker: deriveCircuitBreaker(provider),
+    fallbackChain: deriveFallbackChain(model, all),
+    batchTier: deriveBatchTiers(provider),
+    errorMapping: {},
+  }
+}
+
+// ============================================================================
+// Public API
+// ============================================================================
+
+/** Per-process cache of derived policies, keyed by resolved model id. */
+const policyCache = new Map<string, ModelPolicy>()
+
+/**
+ * Resolve an alias (or full model id) and return its policy.
+ *
+ * Falls back to `defaultPolicy(id)` if the model is not in the catalog —
+ * callers always get a usable policy.
+ *
+ * @example
+ * ```ts
+ * const p = policyFor('sonnet')
+ * // p.fallbackChain → ['anthropic/claude-opus-4.5', 'openai/gpt-4o', ...]
+ * // p.batchTier     → ['immediate', 'batch']
+ * ```
+ */
+export function policyFor(input: string): ModelPolicy {
+  const id = resolve(input)
+  const cached = policyCache.get(id)
+  if (cached) return cached
+
+  const model = get(id)
+  const policy = model ? derivePolicy(model) : defaultPolicy(id)
+  policyCache.set(id, policy)
+  return policy
+}
+
+/**
+ * Reset the policy cache. Useful for tests, or after the catalog is reloaded.
+ */
+export function resetPolicyCache(): void {
+  policyCache.clear()
+}
+
+/**
+ * List all known aliases. Convenience for tooling that wants to enumerate
+ * derived policies (e.g. a static snapshot generator).
+ */
+export function listAliases(): string[] {
+  return Object.keys(ALIASES)
+}

package/src/pricing/index.ts

@@ -0,0 +1,29 @@
+/**
+ * language-models / pricing — canonical LLM model pricing table.
+ *
+ * Consume via the subpath export:
+ *
+ *   import { PRICING_TABLE, priceFor, type ModelPricing } from 'language-models/pricing'
+ *
+ * (Equivalent symbols are also re-exported from the package root —
+ * `import { priceFor } from 'language-models'` works too.)
+ *
+ * Rates are sourced from public Vertex / Bedrock / AI Studio list prices.
+ * `priceFor()` throws on unknown slug/tier rather than returning a silent
+ * zero (per BYOK_GATEWAY_LIES discipline: loud failure beats silent
+ * downgrade).
+ */
+
+export type {
+  HasPricingArgs,
+  ModelPricing,
+  PriceForArgs,
+  PriceForResult,
+  PricingTier,
+  Provider,
+  RateBlock,
+} from './types.js'
+
+export { PRICING_TABLE } from './table.js'
+
+export { priceFor, listSlugs, hasPricing, rowsForSlug } from './lookup.js'

package/src/pricing/lookup.ts

@@ -0,0 +1,124 @@
+/**
+ * language-models / pricing — lookup helpers.
+ *
+ * The single source of truth for cost computation across the Phase-2
+ * three-repo cascade. Throws on unknown slug/tier (per BYOK_GATEWAY_LIES
+ * memory: silent zero is the lying-gateway pattern; loud failure beats
+ * silent downgrade).
+ */
+
+import { PRICING_TABLE } from './table.js'
+import type {
+  HasPricingArgs,
+  ModelPricing,
+  PriceForArgs,
+  PriceForResult,
+  PricingTier,
+} from './types.js'
+
+/**
+ * Compute USD cost for a known generation. Throws on:
+ * - unknown slug (no rows for the slug at all)
+ * - unknown/unmodeled tier for the slug (no row for the (slug, tier) pair)
+ * - negative token counts (programming error — fail loud)
+ *
+ * Honors `contextTierBreakpoint` when present: if `inputTokens >=
+ * breakpoint`, the entire request is billed at `contextTierAbove` rates
+ * (matches Google's published billing model — the rate switches once the
+ * input crosses 200K, applied to the full request).
+ *
+ * `cachedInputTokens` is billed at `cachedInputPer1M` when defined,
+ * falling back to the regular input rate otherwise. Cached tokens are
+ * SUBTRACTED from `inputTokens` to compute the non-cached portion — i.e.
+ * `inputTokens` is the TOTAL input including any cached tokens.
+ */
+export function priceFor(args: PriceForArgs): PriceForResult {
+  const { slug, tier, inputTokens, outputTokens, cachedInputTokens } = args
+
+  if (inputTokens < 0 || outputTokens < 0 || (cachedInputTokens ?? 0) < 0) {
+    throw new RangeError(
+      `priceFor() requires non-negative token counts; got input=${inputTokens}, output=${outputTokens}, cachedInput=${
+        cachedInputTokens ?? 0
+      }`
+    )
+  }
+
+  const row = findRow(slug, tier)
+
+  // Pick the right rate block based on context-tier breakpoint.
+  const useAbove =
+    typeof row.contextTierBreakpoint === 'number' &&
+    row.contextTierAbove !== undefined &&
+    inputTokens >= row.contextTierBreakpoint
+  const block = useAbove
+    ? (row.contextTierAbove as NonNullable<typeof row.contextTierAbove>)
+    : {
+        inputPer1M: row.inputPer1M,
+        outputPer1M: row.outputPer1M,
+        cachedInputPer1M: row.cachedInputPer1M,
+      }
+
+  const cachedTok = Math.min(cachedInputTokens ?? 0, inputTokens)
+  const nonCachedInputTok = inputTokens - cachedTok
+  const cachedRate = block.cachedInputPer1M ?? block.inputPer1M
+
+  const inputUsd =
+    (nonCachedInputTok / 1_000_000) * block.inputPer1M + (cachedTok / 1_000_000) * cachedRate
+  const outputUsd = (outputTokens / 1_000_000) * block.outputPer1M
+  const totalUsd = inputUsd + outputUsd
+
+  return { inputUsd, outputUsd, totalUsd }
+}
+
+/**
+ * Returns the unique set of slugs in the table. Useful for adapters that
+ * want to validate caller-supplied model ids before dispatching.
+ */
+export function listSlugs(): readonly string[] {
+  const set = new Set<string>()
+  for (const row of PRICING_TABLE) set.add(row.slug)
+  return Array.from(set)
+}
+
+/**
+ * Returns true if pricing exists for the given (slug, tier). Use this
+ * when you need a non-throwing existence check before calling
+ * `priceFor()` (e.g. a metering middleware that wants to fall back to
+ * "unknown cost" telemetry rather than throwing on a not-yet-registered
+ * model).
+ */
+export function hasPricing(args: HasPricingArgs): boolean {
+  return PRICING_TABLE.some((row) => row.slug === args.slug && row.tier === args.tier)
+}
+
+/**
+ * Returns all pricing rows for a slug, across all tiers. Useful for
+ * tooling that wants to display "this model has standard + batch tiers"
+ * in a UI.
+ */
+export function rowsForSlug(slug: string): readonly ModelPricing[] {
+  return PRICING_TABLE.filter((row) => row.slug === slug)
+}
+
+// ---------------------------------------------------------------------------
+// Internal
+// ---------------------------------------------------------------------------
+
+function findRow(slug: string, tier: PricingTier): ModelPricing {
+  const slugRows = PRICING_TABLE.filter((row) => row.slug === slug)
+  if (slugRows.length === 0) {
+    throw new Error(
+      `Unknown model slug: '${slug}'. Known slugs: ${listSlugs().slice(0, 8).join(', ')}${
+        listSlugs().length > 8 ? `, ...(${listSlugs().length - 8} more)` : ''
+      }`
+    )
+  }
+  const tierRow = slugRows.find((row) => row.tier === tier)
+  if (tierRow === undefined) {
+    const availableTiers = slugRows.map((row) => row.tier)
+    throw new Error(
+      `No '${tier}' pricing for slug '${slug}'. Available tiers: ${availableTiers.join(', ')}`
+    )
+  }
+  return tierRow
+}