@latimer-woods-tech/llm 0.2.0 → 0.4.1

package/dist/index.mjs

@@ -1,235 +1,760 @@
 // src/index.ts
 import {
-  ErrorCodes,
   InternalError,
   RateLimitError,
-  ValidationError
+  ValidationError,
+  toErrorResponse
 } from "@latimer-woods-tech/errors";
+var MODELS = {
+  anthropic: {
+    fast: "claude-haiku-4-20250514",
+    balanced: "claude-sonnet-4-6",
+    smart: "claude-opus-4-7"
+  },
+  gemini: {
+    smart: "gemini-2.5-pro"
+  },
+  groq: {
+    verifier: "llama-4-maverick"
+  },
+  grok: {
+    fast: "grok-4.3"
+  },
+  deepseek: {
+    workbench: "deepseek-chat"
+  }
+};
 var DEFAULT_MAX_TOKENS = 1024;
 var DEFAULT_TEMPERATURE = 0.7;
-var DEFAULT_ANTHROPIC_MODEL = "claude-sonnet-4-20250514";
-var DEFAULT_GROK_MODEL = "grok-3-fast";
-var DEFAULT_GROQ_MODEL = "llama-3.3-70b-versatile";
-function isFailover(status) {
-  return status === 429 || status >= 500;
+var DEFAULT_LONG_CONTEXT_THRESHOLD = 15e4;
+var BACKOFF_BASE_MS = 500;
+var BACKOFF_CAP_MS = 8e3;
+var BACKOFF_JITTER_MAX_MS = 250;
+var PER_PROVIDER_MAX_ATTEMPTS = 3;
+var providerCooldownUntil = /* @__PURE__ */ new Map();
+var PROVIDER_COOLDOWN_MS = 3e4;
+function isProviderCoolingDown(provider, now = Date.now) {
+  const until = providerCooldownUntil.get(provider);
+  if (until === void 0) return false;
+  return now() < until;
+}
+function isoDate(nowMs) {
+  return new Date(nowMs).toISOString().slice(0, 10);
+}
+async function recordOrgCostUsage(kv, todayKey, monthKey, costUsd, opts) {
+  if (opts.dailyCapUsd !== void 0) {
+    const raw = await kv.get(todayKey).catch(() => null);
+    const spent = parseFloat(raw ?? "0");
+    await kv.put(todayKey, String(spent + costUsd), {
+      expirationTtl: 172800
+      /* 48 h */
+    }).catch(() => void 0);
+  }
+  if (opts.monthlyCapUsd !== void 0) {
+    const raw = await kv.get(monthKey).catch(() => null);
+    const spent = parseFloat(raw ?? "0");
+    await kv.put(monthKey, String(spent + costUsd), {
+      expirationTtl: 3456e3
+      /* 40 d */
+    }).catch(() => void 0);
+  }
+}
+var MODEL_PRICE_PER_1M = {
+  // Anthropic Haiku 4
+  "claude-haiku-4-20250514": { input: 0.8, output: 4, cacheRead: 0.08, cacheWrite: 1 },
+  "claude-haiku-4-5-20251001": { input: 0.8, output: 4, cacheRead: 0.08, cacheWrite: 1 },
+  // Anthropic Sonnet 4
+  "claude-sonnet-4-20250514": { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 },
+  "claude-sonnet-4-6": { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 },
+  // Anthropic Opus 4
+  "claude-opus-4-20250514": { input: 15, output: 75, cacheRead: 1.5, cacheWrite: 18.75 },
+  "claude-opus-4-7": { input: 15, output: 75, cacheRead: 1.5, cacheWrite: 18.75 },
+  // Gemini 2.5 Pro
+  "gemini-2.5-pro": { input: 1.25, output: 10, cacheRead: 0.31, cacheWrite: 4.5 },
+  // Groq Llama 4 Maverick
+  "llama-4-maverick": { input: 0.5, output: 0.77, cacheRead: 0.05, cacheWrite: 0.5 },
+  // Grok 4.3
+  "grok-4.3": { input: 1.25, output: 2.5, cacheRead: 0, cacheWrite: 0 },
+  // DeepSeek API pricing as of 2026-05: cache-write conservatively uses cache-miss input pricing.
+  "deepseek-chat": { input: 0.27, output: 1.1, cacheRead: 0.07, cacheWrite: 0.27 },
+  "deepseek-reasoner": { input: 0.55, output: 2.19, cacheRead: 0.14, cacheWrite: 0.55 },
+  // Deprecated aliases retained for historical ledger rows.
+  "grok-4-fast": { input: 1.25, output: 2.5, cacheRead: 0, cacheWrite: 0 },
+  "grok-3-mini-latest": { input: 1.25, output: 2.5, cacheRead: 0, cacheWrite: 0 }
+};
+var PRICE_FALLBACK = MODEL_PRICE_PER_1M["claude-opus-4-7"];
+function estimateCostUsd(tokens, model) {
+  const price = MODEL_PRICE_PER_1M[model] ?? PRICE_FALLBACK;
+  return (tokens.input * price.input + tokens.output * price.output + (tokens.cacheRead ?? 0) * price.cacheRead + (tokens.cacheWrite ?? 0) * price.cacheWrite) / 1e6;
+}
+function isoMonth(nowMs) {
+  return new Date(nowMs).toISOString().slice(0, 7);
+}
+function markProviderCoolingDown(provider, now = Date.now) {
+  providerCooldownUntil.set(provider, now() + PROVIDER_COOLDOWN_MS);
+}
+function clearProviderCooldown(provider) {
+  providerCooldownUntil.delete(provider);
+}
+var BASE_BACKOFF_MS = 250;
+function isRetryableForBackoff(status) {
+  return status === 429 || status >= 500 && status < 600;
+}
+function estimateTokens(messages, system) {
+  let chars = system?.length ?? 0;
+  for (const m of messages) chars += m.content.length;
+  return Math.ceil(chars / 4);
+}
+function sleep(ms, signal) {
+  return new Promise((resolve, reject) => {
+    const t = setTimeout(resolve, ms);
+    if (signal) {
+      const onAbort = () => {
+        clearTimeout(t);
+        reject(new DOMException("Aborted", "AbortError"));
+      };
+      if (signal.aborted) onAbort();
+      else signal.addEventListener("abort", onAbort, { once: true });
+    }
+  });
 }
-function buildAnthropicBody(messages, opts, system) {
+function computeBackoffMs(attempt) {
+  const jitter = Math.floor(Math.random() * BACKOFF_JITTER_MAX_MS);
+  return Math.min(BACKOFF_BASE_MS * Math.pow(2, attempt) + jitter, BACKOFF_CAP_MS);
+}
+function buildAnthropicRequest(model, messages, opts, env, streaming = false) {
+  const sys = opts.system ?? messages.find((m) => m.role === "system")?.content;
   const filtered = messages.filter((m) => m.role !== "system");
-  const sys = system ?? messages.find((m) => m.role === "system")?.content;
   const body = {
-    model: opts.model ?? DEFAULT_ANTHROPIC_MODEL,
+    model,
     max_tokens: opts.maxTokens ?? DEFAULT_MAX_TOKENS,
     temperature: opts.temperature ?? DEFAULT_TEMPERATURE,
     messages: filtered.map((m) => ({ role: m.role, content: m.content }))
   };
+  if (streaming) {
+    body.stream = true;
+  }
   if (sys) {
-    body.system = sys;
+    const cache = opts.promptCache ?? sys.length >= 4096;
+    body.system = cache ? [{ type: "text", text: sys, cache_control: { type: "ephemeral" } }] : sys;
   }
-  return JSON.stringify(body);
+  return {
+    url: `${env.AI_GATEWAY_BASE_URL}/anthropic/v1/messages`,
+    headers: {
+      "content-type": "application/json",
+      "x-api-key": env.ANTHROPIC_API_KEY,
+      "anthropic-version": "2023-06-01",
+      "anthropic-beta": "prompt-caching-2024-07-31"
+    },
+    body: JSON.stringify(body)
+  };
 }
-function buildOpenAIBody(model, messages, opts, system) {
-  const sys = system ?? messages.find((m) => m.role === "system")?.content;
-  const merged = [];
+function buildGeminiRequest(model, messages, opts, env) {
+  const sys = opts.system ?? messages.find((m) => m.role === "system")?.content;
+  const contents = messages.filter((m) => m.role !== "system").map((m) => ({
+    role: m.role === "assistant" ? "model" : "user",
+    parts: [{ text: m.content }]
+  }));
+  const body = {
+    contents,
+    generationConfig: {
+      maxOutputTokens: opts.maxTokens ?? DEFAULT_MAX_TOKENS,
+      temperature: opts.temperature ?? DEFAULT_TEMPERATURE
+    }
+  };
   if (sys) {
-    merged.push({ role: "system", content: sys });
+    body.systemInstruction = { parts: [{ text: sys }] };
   }
-  for (const m of messages) {
-    if (m.role !== "system") {
-      merged.push(m);
-    }
+  const path = `v1/projects/${env.VERTEX_PROJECT}/locations/${env.VERTEX_LOCATION}/publishers/google/models/${model}:generateContent`;
+  return {
+    url: `${env.AI_GATEWAY_BASE_URL}/google-vertex-ai/${path}`,
+    headers: {
+      "content-type": "application/json",
+      authorization: `Bearer ${env.VERTEX_ACCESS_TOKEN}`
+    },
+    body: JSON.stringify(body)
+  };
+}
+function buildGroqRequest(model, messages, opts, env) {
+  const sys = opts.system ?? messages.find((m) => m.role === "system")?.content;
+  const merged = [];
+  if (sys) merged.push({ role: "system", content: sys });
+  for (const m of messages) if (m.role !== "system") merged.push(m);
+  return {
+    url: `${env.AI_GATEWAY_BASE_URL}/groq/openai/v1/chat/completions`,
+    headers: {
+      "content-type": "application/json",
+      authorization: `Bearer ${env.GROQ_API_KEY}`
+    },
+    body: JSON.stringify({
+      model,
+      max_tokens: opts.maxTokens ?? DEFAULT_MAX_TOKENS,
+      temperature: opts.temperature ?? DEFAULT_TEMPERATURE,
+      messages: merged
+    })
+  };
+}
+function buildGrokRequest(model, messages, opts, env) {
+  if (!env.GROK_API_KEY) {
+    throw new ValidationError("GROK_API_KEY required for grok-* model override");
   }
-  return JSON.stringify({
+  const sys = opts.system ?? messages.find((m) => m.role === "system")?.content;
+  const merged = [];
+  if (sys) merged.push({ role: "system", content: sys });
+  for (const m of messages) if (m.role !== "system") merged.push(m);
+  const body = {
     model,
     max_tokens: opts.maxTokens ?? DEFAULT_MAX_TOKENS,
     temperature: opts.temperature ?? DEFAULT_TEMPERATURE,
     messages: merged
-  });
+  };
+  if (model === MODELS.grok.fast) {
+    body.reasoning_effort = opts.reasoningEffort ?? "none";
+  }
+  return {
+    url: `${env.AI_GATEWAY_BASE_URL}/grok/v1/chat/completions`,
+    headers: {
+      "content-type": "application/json",
+      authorization: `Bearer ${env.GROK_API_KEY}`
+    },
+    body: JSON.stringify(body)
+  };
+}
+function buildDeepSeekRequest(model, messages, opts, env) {
+  if (!env.DEEPSEEK_API_KEY) {
+    throw new ValidationError("DEEPSEEK_API_KEY required for workbench tier or deepseek-* model override");
+  }
+  const sys = opts.system ?? messages.find((m) => m.role === "system")?.content;
+  const merged = [];
+  if (sys) merged.push({ role: "system", content: sys });
+  for (const m of messages) if (m.role !== "system") merged.push(m);
+  return {
+    url: `${env.AI_GATEWAY_BASE_URL}/deepseek/chat/completions`,
+    headers: {
+      "content-type": "application/json",
+      authorization: `Bearer ${env.DEEPSEEK_API_KEY}`
+    },
+    body: JSON.stringify({
+      model,
+      max_tokens: opts.maxTokens ?? DEFAULT_MAX_TOKENS,
+      temperature: opts.temperature ?? DEFAULT_TEMPERATURE,
+      messages: merged
+    })
+  };
 }
 function parseAnthropic(json) {
   const r = json;
-  const text = r.content?.find((c) => c.type === "text")?.text ?? "";
   return {
-    content: text,
+    content: r.content?.find((c) => c.type === "text")?.text ?? "",
     input: r.usage?.input_tokens ?? 0,
-    output: r.usage?.output_tokens ?? 0
+    output: r.usage?.output_tokens ?? 0,
+    cacheRead: r.usage?.cache_read_input_tokens ?? 0,
+    cacheWrite: r.usage?.cache_creation_input_tokens ?? 0,
+    model: r.model
   };
 }
-function parseOpenAI(json) {
+function parseGemini(json) {
   const r = json;
-  const text = r.choices?.[0]?.message?.content ?? "";
+  const text = r.candidates?.[0]?.content?.parts?.map((p) => p.text ?? "").join("") ?? "";
   return {
     content: text,
+    input: r.usageMetadata?.promptTokenCount ?? 0,
+    output: r.usageMetadata?.candidatesTokenCount ?? 0
+  };
+}
+function parseGroq(json) {
+  const r = json;
+  return {
+    content: r.choices?.[0]?.message?.content ?? "",
     input: r.usage?.prompt_tokens ?? 0,
-    output: r.usage?.completion_tokens ?? 0
+    output: r.usage?.completion_tokens ?? 0,
+    model: r.model
   };
 }
-async function callProvider(provider, request, fetchImpl) {
-  const response = await fetchImpl(request.url, {
-    method: "POST",
-    headers: request.headers,
-    body: request.body
-  });
-  if (!response.ok) {
-    const text = await response.text().catch(() => "");
-    const err = {
-      status: response.status,
-      message: `${provider} request failed (${String(response.status)}): ${text.slice(0, 200)}`
-    };
+async function callWithBackoff(provider, request, fetchImpl, signal, logger, nowFn) {
+  function exhaustAndThrow(err) {
+    markProviderCoolingDown(provider, nowFn ?? Date.now);
     throw err;
   }
-  return await response.json();
+  let lastErr;
+  for (let attempt = 1; attempt <= PER_PROVIDER_MAX_ATTEMPTS; attempt++) {
+    try {
+      const response = await fetchImpl(request.url, {
+        method: "POST",
+        headers: request.headers,
+        body: request.body,
+        signal
+      });
+      if (!response.ok) {
+        const text = await response.text().catch(() => "");
+        const retryable = isRetryableForBackoff(response.status);
+        const err = {
+          provider,
+          status: response.status,
+          retryable,
+          message: `${provider} ${String(response.status)}: ${text.slice(0, 300)}`
+        };
+        logger?.warn?.("llm.provider.error", { provider, status: response.status, attempt });
+        if (!err.retryable || attempt === PER_PROVIDER_MAX_ATTEMPTS) {
+          if (err.retryable) exhaustAndThrow(err);
+          throw err;
+        }
+        lastErr = err;
+      } else {
+        const gatewayRequestId = response.headers.get("cf-aig-request-id") ?? void 0;
+        clearProviderCooldown(provider);
+        return { json: await response.json(), gatewayRequestId, attempts: attempt };
+      }
+    } catch (e) {
+      if (e instanceof DOMException && e.name === "AbortError") throw e;
+      if (typeof e === "object" && e !== null && "retryable" in e) {
+        const err = e;
+        if (!err.retryable || attempt === PER_PROVIDER_MAX_ATTEMPTS) {
+          if (err.retryable) exhaustAndThrow(err);
+          throw err;
+        }
+        lastErr = err;
+      } else {
+        const err = {
+          provider,
+          status: 0,
+          retryable: true,
+          message: e instanceof Error ? e.message : String(e)
+        };
+        if (attempt === PER_PROVIDER_MAX_ATTEMPTS) exhaustAndThrow(err);
+        lastErr = err;
+      }
+    }
+    const backoffMs = computeBackoffMs(attempt - 1);
+    await sleep(backoffMs, signal);
+  }
+  markProviderCoolingDown(provider, nowFn ?? Date.now);
+  throw lastErr ?? { provider, status: 0, retryable: false, message: "exhausted" };
 }
 function isProviderError(err) {
-  return typeof err === "object" && err !== null && typeof err.status === "number" && typeof err.message === "string";
+  return typeof err === "object" && err !== null && typeof err.status === "number" && typeof err.message === "string" && typeof err.provider === "string";
+}
+function plan(tier, opts, tokenEstimate) {
+  if (opts.model) {
+    const m = opts.model;
+    if (m.startsWith("claude")) return { primary: { provider: "anthropic", model: m } };
+    if (m.startsWith("gemini")) return { primary: { provider: "gemini", model: m } };
+    if (m.startsWith("grok")) return { primary: { provider: "grok", model: m } };
+    if (m.startsWith("deepseek")) return { primary: { provider: "deepseek", model: m } };
+    return { primary: { provider: "groq", model: m } };
+  }
+  const longContext = tokenEstimate >= (opts.longContextThreshold ?? DEFAULT_LONG_CONTEXT_THRESHOLD);
+  switch (tier) {
+    case "workbench":
+      return {
+        primary: { provider: "deepseek", model: MODELS.deepseek.workbench },
+        fallback: { provider: "groq", model: MODELS.groq.verifier }
+      };
+    case "verifier":
+      return { primary: { provider: "groq", model: MODELS.groq.verifier } };
+    case "smart":
+      return longContext ? {
+        primary: { provider: "gemini", model: MODELS.gemini.smart },
+        fallback: { provider: "anthropic", model: MODELS.anthropic.smart }
+      } : {
+        primary: { provider: "anthropic", model: MODELS.anthropic.smart },
+        fallback: { provider: "gemini", model: MODELS.gemini.smart }
+      };
+    case "fast":
+      return {
+        primary: { provider: "grok", model: MODELS.grok.fast },
+        fallback: { provider: "anthropic", model: MODELS.anthropic.fast }
+      };
+    case "balanced":
+    default:
+      return longContext ? {
+        primary: { provider: "gemini", model: MODELS.gemini.smart },
+        fallback: { provider: "anthropic", model: MODELS.anthropic.balanced }
+      } : {
+        primary: { provider: "anthropic", model: MODELS.anthropic.balanced },
+        fallback: { provider: "gemini", model: MODELS.gemini.smart }
+      };
+  }
+}
+function buildAigMetadata(opts) {
+  const meta = {};
+  if (opts.project) meta.project = opts.project;
+  if (opts.workload) meta.workload = opts.workload;
+  if (opts.actor) meta.actor = opts.actor;
+  if (opts.runId) meta.runId = opts.runId;
+  return Object.keys(meta).length > 0 ? JSON.stringify(meta) : void 0;
+}
+async function callOne(leg, messages, opts, env, fetchImpl, logger, nowFn) {
+  let req;
+  switch (leg.provider) {
+    case "anthropic":
+      req = buildAnthropicRequest(leg.model, messages, opts, env);
+      break;
+    case "gemini":
+      req = buildGeminiRequest(leg.model, messages, opts, env);
+      break;
+    case "groq":
+      req = buildGroqRequest(leg.model, messages, opts, env);
+      break;
+    case "grok":
+      req = buildGrokRequest(leg.model, messages, opts, env);
+      break;
+    case "deepseek":
+      req = buildDeepSeekRequest(leg.model, messages, opts, env);
+      break;
+  }
+  const aigMetadata = buildAigMetadata(opts);
+  if (aigMetadata) req.headers["cf-aig-metadata"] = aigMetadata;
+  const { json, gatewayRequestId, attempts } = await callWithBackoff(
+    leg.provider,
+    req,
+    fetchImpl,
+    opts.signal,
+    logger,
+    nowFn
+  );
+  switch (leg.provider) {
+    case "anthropic":
+      return { parsed: parseAnthropic(json), gatewayRequestId, attempts };
+    case "gemini":
+      return { parsed: parseGemini(json), gatewayRequestId, attempts };
+    case "groq":
+      return { parsed: parseGroq(json), gatewayRequestId, attempts };
+    case "grok":
+      return { parsed: parseGroq(json), gatewayRequestId, attempts };
+    case "deepseek":
+      return { parsed: parseGroq(json), gatewayRequestId, attempts };
+  }
 }
 async function complete(messages, env, opts = {}, deps = {}) {
   if (messages.length === 0) {
     throw new ValidationError("messages must not be empty");
   }
+  if (!env.AI_GATEWAY_BASE_URL) {
+    throw new ValidationError("AI_GATEWAY_BASE_URL is required in 0.3.0");
+  }
   const fetchImpl = deps.fetch ?? fetch;
   const now = deps.now ?? (() => Date.now());
   const logger = deps.logger;
   const startedAt = now();
-  const attempts = [];
-  try {
-    const json = await callProvider(
-      "anthropic",
-      {
-        url: "https://api.anthropic.com/v1/messages",
-        headers: {
-          "content-type": "application/json",
-          "x-api-key": env.ANTHROPIC_API_KEY,
-          "anthropic-version": "2023-06-01"
-        },
-        body: buildAnthropicBody(messages, opts, opts.system)
-      },
-      fetchImpl
-    );
-    const parsed = parseAnthropic(json);
-    return {
-      data: {
-        content: parsed.content,
-        provider: "anthropic",
-        tokens: { input: parsed.input, output: parsed.output },
-        latency: now() - startedAt
-      },
-      error: null
-    };
-  } catch (err) {
-    const status = isProviderError(err) ? err.status : 0;
-    const message = isProviderError(err) ? err.message : err.message;
-    attempts.push({ provider: "anthropic", status, message });
-    if (status !== 0 && !isFailover(status)) {
-      return providerErrorResponse(attempts, status === 429);
+  const tier = opts.tier ?? "balanced";
+  const system = opts.system ?? messages.find((m) => m.role === "system")?.content;
+  const tokenEstimate = estimateTokens(messages, system);
+  const route = plan(tier, opts, tokenEstimate);
+  const kv = env.LLM_COST_KV;
+  const todayKey = `llm:daily-cost:${isoDate(now())}`;
+  const monthKey = `llm:monthly-cost:${isoMonth(now())}`;
+  if (kv) {
+    if (opts.dailyCapUsd !== void 0) {
+      const raw = await kv.get(todayKey).catch(() => null);
+      const spent = parseFloat(raw ?? "0");
+      if (spent >= opts.dailyCapUsd) {
+        return toErrorResponse(
+          new RateLimitError("LLM_DAILY_CAP_EXCEEDED", {
+            spentUsd: spent,
+            dailyCapUsd: opts.dailyCapUsd
+          })
+        );
+      }
+    }
+    if (opts.monthlyCapUsd !== void 0) {
+      const raw = await kv.get(monthKey).catch(() => null);
+      const spent = parseFloat(raw ?? "0");
+      if (spent >= opts.monthlyCapUsd) {
+        return toErrorResponse(
+          new RateLimitError("LLM_MONTHLY_CAP_EXCEEDED", {
+            spentUsd: spent,
+            monthlyCapUsd: opts.monthlyCapUsd
+          })
+        );
+      }
     }
-    logger?.warn("llm.failover", { from: "anthropic", to: "grok", status, message });
-  }
-  try {
-    const json = await callProvider(
-      "grok",
-      {
-        url: "https://api.x.ai/v1/chat/completions",
-        headers: {
-          "content-type": "application/json",
-          authorization: `Bearer ${env.GROK_API_KEY}`
-        },
-        body: buildOpenAIBody(opts.model ?? DEFAULT_GROK_MODEL, messages, opts, opts.system)
-      },
-      fetchImpl
-    );
-    const parsed = parseOpenAI(json);
-    return {
-      data: {
-        content: parsed.content,
-        provider: "grok",
-        tokens: { input: parsed.input, output: parsed.output },
-        latency: now() - startedAt
-      },
-      error: null
-    };
-  } catch (err) {
-    const status = isProviderError(err) ? err.status : 0;
-    const message = isProviderError(err) ? err.message : err.message;
-    attempts.push({ provider: "grok", status, message });
-    logger?.warn("llm.failover", { from: "grok", to: "groq", status, message });
   }
-  try {
-    const json = await callProvider(
-      "groq",
-      {
-        url: "https://api.groq.com/openai/v1/chat/completions",
-        headers: {
-          "content-type": "application/json",
-          authorization: `Bearer ${env.GROQ_API_KEY}`
+  const attemptLog = [];
+  const routeLegs = [route.primary, route.fallback].filter(Boolean);
+  for (const [legIndex, leg] of routeLegs.entries()) {
+    if (isProviderCoolingDown(leg.provider, now)) {
+      logger?.warn?.("llm.provider.coolingDown", { provider: leg.provider });
+      attemptLog.push({ provider: leg.provider, message: "skipped: cooling down" });
+      continue;
+    }
+    if (opts.signal?.aborted) {
+      return toErrorResponse(
+        new InternalError("llm call aborted", { provider: leg.provider, model: leg.model })
+      );
+    }
+    try {
+      const result = await callOne(leg, messages, opts, env, fetchImpl, logger, now);
+      if (!result.parsed.content) {
+        throw { provider: leg.provider, status: 200, retryable: false, message: "empty content" };
+      }
+      logger?.info?.("llm.complete", {
+        provider: leg.provider,
+        model: leg.model,
+        tier,
+        tokenEstimate,
+        attempts: result.attempts,
+        runId: opts.runId,
+        project: opts.project,
+        actor: opts.actor,
+        workload: opts.workload
+      });
+      const llmResult = {
+        content: result.parsed.content,
+        provider: leg.provider,
+        model: result.parsed.model ?? leg.model,
+        tier,
+        tokens: {
+          input: result.parsed.input,
+          output: result.parsed.output,
+          cacheRead: result.parsed.cacheRead,
+          cacheWrite: result.parsed.cacheWrite
         },
-        body: buildOpenAIBody(opts.model ?? DEFAULT_GROQ_MODEL, messages, opts, opts.system)
-      },
-      fetchImpl
-    );
-    const parsed = parseOpenAI(json);
-    return {
-      data: {
-        content: parsed.content,
-        provider: "groq",
-        tokens: { input: parsed.input, output: parsed.output },
-        latency: now() - startedAt
-      },
-      error: null
-    };
-  } catch (err) {
-    const status = isProviderError(err) ? err.status : 0;
-    const message = isProviderError(err) ? err.message : err.message;
-    attempts.push({ provider: "groq", status, message });
-    logger?.error("llm.all_providers_failed", void 0, { attempts });
-  }
-  return providerErrorResponse(attempts, false);
-}
-function providerErrorResponse(attempts, rateLimited) {
-  const base = rateLimited ? new RateLimitError("LLM provider rate limited", { code: ErrorCodes.LLM_RATE_LIMITED, attempts }) : new InternalError("All LLM providers failed", {
-    code: ErrorCodes.LLM_ALL_PROVIDERS_FAILED,
-    attempts
-  });
-  return {
-    data: null,
-    error: {
-      code: rateLimited ? ErrorCodes.LLM_RATE_LIMITED : ErrorCodes.LLM_ALL_PROVIDERS_FAILED,
-      message: base.message,
-      status: base.status,
-      retryable: base.retryable,
-      context: base.context
+        latency: now() - startedAt,
+        attempts: result.attempts,
+        gatewayRequestId: result.gatewayRequestId
+      };
+      const costUsd = estimateCostUsd(llmResult.tokens, llmResult.model);
+      if (opts.maxCostUsd !== void 0 && costUsd > opts.maxCostUsd) {
+        if (kv && (opts.dailyCapUsd !== void 0 || opts.monthlyCapUsd !== void 0)) {
+          await recordOrgCostUsage(kv, todayKey, monthKey, costUsd, opts);
+        }
+        return toErrorResponse(
+          new RateLimitError("LLM_COST_CAP_EXCEEDED", {
+            costUsd,
+            maxCostUsd: opts.maxCostUsd,
+            model: llmResult.model,
+            tokens: llmResult.tokens
+          })
+        );
+      }
+      if (kv && (opts.dailyCapUsd !== void 0 || opts.monthlyCapUsd !== void 0)) {
+        await recordOrgCostUsage(kv, todayKey, monthKey, costUsd, opts);
+      }
+      if (deps.onRecord && opts.ledger) {
+        const row = {
+          ...opts.ledger,
+          model: llmResult.model,
+          provider: llmResult.provider,
+          tier: llmResult.tier,
+          inputTokens: llmResult.tokens.input,
+          outputTokens: llmResult.tokens.output,
+          cacheReadTokens: llmResult.tokens.cacheRead ?? 0,
+          cacheWriteTokens: llmResult.tokens.cacheWrite ?? 0,
+          latencyMs: llmResult.latency,
+          costUsd,
+          yyyyMm: isoMonth(now())
+        };
+        deps.onRecord(row).catch((e) => {
+          logger?.warn?.("llm.onRecord.error", { message: e instanceof Error ? e.message : String(e) });
+        });
+      }
+      return { data: llmResult, error: null };
+    } catch (e) {
+      if (e instanceof DOMException && e.name === "AbortError") {
+        return toErrorResponse(
+          new InternalError("llm call aborted", { provider: leg.provider, model: leg.model })
+        );
+      }
+      if (isProviderError(e)) {
+        attemptLog.push({ provider: e.provider, status: e.status, message: e.message });
+        if (e.status === 429 && legIndex === routeLegs.length - 1) {
+          return toErrorResponse(
+            new RateLimitError(`llm rate limited on ${e.provider}`, { attempts: attemptLog })
+          );
+        }
+        logger?.warn?.("llm.leg.failed", { provider: leg.provider, status: e.status });
+        continue;
+      }
+      attemptLog.push({ provider: leg.provider, message: e instanceof Error ? e.message : String(e) });
     }
-  };
+  }
+  return toErrorResponse(
+    new InternalError("LLM_ALL_PROVIDERS_FAILED", { attempts: attemptLog, tier, tokenEstimate })
+  );
 }
-async function stream(messages, env, opts = {}, deps = {}) {
+async function* completionStream(messages, env, opts = {}) {
   if (messages.length === 0) {
     throw new ValidationError("messages must not be empty");
   }
+  if (!env.AI_GATEWAY_BASE_URL) {
+    throw new ValidationError("AI_GATEWAY_BASE_URL is required in 0.3.0");
+  }
+  const deps = opts.deps ?? {};
   const fetchImpl = deps.fetch ?? fetch;
-  const body = JSON.parse(buildAnthropicBody(messages, opts, opts.system));
-  body.stream = true;
-  const response = await fetchImpl("https://api.anthropic.com/v1/messages", {
-    method: "POST",
-    headers: {
-      "content-type": "application/json",
-      "x-api-key": env.ANTHROPIC_API_KEY,
-      "anthropic-version": "2023-06-01"
-    },
-    body: JSON.stringify(body)
-  });
-  if (!response.ok || !response.body) {
-    throw new InternalError("Anthropic stream failed", {
-      code: ErrorCodes.LLM_ALL_PROVIDERS_FAILED,
-      status: response.status
+  const now = deps.now ?? (() => Date.now());
+  const logger = deps.logger;
+  const startedAt = now();
+  const tier = opts.tier ?? "balanced";
+  const system = opts.system ?? messages.find((m) => m.role === "system")?.content;
+  const tokenEstimate = estimateTokens(messages, system);
+  const route = plan(tier, opts, tokenEstimate);
+  const streamLeg = route.primary.provider === "grok" && !env.GROK_API_KEY && route.fallback?.provider === "anthropic" ? route.fallback : route.primary;
+  if (streamLeg.provider !== "anthropic") {
+    const result = await complete(messages, env, opts, deps);
+    if (result.error !== null || result.data === null) {
+      throw new InternalError("LLM_ALL_PROVIDERS_FAILED", { error: result.error });
+    }
+    yield result.data.content;
+    return result.data;
+  }
+  if (isProviderCoolingDown(streamLeg.provider, now)) {
+    logger?.warn?.("llm.provider.coolingDown", { provider: streamLeg.provider });
+    const result = await complete(messages, env, opts, deps);
+    if (result.error !== null || result.data === null) {
+      throw new InternalError("LLM_ALL_PROVIDERS_FAILED", { error: result.error });
+    }
+    yield result.data.content;
+    return result.data;
+  }
+  const req = buildAnthropicRequest(streamLeg.model, messages, opts, env, true);
+  const streamAigMetadata = buildAigMetadata(opts);
+  if (streamAigMetadata) req.headers["cf-aig-metadata"] = streamAigMetadata;
+  let response;
+  try {
+    response = await fetchImpl(req.url, {
+      method: "POST",
+      headers: req.headers,
+      body: req.body,
+      // Fall back to a 60 s default when the caller provides no signal — prevents
+      // a hung provider connection from consuming the Worker's wall-clock budget.
+      signal: opts.signal ?? AbortSignal.timeout(6e4)
+    });
+  } catch (e) {
+    if (e instanceof DOMException && e.name === "AbortError") {
+      throw new InternalError("llm call aborted", {
+        provider: streamLeg.provider,
+        model: streamLeg.model
+      });
+    }
+    throw new InternalError("llm stream fetch failed", {
+      message: e instanceof Error ? e.message : String(e)
     });
   }
-  return response.body;
+  if (!response.ok) {
+    const text = await response.text().catch(() => "");
+    const retryable = isRetryableForBackoff(response.status);
+    if (retryable && response.status === 429) {
+      markProviderCoolingDown(streamLeg.provider, now);
+    }
+    const result = await complete(messages, env, opts, deps);
+    if (result.error !== null || result.data === null) {
+      throw new InternalError("LLM_ALL_PROVIDERS_FAILED", {
+        streamError: `${streamLeg.provider} ${String(response.status)}: ${text.slice(0, 300)}`,
+        error: result.error
+      });
+    }
+    yield result.data.content;
+    return result.data;
+  }
+  if (!response.body) {
+    throw new InternalError("llm stream response body is null", {
+      provider: streamLeg.provider
+    });
+  }
+  const decoder = new TextDecoder();
+  let accumulatedText = "";
+  let inputTokens = 0;
+  let outputTokens = 0;
+  let cacheRead = 0;
+  let cacheWrite = 0;
+  let modelName;
+  const gatewayRequestId = response.headers.get("cf-aig-request-id") ?? void 0;
+  const reader = response.body.getReader();
+  let buffer = "";
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      buffer += decoder.decode(value, { stream: true });
+      const lines = buffer.split("\n");
+      buffer = lines.pop() ?? "";
+      for (const line of lines) {
+        if (!line.startsWith("data: ")) continue;
+        const data = line.slice(6).trim();
+        if (data === "[DONE]") break;
+        let event;
+        try {
+          event = JSON.parse(data);
+        } catch {
+          continue;
+        }
+        switch (event.type) {
+          case "message_start":
+            inputTokens = event.message?.usage?.input_tokens ?? 0;
+            cacheRead = event.message?.usage?.cache_read_input_tokens ?? 0;
+            cacheWrite = event.message?.usage?.cache_creation_input_tokens ?? 0;
+            modelName = event.message?.model;
+            break;
+          case "content_block_delta":
+            if (event.delta?.type === "text_delta" && typeof event.delta.text === "string") {
+              accumulatedText += event.delta.text;
+              yield event.delta.text;
+            }
+            break;
+          case "message_delta":
+            outputTokens = event.usage?.output_tokens ?? outputTokens;
+            break;
+          default:
+            break;
+        }
+      }
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  clearProviderCooldown(streamLeg.provider);
+  logger?.info?.("llm.completionStream", {
+    provider: streamLeg.provider,
+    model: streamLeg.model,
+    tier,
+    tokenEstimate,
+    runId: opts.runId,
+    project: opts.project,
+    actor: opts.actor,
+    workload: opts.workload
+  });
+  return {
+    content: accumulatedText,
+    provider: streamLeg.provider,
+    model: modelName ?? streamLeg.model,
+    tier,
+    tokens: { input: inputTokens, output: outputTokens, cacheRead, cacheWrite },
+    latency: now() - startedAt,
+    attempts: 1,
+    gatewayRequestId
+  };
 }
-function withSystem(system) {
-  return (messages, env, opts = {}, deps = {}) => complete(messages, env, { ...opts, system }, deps);
+function assertGrounding(response, sources) {
+  if (sources.length === 0) return true;
+  const WINDOW = 5;
+  const responseTokens = response.split(/\s+/).filter((t) => t.length > 0);
+  if (responseTokens.length < WINDOW) return false;
+  const sourceNgrams = /* @__PURE__ */ new Set();
+  for (const source of sources) {
+    const tokens = source.split(/\s+/).filter((t) => t.length > 0);
+    for (let i = 0; i <= tokens.length - WINDOW; i++) {
+      const ngram = tokens.slice(i, i + WINDOW).join(" ");
+      sourceNgrams.add(ngram);
+    }
+  }
+  if (sourceNgrams.size === 0) return false;
+  for (let i = 0; i <= responseTokens.length - WINDOW; i++) {
+    const ngram = responseTokens.slice(i, i + WINDOW).join(" ");
+    if (sourceNgrams.has(ngram)) return true;
+  }
+  return false;
 }
 export {
+  BASE_BACKOFF_MS,
+  MODELS,
+  MODEL_PRICE_PER_1M,
+  PROVIDER_COOLDOWN_MS,
+  assertGrounding,
+  clearProviderCooldown,
   complete,
-  stream,
-  withSystem
+  completionStream,
+  isProviderCoolingDown,
+  markProviderCoolingDown
 };
 //# sourceMappingURL=index.mjs.map