@latimer-woods-tech/llm 0.2.0 → 0.3.1

package/dist/index.mjs

@@ -1,235 +1,583 @@
 // src/index.ts
 import {
-  ErrorCodes,
   InternalError,
   RateLimitError,
-  ValidationError
+  ValidationError,
+  toErrorResponse
 } from "@latimer-woods-tech/errors";
+var MODELS = {
+  anthropic: {
+    fast: "claude-haiku-4-20250514",
+    balanced: "claude-sonnet-4-6",
+    smart: "claude-opus-4-7"
+  },
+  gemini: {
+    smart: "gemini-2.5-pro"
+  },
+  groq: {
+    verifier: "llama-3.3-70b-versatile"
+  },
+  grok: {
+    /** Opt-in only via `{ model: 'grok-*' }`. Not in default tier routing. */
+    fast: "grok-4-fast",
+    mini: "grok-3-mini-latest"
+  }
+};
 var DEFAULT_MAX_TOKENS = 1024;
 var DEFAULT_TEMPERATURE = 0.7;
-var DEFAULT_ANTHROPIC_MODEL = "claude-sonnet-4-20250514";
-var DEFAULT_GROK_MODEL = "grok-3-fast";
-var DEFAULT_GROQ_MODEL = "llama-3.3-70b-versatile";
-function isFailover(status) {
-  return status === 429 || status >= 500;
+var DEFAULT_LONG_CONTEXT_THRESHOLD = 15e4;
+var BACKOFF_BASE_MS = 500;
+var BACKOFF_CAP_MS = 8e3;
+var BACKOFF_JITTER_MAX_MS = 250;
+var PER_PROVIDER_MAX_ATTEMPTS = 3;
+var providerCooldownUntil = /* @__PURE__ */ new Map();
+var PROVIDER_COOLDOWN_MS = 3e4;
+function isProviderCoolingDown(provider, now = Date.now) {
+  const until = providerCooldownUntil.get(provider);
+  if (until === void 0) return false;
+  return now() < until;
+}
+function markProviderCoolingDown(provider, now = Date.now) {
+  providerCooldownUntil.set(provider, now() + PROVIDER_COOLDOWN_MS);
+}
+function clearProviderCooldown(provider) {
+  providerCooldownUntil.delete(provider);
+}
+var BASE_BACKOFF_MS = 250;
+function isRetryableForBackoff(status) {
+  return status === 429 || status >= 500 && status < 600;
+}
+function estimateTokens(messages, system) {
+  let chars = system?.length ?? 0;
+  for (const m of messages) chars += m.content.length;
+  return Math.ceil(chars / 4);
+}
+function sleep(ms, signal) {
+  return new Promise((resolve, reject) => {
+    const t = setTimeout(resolve, ms);
+    if (signal) {
+      const onAbort = () => {
+        clearTimeout(t);
+        reject(new DOMException("Aborted", "AbortError"));
+      };
+      if (signal.aborted) onAbort();
+      else signal.addEventListener("abort", onAbort, { once: true });
+    }
+  });
 }
-function buildAnthropicBody(messages, opts, system) {
+function computeBackoffMs(attempt) {
+  const jitter = Math.floor(Math.random() * BACKOFF_JITTER_MAX_MS);
+  return Math.min(BACKOFF_BASE_MS * Math.pow(2, attempt) + jitter, BACKOFF_CAP_MS);
+}
+function buildAnthropicRequest(model, messages, opts, env, streaming = false) {
+  const sys = opts.system ?? messages.find((m) => m.role === "system")?.content;
   const filtered = messages.filter((m) => m.role !== "system");
-  const sys = system ?? messages.find((m) => m.role === "system")?.content;
   const body = {
-    model: opts.model ?? DEFAULT_ANTHROPIC_MODEL,
+    model,
     max_tokens: opts.maxTokens ?? DEFAULT_MAX_TOKENS,
     temperature: opts.temperature ?? DEFAULT_TEMPERATURE,
     messages: filtered.map((m) => ({ role: m.role, content: m.content }))
   };
+  if (streaming) {
+    body.stream = true;
+  }
   if (sys) {
-    body.system = sys;
+    const cache = opts.promptCache ?? sys.length >= 4096;
+    body.system = cache ? [{ type: "text", text: sys, cache_control: { type: "ephemeral" } }] : sys;
   }
-  return JSON.stringify(body);
+  return {
+    url: `${env.AI_GATEWAY_BASE_URL}/anthropic/v1/messages`,
+    headers: {
+      "content-type": "application/json",
+      "x-api-key": env.ANTHROPIC_API_KEY,
+      "anthropic-version": "2023-06-01",
+      "anthropic-beta": "prompt-caching-2024-07-31"
+    },
+    body: JSON.stringify(body)
+  };
 }
-function buildOpenAIBody(model, messages, opts, system) {
-  const sys = system ?? messages.find((m) => m.role === "system")?.content;
-  const merged = [];
+function buildGeminiRequest(model, messages, opts, env) {
+  const sys = opts.system ?? messages.find((m) => m.role === "system")?.content;
+  const contents = messages.filter((m) => m.role !== "system").map((m) => ({
+    role: m.role === "assistant" ? "model" : "user",
+    parts: [{ text: m.content }]
+  }));
+  const body = {
+    contents,
+    generationConfig: {
+      maxOutputTokens: opts.maxTokens ?? DEFAULT_MAX_TOKENS,
+      temperature: opts.temperature ?? DEFAULT_TEMPERATURE
+    }
+  };
   if (sys) {
-    merged.push({ role: "system", content: sys });
+    body.systemInstruction = { parts: [{ text: sys }] };
   }
-  for (const m of messages) {
-    if (m.role !== "system") {
-      merged.push(m);
-    }
+  const path = `v1/projects/${env.VERTEX_PROJECT}/locations/${env.VERTEX_LOCATION}/publishers/google/models/${model}:generateContent`;
+  return {
+    url: `${env.AI_GATEWAY_BASE_URL}/google-vertex-ai/${path}`,
+    headers: {
+      "content-type": "application/json",
+      authorization: `Bearer ${env.VERTEX_ACCESS_TOKEN}`
+    },
+    body: JSON.stringify(body)
+  };
+}
+function buildGroqRequest(model, messages, opts, env) {
+  const sys = opts.system ?? messages.find((m) => m.role === "system")?.content;
+  const merged = [];
+  if (sys) merged.push({ role: "system", content: sys });
+  for (const m of messages) if (m.role !== "system") merged.push(m);
+  return {
+    url: `${env.AI_GATEWAY_BASE_URL}/groq/openai/v1/chat/completions`,
+    headers: {
+      "content-type": "application/json",
+      authorization: `Bearer ${env.GROQ_API_KEY}`
+    },
+    body: JSON.stringify({
+      model,
+      max_tokens: opts.maxTokens ?? DEFAULT_MAX_TOKENS,
+      temperature: opts.temperature ?? DEFAULT_TEMPERATURE,
+      messages: merged
+    })
+  };
+}
+function buildGrokRequest(model, messages, opts, env) {
+  if (!env.GROK_API_KEY) {
+    throw new ValidationError("GROK_API_KEY required for grok-* model override");
   }
-  return JSON.stringify({
-    model,
-    max_tokens: opts.maxTokens ?? DEFAULT_MAX_TOKENS,
-    temperature: opts.temperature ?? DEFAULT_TEMPERATURE,
-    messages: merged
-  });
+  const sys = opts.system ?? messages.find((m) => m.role === "system")?.content;
+  const merged = [];
+  if (sys) merged.push({ role: "system", content: sys });
+  for (const m of messages) if (m.role !== "system") merged.push(m);
+  return {
+    url: `${env.AI_GATEWAY_BASE_URL}/grok/v1/chat/completions`,
+    headers: {
+      "content-type": "application/json",
+      authorization: `Bearer ${env.GROK_API_KEY}`
+    },
+    body: JSON.stringify({
+      model,
+      max_tokens: opts.maxTokens ?? DEFAULT_MAX_TOKENS,
+      temperature: opts.temperature ?? DEFAULT_TEMPERATURE,
+      messages: merged
+    })
+  };
 }
 function parseAnthropic(json) {
   const r = json;
-  const text = r.content?.find((c) => c.type === "text")?.text ?? "";
   return {
-    content: text,
+    content: r.content?.find((c) => c.type === "text")?.text ?? "",
     input: r.usage?.input_tokens ?? 0,
-    output: r.usage?.output_tokens ?? 0
+    output: r.usage?.output_tokens ?? 0,
+    cacheRead: r.usage?.cache_read_input_tokens ?? 0,
+    cacheWrite: r.usage?.cache_creation_input_tokens ?? 0,
+    model: r.model
   };
 }
-function parseOpenAI(json) {
+function parseGemini(json) {
   const r = json;
-  const text = r.choices?.[0]?.message?.content ?? "";
+  const text = r.candidates?.[0]?.content?.parts?.map((p) => p.text ?? "").join("") ?? "";
   return {
     content: text,
+    input: r.usageMetadata?.promptTokenCount ?? 0,
+    output: r.usageMetadata?.candidatesTokenCount ?? 0
+  };
+}
+function parseGroq(json) {
+  const r = json;
+  return {
+    content: r.choices?.[0]?.message?.content ?? "",
     input: r.usage?.prompt_tokens ?? 0,
-    output: r.usage?.completion_tokens ?? 0
+    output: r.usage?.completion_tokens ?? 0,
+    model: r.model
   };
 }
-async function callProvider(provider, request, fetchImpl) {
-  const response = await fetchImpl(request.url, {
-    method: "POST",
-    headers: request.headers,
-    body: request.body
-  });
-  if (!response.ok) {
-    const text = await response.text().catch(() => "");
-    const err = {
-      status: response.status,
-      message: `${provider} request failed (${String(response.status)}): ${text.slice(0, 200)}`
-    };
+async function callWithBackoff(provider, request, fetchImpl, signal, logger, nowFn) {
+  function exhaustAndThrow(err) {
+    markProviderCoolingDown(provider, nowFn ?? Date.now);
     throw err;
   }
-  return await response.json();
+  let lastErr;
+  for (let attempt = 1; attempt <= PER_PROVIDER_MAX_ATTEMPTS; attempt++) {
+    try {
+      const response = await fetchImpl(request.url, {
+        method: "POST",
+        headers: request.headers,
+        body: request.body,
+        signal
+      });
+      if (!response.ok) {
+        const text = await response.text().catch(() => "");
+        const retryable = isRetryableForBackoff(response.status);
+        const err = {
+          provider,
+          status: response.status,
+          retryable,
+          message: `${provider} ${String(response.status)}: ${text.slice(0, 300)}`
+        };
+        logger?.warn?.("llm.provider.error", { provider, status: response.status, attempt });
+        if (!err.retryable || attempt === PER_PROVIDER_MAX_ATTEMPTS) {
+          if (err.retryable) exhaustAndThrow(err);
+          throw err;
+        }
+        lastErr = err;
+      } else {
+        const gatewayRequestId = response.headers.get("cf-aig-request-id") ?? void 0;
+        clearProviderCooldown(provider);
+        return { json: await response.json(), gatewayRequestId, attempts: attempt };
+      }
+    } catch (e) {
+      if (e instanceof DOMException && e.name === "AbortError") throw e;
+      if (typeof e === "object" && e !== null && "retryable" in e) {
+        const err = e;
+        if (!err.retryable || attempt === PER_PROVIDER_MAX_ATTEMPTS) {
+          if (err.retryable) exhaustAndThrow(err);
+          throw err;
+        }
+        lastErr = err;
+      } else {
+        const err = {
+          provider,
+          status: 0,
+          retryable: true,
+          message: e instanceof Error ? e.message : String(e)
+        };
+        if (attempt === PER_PROVIDER_MAX_ATTEMPTS) exhaustAndThrow(err);
+        lastErr = err;
+      }
+    }
+    const backoffMs = computeBackoffMs(attempt - 1);
+    await sleep(backoffMs, signal);
+  }
+  markProviderCoolingDown(provider, nowFn ?? Date.now);
+  throw lastErr ?? { provider, status: 0, retryable: false, message: "exhausted" };
 }
 function isProviderError(err) {
-  return typeof err === "object" && err !== null && typeof err.status === "number" && typeof err.message === "string";
+  return typeof err === "object" && err !== null && typeof err.status === "number" && typeof err.message === "string" && typeof err.provider === "string";
+}
+function plan(tier, opts, tokenEstimate) {
+  if (opts.model) {
+    const m = opts.model;
+    if (m.startsWith("claude")) return { primary: { provider: "anthropic", model: m } };
+    if (m.startsWith("gemini")) return { primary: { provider: "gemini", model: m } };
+    if (m.startsWith("grok")) return { primary: { provider: "grok", model: m } };
+    return { primary: { provider: "groq", model: m } };
+  }
+  const longContext = tokenEstimate >= (opts.longContextThreshold ?? DEFAULT_LONG_CONTEXT_THRESHOLD);
+  switch (tier) {
+    case "verifier":
+      return { primary: { provider: "groq", model: MODELS.groq.verifier } };
+    case "smart":
+      return longContext ? {
+        primary: { provider: "gemini", model: MODELS.gemini.smart },
+        fallback: { provider: "anthropic", model: MODELS.anthropic.smart }
+      } : {
+        primary: { provider: "anthropic", model: MODELS.anthropic.smart },
+        fallback: { provider: "gemini", model: MODELS.gemini.smart }
+      };
+    case "fast":
+      return { primary: { provider: "anthropic", model: MODELS.anthropic.fast } };
+    case "balanced":
+    default:
+      return longContext ? {
+        primary: { provider: "gemini", model: MODELS.gemini.smart },
+        fallback: { provider: "anthropic", model: MODELS.anthropic.balanced }
+      } : {
+        primary: { provider: "anthropic", model: MODELS.anthropic.balanced },
+        fallback: { provider: "gemini", model: MODELS.gemini.smart }
+      };
+  }
+}
+async function callOne(leg, messages, opts, env, fetchImpl, logger, nowFn) {
+  let req;
+  switch (leg.provider) {
+    case "anthropic":
+      req = buildAnthropicRequest(leg.model, messages, opts, env);
+      break;
+    case "gemini":
+      req = buildGeminiRequest(leg.model, messages, opts, env);
+      break;
+    case "groq":
+      req = buildGroqRequest(leg.model, messages, opts, env);
+      break;
+    case "grok":
+      req = buildGrokRequest(leg.model, messages, opts, env);
+      break;
+  }
+  const { json, gatewayRequestId, attempts } = await callWithBackoff(
+    leg.provider,
+    req,
+    fetchImpl,
+    opts.signal,
+    logger,
+    nowFn
+  );
+  switch (leg.provider) {
+    case "anthropic":
+      return { parsed: parseAnthropic(json), gatewayRequestId, attempts };
+    case "gemini":
+      return { parsed: parseGemini(json), gatewayRequestId, attempts };
+    case "groq":
+      return { parsed: parseGroq(json), gatewayRequestId, attempts };
+    case "grok":
+      return { parsed: parseGroq(json), gatewayRequestId, attempts };
+  }
 }
 async function complete(messages, env, opts = {}, deps = {}) {
   if (messages.length === 0) {
     throw new ValidationError("messages must not be empty");
   }
+  if (!env.AI_GATEWAY_BASE_URL) {
+    throw new ValidationError("AI_GATEWAY_BASE_URL is required in 0.3.0");
+  }
   const fetchImpl = deps.fetch ?? fetch;
   const now = deps.now ?? (() => Date.now());
   const logger = deps.logger;
   const startedAt = now();
-  const attempts = [];
-  try {
-    const json = await callProvider(
-      "anthropic",
-      {
-        url: "https://api.anthropic.com/v1/messages",
-        headers: {
-          "content-type": "application/json",
-          "x-api-key": env.ANTHROPIC_API_KEY,
-          "anthropic-version": "2023-06-01"
+  const tier = opts.tier ?? "balanced";
+  const system = opts.system ?? messages.find((m) => m.role === "system")?.content;
+  const tokenEstimate = estimateTokens(messages, system);
+  const route = plan(tier, opts, tokenEstimate);
+  const attemptLog = [];
+  for (const leg of [route.primary, route.fallback].filter(Boolean)) {
+    if (isProviderCoolingDown(leg.provider, now)) {
+      logger?.warn?.("llm.provider.coolingDown", { provider: leg.provider });
+      attemptLog.push({ provider: leg.provider, message: "skipped: cooling down" });
+      continue;
+    }
+    try {
+      const result = await callOne(leg, messages, opts, env, fetchImpl, logger, now);
+      if (!result.parsed.content) {
+        throw { provider: leg.provider, status: 200, retryable: false, message: "empty content" };
+      }
+      logger?.info?.("llm.complete", {
+        provider: leg.provider,
+        model: leg.model,
+        tier,
+        tokenEstimate,
+        attempts: result.attempts,
+        runId: opts.runId,
+        project: opts.project,
+        actor: opts.actor
+      });
+      return {
+        data: {
+          content: result.parsed.content,
+          provider: leg.provider,
+          model: result.parsed.model ?? leg.model,
+          tier,
+          tokens: {
+            input: result.parsed.input,
+            output: result.parsed.output,
+            cacheRead: result.parsed.cacheRead,
+            cacheWrite: result.parsed.cacheWrite
+          },
+          latency: now() - startedAt,
+          attempts: result.attempts,
+          gatewayRequestId: result.gatewayRequestId
         },
-        body: buildAnthropicBody(messages, opts, opts.system)
-      },
-      fetchImpl
-    );
-    const parsed = parseAnthropic(json);
-    return {
-      data: {
-        content: parsed.content,
-        provider: "anthropic",
-        tokens: { input: parsed.input, output: parsed.output },
-        latency: now() - startedAt
-      },
-      error: null
-    };
-  } catch (err) {
-    const status = isProviderError(err) ? err.status : 0;
-    const message = isProviderError(err) ? err.message : err.message;
-    attempts.push({ provider: "anthropic", status, message });
-    if (status !== 0 && !isFailover(status)) {
-      return providerErrorResponse(attempts, status === 429);
+        error: null
+      };
+    } catch (e) {
+      if (e instanceof DOMException && e.name === "AbortError") {
+        return toErrorResponse(
+          new InternalError("llm call aborted", { provider: leg.provider, model: leg.model })
+        );
+      }
+      if (isProviderError(e)) {
+        attemptLog.push({ provider: e.provider, status: e.status, message: e.message });
+        if (e.status === 429 && !route.fallback) {
+          return toErrorResponse(
+            new RateLimitError(`llm rate limited on ${e.provider}`, { attempts: attemptLog })
+          );
+        }
+        logger?.warn?.("llm.leg.failed", { provider: leg.provider, status: e.status });
+        continue;
+      }
+      attemptLog.push({ provider: leg.provider, message: e instanceof Error ? e.message : String(e) });
     }
-    logger?.warn("llm.failover", { from: "anthropic", to: "grok", status, message });
   }
+  return toErrorResponse(
+    new InternalError("LLM_ALL_PROVIDERS_FAILED", { attempts: attemptLog, tier, tokenEstimate })
+  );
+}
+async function* completionStream(messages, env, opts = {}) {
+  if (messages.length === 0) {
+    throw new ValidationError("messages must not be empty");
+  }
+  if (!env.AI_GATEWAY_BASE_URL) {
+    throw new ValidationError("AI_GATEWAY_BASE_URL is required in 0.3.0");
+  }
+  const deps = opts.deps ?? {};
+  const fetchImpl = deps.fetch ?? fetch;
+  const now = deps.now ?? (() => Date.now());
+  const logger = deps.logger;
+  const startedAt = now();
+  const tier = opts.tier ?? "balanced";
+  const system = opts.system ?? messages.find((m) => m.role === "system")?.content;
+  const tokenEstimate = estimateTokens(messages, system);
+  const route = plan(tier, opts, tokenEstimate);
+  if (route.primary.provider !== "anthropic") {
+    const result = await complete(messages, env, opts, deps);
+    if (result.error !== null || result.data === null) {
+      throw new InternalError("LLM_ALL_PROVIDERS_FAILED", { error: result.error });
+    }
+    yield result.data.content;
+    return result.data;
+  }
+  if (isProviderCoolingDown(route.primary.provider, now)) {
+    logger?.warn?.("llm.provider.coolingDown", { provider: route.primary.provider });
+    const result = await complete(messages, env, opts, deps);
+    if (result.error !== null || result.data === null) {
+      throw new InternalError("LLM_ALL_PROVIDERS_FAILED", { error: result.error });
+    }
+    yield result.data.content;
+    return result.data;
+  }
+  const req = buildAnthropicRequest(route.primary.model, messages, opts, env, true);
+  let response;
   try {
-    const json = await callProvider(
-      "grok",
-      {
-        url: "https://api.x.ai/v1/chat/completions",
-        headers: {
-          "content-type": "application/json",
-          authorization: `Bearer ${env.GROK_API_KEY}`
-        },
-        body: buildOpenAIBody(opts.model ?? DEFAULT_GROK_MODEL, messages, opts, opts.system)
-      },
-      fetchImpl
-    );
-    const parsed = parseOpenAI(json);
-    return {
-      data: {
-        content: parsed.content,
-        provider: "grok",
-        tokens: { input: parsed.input, output: parsed.output },
-        latency: now() - startedAt
-      },
-      error: null
-    };
-  } catch (err) {
-    const status = isProviderError(err) ? err.status : 0;
-    const message = isProviderError(err) ? err.message : err.message;
-    attempts.push({ provider: "grok", status, message });
-    logger?.warn("llm.failover", { from: "grok", to: "groq", status, message });
+    response = await fetchImpl(req.url, {
+      method: "POST",
+      headers: req.headers,
+      body: req.body,
+      // Fall back to a 60 s default when the caller provides no signal — prevents
+      // a hung provider connection from consuming the Worker's wall-clock budget.
+      signal: opts.signal ?? AbortSignal.timeout(6e4)
+    });
+  } catch (e) {
+    if (e instanceof DOMException && e.name === "AbortError") {
+      throw new InternalError("llm call aborted", {
+        provider: route.primary.provider,
+        model: route.primary.model
+      });
+    }
+    throw new InternalError("llm stream fetch failed", {
+      message: e instanceof Error ? e.message : String(e)
+    });
   }
+  if (!response.ok) {
+    const text = await response.text().catch(() => "");
+    const retryable = isRetryableForBackoff(response.status);
+    if (retryable && response.status === 429) {
+      markProviderCoolingDown(route.primary.provider, now);
+    }
+    const result = await complete(messages, env, opts, deps);
+    if (result.error !== null || result.data === null) {
+      throw new InternalError("LLM_ALL_PROVIDERS_FAILED", {
+        streamError: `${route.primary.provider} ${String(response.status)}: ${text.slice(0, 300)}`,
+        error: result.error
+      });
+    }
+    yield result.data.content;
+    return result.data;
+  }
+  if (!response.body) {
+    throw new InternalError("llm stream response body is null", {
+      provider: route.primary.provider
+    });
+  }
+  const decoder = new TextDecoder();
+  let accumulatedText = "";
+  let inputTokens = 0;
+  let outputTokens = 0;
+  let cacheRead = 0;
+  let cacheWrite = 0;
+  let modelName;
+  const gatewayRequestId = response.headers.get("cf-aig-request-id") ?? void 0;
+  const reader = response.body.getReader();
+  let buffer = "";
   try {
-    const json = await callProvider(
-      "groq",
-      {
-        url: "https://api.groq.com/openai/v1/chat/completions",
-        headers: {
-          "content-type": "application/json",
-          authorization: `Bearer ${env.GROQ_API_KEY}`
-        },
-        body: buildOpenAIBody(opts.model ?? DEFAULT_GROQ_MODEL, messages, opts, opts.system)
-      },
-      fetchImpl
-    );
-    const parsed = parseOpenAI(json);
-    return {
-      data: {
-        content: parsed.content,
-        provider: "groq",
-        tokens: { input: parsed.input, output: parsed.output },
-        latency: now() - startedAt
-      },
-      error: null
-    };
-  } catch (err) {
-    const status = isProviderError(err) ? err.status : 0;
-    const message = isProviderError(err) ? err.message : err.message;
-    attempts.push({ provider: "groq", status, message });
-    logger?.error("llm.all_providers_failed", void 0, { attempts });
-  }
-  return providerErrorResponse(attempts, false);
-}
-function providerErrorResponse(attempts, rateLimited) {
-  const base = rateLimited ? new RateLimitError("LLM provider rate limited", { code: ErrorCodes.LLM_RATE_LIMITED, attempts }) : new InternalError("All LLM providers failed", {
-    code: ErrorCodes.LLM_ALL_PROVIDERS_FAILED,
-    attempts
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      buffer += decoder.decode(value, { stream: true });
+      const lines = buffer.split("\n");
+      buffer = lines.pop() ?? "";
+      for (const line of lines) {
+        if (!line.startsWith("data: ")) continue;
+        const data = line.slice(6).trim();
+        if (data === "[DONE]") break;
+        let event;
+        try {
+          event = JSON.parse(data);
+        } catch {
+          continue;
+        }
+        switch (event.type) {
+          case "message_start":
+            inputTokens = event.message?.usage?.input_tokens ?? 0;
+            cacheRead = event.message?.usage?.cache_read_input_tokens ?? 0;
+            cacheWrite = event.message?.usage?.cache_creation_input_tokens ?? 0;
+            modelName = event.message?.model;
+            break;
+          case "content_block_delta":
+            if (event.delta?.type === "text_delta" && typeof event.delta.text === "string") {
+              accumulatedText += event.delta.text;
+              yield event.delta.text;
+            }
+            break;
+          case "message_delta":
+            outputTokens = event.usage?.output_tokens ?? outputTokens;
+            break;
+          default:
+            break;
+        }
+      }
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  clearProviderCooldown(route.primary.provider);
+  logger?.info?.("llm.completionStream", {
+    provider: route.primary.provider,
+    model: route.primary.model,
+    tier,
+    tokenEstimate,
+    runId: opts.runId,
+    project: opts.project,
+    actor: opts.actor
   });
   return {
-    data: null,
-    error: {
-      code: rateLimited ? ErrorCodes.LLM_RATE_LIMITED : ErrorCodes.LLM_ALL_PROVIDERS_FAILED,
-      message: base.message,
-      status: base.status,
-      retryable: base.retryable,
-      context: base.context
-    }
+    content: accumulatedText,
+    provider: route.primary.provider,
+    model: modelName ?? route.primary.model,
+    tier,
+    tokens: { input: inputTokens, output: outputTokens, cacheRead, cacheWrite },
+    latency: now() - startedAt,
+    attempts: 1,
+    gatewayRequestId
   };
 }
-async function stream(messages, env, opts = {}, deps = {}) {
-  if (messages.length === 0) {
-    throw new ValidationError("messages must not be empty");
+function assertGrounding(response, sources) {
+  if (sources.length === 0) return true;
+  const WINDOW = 5;
+  const responseTokens = response.split(/\s+/).filter((t) => t.length > 0);
+  if (responseTokens.length < WINDOW) return false;
+  const sourceNgrams = /* @__PURE__ */ new Set();
+  for (const source of sources) {
+    const tokens = source.split(/\s+/).filter((t) => t.length > 0);
+    for (let i = 0; i <= tokens.length - WINDOW; i++) {
+      const ngram = tokens.slice(i, i + WINDOW).join(" ");
+      sourceNgrams.add(ngram);
+    }
   }
-  const fetchImpl = deps.fetch ?? fetch;
-  const body = JSON.parse(buildAnthropicBody(messages, opts, opts.system));
-  body.stream = true;
-  const response = await fetchImpl("https://api.anthropic.com/v1/messages", {
-    method: "POST",
-    headers: {
-      "content-type": "application/json",
-      "x-api-key": env.ANTHROPIC_API_KEY,
-      "anthropic-version": "2023-06-01"
-    },
-    body: JSON.stringify(body)
-  });
-  if (!response.ok || !response.body) {
-    throw new InternalError("Anthropic stream failed", {
-      code: ErrorCodes.LLM_ALL_PROVIDERS_FAILED,
-      status: response.status
-    });
+  if (sourceNgrams.size === 0) return false;
+  for (let i = 0; i <= responseTokens.length - WINDOW; i++) {
+    const ngram = responseTokens.slice(i, i + WINDOW).join(" ");
+    if (sourceNgrams.has(ngram)) return true;
   }
-  return response.body;
-}
-function withSystem(system) {
-  return (messages, env, opts = {}, deps = {}) => complete(messages, env, { ...opts, system }, deps);
+  return false;
 }
 export {
+  BASE_BACKOFF_MS,
+  MODELS,
+  PROVIDER_COOLDOWN_MS,
+  assertGrounding,
+  clearProviderCooldown,
   complete,
-  stream,
-  withSystem
+  completionStream,
+  isProviderCoolingDown,
+  markProviderCoolingDown
 };
 //# sourceMappingURL=index.mjs.map