@latimer-woods-tech/llm 0.5.0 → 0.6.1

package/dist/index.mjs

@@ -1,12 +1,143 @@
 // src/index.ts
 import {
-  InternalError,
+  InternalError as InternalError3,
   RateLimitError,
-  ValidationError,
+  ValidationError as ValidationError2,
   toErrorResponse
 } from "@latimer-woods-tech/errors";
 
+// src/gcp-token.ts
+import { InternalError, ValidationError } from "@latimer-woods-tech/errors";
+var EXPIRY_SKEW_MS = 3e5;
+var tokenCache = /* @__PURE__ */ new Map();
+function parseServiceAccountKey(gcpSaKey) {
+  const text = gcpSaKey.trimStart().startsWith("{") ? gcpSaKey : atob(gcpSaKey);
+  const key = JSON.parse(text);
+  if (!key.client_email || !key.private_key || !key.token_uri) {
+    throw new ValidationError("GCP_SA_KEY is missing client_email, private_key, or token_uri");
+  }
+  return key;
+}
+function base64UrlEncode(bytes) {
+  let binary = "";
+  for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
+  return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/g, "");
+}
+async function importPrivateKey(pem) {
+  const body = pem.split("\n").filter((line) => !line.startsWith("-----")).join("");
+  const binary = atob(body);
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+  return crypto.subtle.importKey(
+    "pkcs8",
+    bytes.buffer,
+    { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" },
+    false,
+    ["sign"]
+  );
+}
+async function createAssertion(key, nowSeconds) {
+  const encoder = new TextEncoder();
+  const header = base64UrlEncode(encoder.encode(JSON.stringify({ alg: "RS256", typ: "JWT" })));
+  const payload = base64UrlEncode(
+    encoder.encode(
+      JSON.stringify({
+        iss: key.client_email,
+        scope: "https://www.googleapis.com/auth/cloud-platform",
+        aud: key.token_uri,
+        exp: nowSeconds + 3600,
+        iat: nowSeconds
+      })
+    )
+  );
+  const signingInput = `${header}.${payload}`;
+  const cryptoKey = await importPrivateKey(key.private_key);
+  const signature = await crypto.subtle.sign(
+    "RSASSA-PKCS1-v1_5",
+    cryptoKey,
+    encoder.encode(signingInput).buffer
+  );
+  return `${signingInput}.${base64UrlEncode(new Uint8Array(signature))}`;
+}
+async function mintGcpAccessToken(gcpSaKey, fetchImpl) {
+  const key = parseServiceAccountKey(gcpSaKey);
+  const now = Date.now();
+  const cached = tokenCache.get(key.client_email);
+  if (cached && cached.expiresAt - EXPIRY_SKEW_MS > now) return cached.token;
+  const assertion = await createAssertion(key, Math.floor(now / 1e3));
+  const response = await fetchImpl(key.token_uri, {
+    method: "POST",
+    headers: { "content-type": "application/x-www-form-urlencoded" },
+    body: new URLSearchParams({
+      grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
+      assertion
+    }).toString()
+  }).catch((cause) => {
+    throw new InternalError(`GCP token exchange failed: ${String(cause)}`);
+  });
+  if (!response.ok) {
+    throw new InternalError(`GCP token exchange returned ${response.status}`);
+  }
+  const data = await response.json();
+  if (!data.access_token) {
+    throw new InternalError("GCP token exchange returned no access_token");
+  }
+  const ttlMs = (data.expires_in > 0 ? data.expires_in : 3600) * 1e3;
+  tokenCache.set(key.client_email, { token: data.access_token, expiresAt: now + ttlMs });
+  return data.access_token;
+}
+var METADATA_BASE = "http://metadata.google.internal/computeMetadata/v1";
+var METADATA_HEADER = { "Metadata-Flavor": "Google" };
+var METADATA_TIMEOUT_MS = 2e3;
+var adcCache;
+async function fetchAdcAccessToken(fetchImpl) {
+  const now = Date.now();
+  if (adcCache && adcCache.expiresAt - EXPIRY_SKEW_MS > now) {
+    return { token: adcCache.token, project: adcCache.project };
+  }
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), METADATA_TIMEOUT_MS);
+  try {
+    const tokenRes = await fetchImpl(
+      `${METADATA_BASE}/instance/service-accounts/default/token`,
+      { headers: METADATA_HEADER, signal: controller.signal }
+    );
+    if (!tokenRes.ok) {
+      throw new InternalError(`GCP metadata token endpoint returned ${tokenRes.status}`);
+    }
+    const data = await tokenRes.json();
+    if (!data.access_token) {
+      throw new InternalError("GCP metadata token endpoint returned no access_token");
+    }
+    let project;
+    try {
+      const projRes = await fetchImpl(`${METADATA_BASE}/project/project-id`, {
+        headers: METADATA_HEADER,
+        signal: controller.signal
+      });
+      if (projRes.ok) project = (await projRes.text()).trim() || void 0;
+    } catch {
+    }
+    const ttlMs = (data.expires_in > 0 ? data.expires_in : 3600) * 1e3;
+    adcCache = { token: data.access_token, project, expiresAt: now + ttlMs };
+    return { token: data.access_token, project };
+  } catch (cause) {
+    if (cause instanceof InternalError) throw cause;
+    throw new InternalError(`GCP metadata credential unavailable: ${String(cause)}`);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+function clearGcpTokenCache() {
+  tokenCache.clear();
+  adcCache = void 0;
+}
+function serviceAccountProjectId(gcpSaKey) {
+  return parseServiceAccountKey(gcpSaKey).project_id;
+}
+
 // src/embed.ts
+import { InternalError as InternalError2 } from "@latimer-woods-tech/errors";
 var DEFAULT_EMBEDDING_MODEL = "@cf/baai/bge-base-en-v1.5";
 async function embed(ai, input, opts) {
   const model = opts?.model ?? DEFAULT_EMBEDDING_MODEL;
@@ -22,6 +153,41 @@ async function embed(ai, input, opts) {
     dims: vectors[0].length
   };
 }
+var LOCAL_EMBEDDING_MODEL = "nomic-embed-text";
+async function embedLocal(env, input) {
+  const texts = Array.isArray(input) ? input : [input];
+  const res = await fetch(
+    `${env.AI_GATEWAY_BASE_URL}/custom-local-gpu/v1/embeddings`,
+    {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        // CF Access service-token headers only sent when present (bearer-only rails still work).
+        ...env.GPU_LLM_API_TOKEN ? { Authorization: `Bearer ${env.GPU_LLM_API_TOKEN}` } : {},
+        ...env.GPU_LLM_ACCESS_CLIENT_ID ? { "CF-Access-Client-Id": env.GPU_LLM_ACCESS_CLIENT_ID } : {},
+        ...env.GPU_LLM_ACCESS_CLIENT_SECRET ? { "CF-Access-Client-Secret": env.GPU_LLM_ACCESS_CLIENT_SECRET } : {}
+      },
+      body: JSON.stringify({ model: LOCAL_EMBEDDING_MODEL, input: texts })
+    }
+  );
+  if (!res.ok) {
+    throw new InternalError2(
+      `embedLocal(): rail ${res.status}: ${(await res.text()).slice(0, 160)}`
+    );
+  }
+  const json = await res.json();
+  const vectors = (json.data ?? []).map((d) => d.embedding);
+  if (vectors.length === 0) {
+    throw new InternalError2(
+      `embedLocal(): rail returned no vectors for model ${LOCAL_EMBEDDING_MODEL}`
+    );
+  }
+  return {
+    vectors,
+    model: LOCAL_EMBEDDING_MODEL,
+    dims: vectors[0].length
+  };
+}
 
 // src/index.ts
 function contentToText(content) {
@@ -35,25 +201,60 @@ function systemText(opts, messages) {
 }
 var MODELS = {
   anthropic: {
-    fast: "claude-haiku-4-20250514",
-    balanced: "claude-sonnet-4-6",
-    smart: "claude-opus-4-7"
+    // `claude-haiku-4-20250514` was never a model Anthropic served — it 404s
+    // (verified live against /v1/messages). Date suffixes are never appended to
+    // an alias; the alias is `claude-haiku-4-5`, which resolves server-side to
+    // `claude-haiku-4-5-20251001`. This is the `fast` tier's FALLBACK, so the
+    // 404 stayed invisible for as long as the Grok primary held.
+    fast: "claude-haiku-4-5",
+    // Current-generation Sonnet/Opus (2026-08 refresh). The previous pins —
+    // `claude-sonnet-4-6` / `claude-opus-4-7` — were a generation behind, and
+    // the opus-4-7 leg was HARD-BROKEN: Claude 4.7+ rejects `temperature`
+    // (400 "`temperature` is deprecated for this model", verified live
+    // 2026-08-14) and buildAnthropicRequest always sent one, so every `smart`
+    // call 400'd on its Anthropic primary and silently served Gemini Flash.
+    // Sonnet 5 / Opus 5 cost the same list price or less than the models they
+    // replace ($3/$15, $5/$25 per 1M). The sampling/thinking request-shape
+    // differences these models introduce are handled in buildAnthropicRequest.
+    balanced: "claude-sonnet-5",
+    smart: "claude-opus-5"
   },
   gemini: {
-    smart: "gemini-2.5-pro"
+    // `gemini-2.5-flash`, not `-pro`: the leg's job here is a fast, reliable,
+    // JSON-returning fallback. Gemini 2.5 *Pro* mandates a thinking budget of
+    // ≥128 tokens that is drawn from `maxOutputTokens` and CANNOT be disabled
+    // (thinkingBudget=0 is rejected). On the render-runner's large judge /
+    // generation prompts — and any low-`maxTokens` call (headline uses 40) —
+    // the thinking phase exhausts the whole budget and Vertex returns 200 with
+    // an empty candidate (finishReason MAX_TOKENS, no text), which the router
+    // treats as a failed leg. Flash supports `thinkingBudget: 0` (set in
+    // buildGeminiRequest), so text is always emitted. Both are Vertex-served.
+    smart: "gemini-2.5-flash"
   },
   groq: {
-    verifier: "llama-4-maverick"
+    // `llama-4-maverick` was NOT a model Groq serves — every `verifier` call
+    // 404'd (the tier has no fallback, so it was hard-broken), and it was also
+    // the `workbench` fallback. Groq's llama-4 offering is `scout`, which is
+    // blocked at our org level; `llama-3.3-70b-versatile` is served and
+    // unblocked (verified against /v1/models and a live completion).
+    verifier: "llama-3.3-70b-versatile"
   },
   grok: {
     fast: "grok-4.3"
   },
   deepseek: {
     workbench: "deepseek-chat"
+  },
+  local: {
+    // Self-hosted qwen3:8b on the dedicated fast GPU, reached through the
+    // `custom-local-fast-gpu` AI Gateway provider's native Ollama chat route.
+    fast: "qwen3:8b",
+    workbench: "qwen3.6:27b"
   }
 };
 var DEFAULT_MAX_TOKENS = 1024;
 var DEFAULT_TEMPERATURE = 0.7;
+var DEFAULT_VERTEX_LOCATION = "us-central1";
 var DEFAULT_LONG_CONTEXT_THRESHOLD = 15e4;
 var BACKOFF_BASE_MS = 500;
 var BACKOFF_CAP_MS = 8e3;
@@ -88,19 +289,33 @@ async function recordOrgCostUsage(kv, todayKey, monthKey, costUsd, opts) {
   }
 }
 var MODEL_PRICE_PER_1M = {
-  // Anthropic Haiku 4
-  "claude-haiku-4-20250514": { input: 0.8, output: 4, cacheRead: 0.08, cacheWrite: 1 },
-  "claude-haiku-4-5-20251001": { input: 0.8, output: 4, cacheRead: 0.08, cacheWrite: 1 },
+  // Anthropic Haiku 4.5 — `claude-haiku-4-5` is the routed alias; the dated id
+  // is what the API echoes back in `response.model`, so both must price.
+  // (Rates corrected 2026-08: the previous $0.80/$4.00 rows UNDERSTATED the
+  // actual $1/$5 list price.)
+  "claude-haiku-4-5": { input: 1, output: 5, cacheRead: 0.1, cacheWrite: 1.25 },
+  "claude-haiku-4-5-20251001": { input: 1, output: 5, cacheRead: 0.1, cacheWrite: 1.25 },
   // Anthropic Sonnet 4
   "claude-sonnet-4-20250514": { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 },
   "claude-sonnet-4-6": { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 },
-  // Anthropic Opus 4
+  // Anthropic Sonnet 5 — the routed `balanced` primary. List $3/$15; intro
+  // pricing ($2/$10) runs through 2026-08-31 — priced at list here, the
+  // conservative upper bound (table convention).
+  "claude-sonnet-5": { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 },
+  // Anthropic Opus 4 (dated id) — genuinely the $15/$75 era.
   "claude-opus-4-20250514": { input: 15, output: 75, cacheRead: 1.5, cacheWrite: 18.75 },
-  "claude-opus-4-7": { input: 15, output: 75, cacheRead: 1.5, cacheWrite: 18.75 },
-  // Gemini 2.5 Pro
+  // Opus 4.7 was NEVER $15/$75 — it launched at $5/$25. The old row copied the
+  // Opus 4 rate and overstated every smart-tier ledger entry 3x.
+  "claude-opus-4-7": { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 },
+  // Anthropic Opus 5 — the routed `smart` primary. Same $5/$25 as Opus 4.8/4.7.
+  "claude-opus-5": { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 },
+  // Gemini 2.5 Flash — the routed `smart`/long-context model (JSON fallback leg).
+  "gemini-2.5-flash": { input: 0.3, output: 2.5, cacheRead: 0.075, cacheWrite: 0.3 },
+  // Gemini 2.5 Pro — retained for historical ledger rows (was the routed model).
   "gemini-2.5-pro": { input: 1.25, output: 10, cacheRead: 0.31, cacheWrite: 4.5 },
-  // Groq Llama 4 Maverick
-  "llama-4-maverick": { input: 0.5, output: 0.77, cacheRead: 0.05, cacheWrite: 0.5 },
+  // Groq Llama 3.3 70B Versatile (`verifier` tier). Groq has no prompt caching,
+  // so cache rates are 0.00 — same convention as grok-4.3 below.
+  "llama-3.3-70b-versatile": { input: 0.59, output: 0.79, cacheRead: 0, cacheWrite: 0 },
   // Grok 4.3
   "grok-4.3": { input: 1.25, output: 2.5, cacheRead: 0, cacheWrite: 0 },
   // DeepSeek API pricing as of 2026-05: cache-write conservatively uses cache-miss input pricing.
@@ -108,7 +323,10 @@ var MODEL_PRICE_PER_1M = {
   "deepseek-reasoner": { input: 0.55, output: 2.19, cacheRead: 0.14, cacheWrite: 0.55 },
   // Deprecated aliases retained for historical ledger rows.
   "grok-4-fast": { input: 1.25, output: 2.5, cacheRead: 0, cacheWrite: 0 },
-  "grok-3-mini-latest": { input: 1.25, output: 2.5, cacheRead: 0, cacheWrite: 0 }
+  "grok-3-mini-latest": { input: 1.25, output: 2.5, cacheRead: 0, cacheWrite: 0 },
+  // Self-hosted qwen3 on the GPU box — zero marginal cost (electricity aside).
+  "qwen3:8b": { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+  "qwen3.6:27b": { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }
 };
 var PRICE_FALLBACK = MODEL_PRICE_PER_1M["claude-opus-4-7"];
 function estimateCostUsd(tokens, model) {
@@ -150,15 +368,22 @@ function computeBackoffMs(attempt) {
   const jitter = Math.floor(Math.random() * BACKOFF_JITTER_MAX_MS);
   return Math.min(BACKOFF_BASE_MS * Math.pow(2, attempt) + jitter, BACKOFF_CAP_MS);
 }
+var ANTHROPIC_SAMPLING_REMOVED = /^claude-(opus-4-[78]|opus-5|sonnet-5|fable-5|mythos-5)/;
+var ANTHROPIC_THINKING_DEFAULT_ON = /^claude-(opus-5|sonnet-5)$/;
 function buildAnthropicRequest(model, messages, opts, env, streaming = false) {
   const sys = systemText(opts, messages);
   const filtered = messages.filter((m) => m.role !== "system");
   const body = {
     model,
     max_tokens: opts.maxTokens ?? DEFAULT_MAX_TOKENS,
-    temperature: opts.temperature ?? DEFAULT_TEMPERATURE,
     messages: filtered.map((m) => ({ role: m.role, content: m.content }))
   };
+  if (!ANTHROPIC_SAMPLING_REMOVED.test(model)) {
+    body.temperature = opts.temperature ?? DEFAULT_TEMPERATURE;
+  }
+  if (ANTHROPIC_THINKING_DEFAULT_ON.test(model)) {
+    body.thinking = { type: "disabled" };
+  }
   if (streaming) {
     body.stream = true;
   }
@@ -186,7 +411,18 @@ function buildAnthropicRequest(model, messages, opts, env, streaming = false) {
     body: JSON.stringify(body)
   };
 }
-function buildGeminiRequest(model, messages, opts, env) {
+async function resolveVertexAuth(env, fetchImpl) {
+  if (env.GCP_SA_KEY) {
+    try {
+      return { token: await mintGcpAccessToken(env.GCP_SA_KEY, fetchImpl) };
+    } catch (error) {
+      if (!env.VERTEX_ACCESS_TOKEN) throw error;
+    }
+  }
+  if (env.VERTEX_ACCESS_TOKEN) return { token: env.VERTEX_ACCESS_TOKEN };
+  return fetchAdcAccessToken(fetchImpl);
+}
+function buildGeminiRequest(model, messages, opts, env, accessToken, adcProject) {
   const sys = systemText(opts, messages);
   const contents = messages.filter((m) => m.role !== "system").map((m) => ({
     role: m.role === "assistant" ? "model" : "user",
@@ -196,18 +432,26 @@ function buildGeminiRequest(model, messages, opts, env) {
     contents,
     generationConfig: {
       maxOutputTokens: opts.maxTokens ?? DEFAULT_MAX_TOKENS,
-      temperature: opts.temperature ?? DEFAULT_TEMPERATURE
+      temperature: opts.temperature ?? DEFAULT_TEMPERATURE,
+      // Disable "thinking": Gemini 2.5 draws thinking tokens from
+      // maxOutputTokens, so on a large prompt (or a small token budget) the
+      // model can spend the entire budget thinking and return an empty
+      // candidate (finishReason MAX_TOKENS). This leg wants deterministic text
+      // out, so thinking is turned off (supported by gemini-2.5-flash).
+      thinkingConfig: { thinkingBudget: 0 }
     }
   };
   if (sys) {
     body.systemInstruction = { parts: [{ text: sys }] };
   }
-  const path = `v1/projects/${env.VERTEX_PROJECT}/locations/${env.VERTEX_LOCATION}/publishers/google/models/${model}:generateContent`;
+  const project = env.VERTEX_PROJECT || (env.GCP_SA_KEY ? serviceAccountProjectId(env.GCP_SA_KEY) : adcProject ?? "");
+  const location = env.VERTEX_LOCATION || DEFAULT_VERTEX_LOCATION;
+  const path = `v1/projects/${project}/locations/${location}/publishers/google/models/${model}:generateContent`;
   return {
     url: `${env.AI_GATEWAY_BASE_URL}/google-vertex-ai/${path}`,
     headers: {
       "content-type": "application/json",
-      authorization: `Bearer ${env.VERTEX_ACCESS_TOKEN}`
+      authorization: `Bearer ${accessToken}`
     },
     body: JSON.stringify(body)
   };
@@ -241,6 +485,53 @@ function toOpenAiMessages(messages, sys) {
   }
   return out;
 }
+function toOllamaMessages(messages, sys) {
+  const toolNames = /* @__PURE__ */ new Map();
+  for (const message of messages) {
+    if (typeof message.content === "string") continue;
+    for (const block of message.content) {
+      if (block.type === "tool_use") toolNames.set(block.id, block.name);
+    }
+  }
+  const out = [];
+  if (sys) out.push({ role: "system", content: sys });
+  for (const message of messages) {
+    if (message.role === "system") continue;
+    if (typeof message.content === "string") {
+      out.push({ role: message.role, content: message.content });
+      continue;
+    }
+    let text = "";
+    const toolCalls = [];
+    const results = [];
+    for (const block of message.content) {
+      if (block.type === "text") text += block.text;
+      else if (block.type === "tool_use") {
+        toolCalls.push({
+          type: "function",
+          function: { index: toolCalls.length, name: block.name, arguments: block.input }
+        });
+      } else if (block.type === "tool_result") {
+        results.push({ toolUseId: block.tool_use_id, content: block.content });
+      }
+    }
+    if (results.length > 0) {
+      for (const result of results) {
+        out.push({
+          role: "tool",
+          tool_name: toolNames.get(result.toolUseId) ?? result.toolUseId,
+          content: result.content
+        });
+      }
+      if (text) out.push({ role: "user", content: text });
+    } else if (toolCalls.length > 0) {
+      out.push({ role: "assistant", content: text, tool_calls: toolCalls });
+    } else {
+      out.push({ role: message.role, content: text });
+    }
+  }
+  return out;
+}
 function openAiTools(opts) {
   if (!opts.tools || opts.tools.length === 0) return void 0;
   return opts.tools.map((t) => ({
@@ -274,7 +565,7 @@ function buildGroqRequest(model, messages, opts, env) {
 }
 function buildGrokRequest(model, messages, opts, env) {
   if (!env.GROK_API_KEY) {
-    throw new ValidationError("GROK_API_KEY required for grok-* model override");
+    throw new ValidationError2("GROK_API_KEY required for grok-* model override");
   }
   const sys = systemText(opts, messages);
   const body = {
@@ -303,7 +594,7 @@ function buildGrokRequest(model, messages, opts, env) {
 }
 function buildDeepSeekRequest(model, messages, opts, env) {
   if (!env.DEEPSEEK_API_KEY) {
-    throw new ValidationError("DEEPSEEK_API_KEY required for workbench tier or deepseek-* model override");
+    throw new ValidationError2("DEEPSEEK_API_KEY required for workbench tier or deepseek-* model override");
   }
   const sys = systemText(opts, messages);
   const body = {
@@ -327,6 +618,51 @@ function buildDeepSeekRequest(model, messages, opts, env) {
     body: JSON.stringify(body)
   };
 }
+function buildLocalRequest(model, messages, opts, env) {
+  if (!env.GPU_LLM_API_TOKEN) {
+    throw new ValidationError2("GPU_LLM_API_TOKEN required for the local provider");
+  }
+  const sys = systemText(opts, messages);
+  const fastMode = model === MODELS.local.fast;
+  const headers = {
+    "content-type": "application/json",
+    authorization: `Bearer ${env.GPU_LLM_API_TOKEN}`
+  };
+  if (env.GPU_LLM_ACCESS_CLIENT_ID) headers["CF-Access-Client-Id"] = env.GPU_LLM_ACCESS_CLIENT_ID;
+  if (env.GPU_LLM_ACCESS_CLIENT_SECRET) headers["CF-Access-Client-Secret"] = env.GPU_LLM_ACCESS_CLIENT_SECRET;
+  const tools = openAiTools(opts);
+  if (fastMode) {
+    if (typeof opts.toolChoice === "object") {
+      throw new ValidationError2("named tool choice is not supported by the native Qwen 8B route");
+    }
+    return {
+      url: `${env.AI_GATEWAY_BASE_URL}/custom-local-fast-gpu/api/chat`,
+      headers,
+      body: JSON.stringify({
+        model,
+        stream: false,
+        think: false,
+        messages: toOllamaMessages(messages, sys),
+        options: {
+          num_predict: opts.maxTokens ?? DEFAULT_MAX_TOKENS,
+          temperature: opts.temperature ?? DEFAULT_TEMPERATURE
+        },
+        ...tools && opts.toolChoice !== "none" ? { tools } : {}
+      })
+    };
+  }
+  return {
+    url: `${env.AI_GATEWAY_BASE_URL}/custom-local-gpu/v1/chat/completions`,
+    headers,
+    body: JSON.stringify({
+      model,
+      max_tokens: opts.maxTokens ?? DEFAULT_MAX_TOKENS,
+      temperature: opts.temperature ?? DEFAULT_TEMPERATURE,
+      messages: toOpenAiMessages(messages, sys),
+      ...tools ? { tools, tool_choice: openAiToolChoice(opts.toolChoice ?? "auto") } : {}
+    })
+  };
+}
 function normalizeAnthropicStop(reason) {
   switch (reason) {
     case "end_turn":
@@ -411,6 +747,29 @@ function parseOpenAi(json) {
     stopReason: normalizeOpenAiStop(choice?.finish_reason)
   };
 }
+function parseOllamaChat(json) {
+  const response = json;
+  const toolCalls = (response.message?.tool_calls ?? []).filter((call) => typeof call.function?.name === "string").map((call, index) => ({
+    id: `call_${index}`,
+    name: call.function.name,
+    arguments: typeof call.function?.arguments === "string" ? parseToolArgs(call.function.arguments) : call.function?.arguments ?? {}
+  }));
+  let stopReason;
+  if (toolCalls.length > 0) stopReason = "tool_use";
+  else if (response.done_reason === "length") stopReason = "max_tokens";
+  else if (response.done) stopReason = "end";
+  else if (response.done_reason) stopReason = "other";
+  return {
+    // Deliberately exclude message.thinking: callers receive only the bounded
+    // classification answer even if a non-conforming origin returns a trace.
+    content: response.message?.content ?? "",
+    input: response.prompt_eval_count ?? 0,
+    output: response.eval_count ?? 0,
+    model: response.model,
+    toolCalls: toolCalls.length > 0 ? toolCalls : void 0,
+    stopReason
+  };
+}
 async function callWithBackoff(provider, request, fetchImpl, signal, logger, nowFn) {
   function exhaustAndThrow(err) {
     markProviderCoolingDown(provider, nowFn ?? Date.now);
@@ -474,7 +833,7 @@ async function callWithBackoff(provider, request, fetchImpl, signal, logger, now
 function isProviderError(err) {
   return typeof err === "object" && err !== null && typeof err.status === "number" && typeof err.message === "string" && typeof err.provider === "string";
 }
-var TOOL_CAPABLE_PROVIDERS = /* @__PURE__ */ new Set(["anthropic", "grok", "deepseek"]);
+var TOOL_CAPABLE_PROVIDERS = /* @__PURE__ */ new Set(["anthropic", "grok", "deepseek", "local"]);
 function plan(tier, opts, tokenEstimate) {
   if (opts.model) {
     const m = opts.model;
@@ -482,6 +841,7 @@ function plan(tier, opts, tokenEstimate) {
     if (m.startsWith("gemini")) return { primary: { provider: "gemini", model: m } };
     if (m.startsWith("grok")) return { primary: { provider: "grok", model: m } };
     if (m.startsWith("deepseek")) return { primary: { provider: "deepseek", model: m } };
+    if (m.startsWith("qwen")) return { primary: { provider: "local", model: m } };
     return { primary: { provider: "groq", model: m } };
   }
   const longContext = tokenEstimate >= (opts.longContextThreshold ?? DEFAULT_LONG_CONTEXT_THRESHOLD);
@@ -531,9 +891,11 @@ async function callOne(leg, messages, opts, env, fetchImpl, logger, nowFn) {
     case "anthropic":
       req = buildAnthropicRequest(leg.model, messages, opts, env);
       break;
-    case "gemini":
-      req = buildGeminiRequest(leg.model, messages, opts, env);
+    case "gemini": {
+      const vertexAuth = await resolveVertexAuth(env, fetchImpl);
+      req = buildGeminiRequest(leg.model, messages, opts, env, vertexAuth.token, vertexAuth.project);
       break;
+    }
     case "groq":
       req = buildGroqRequest(leg.model, messages, opts, env);
       break;
@@ -543,6 +905,9 @@ async function callOne(leg, messages, opts, env, fetchImpl, logger, nowFn) {
     case "deepseek":
       req = buildDeepSeekRequest(leg.model, messages, opts, env);
       break;
+    case "local":
+      req = buildLocalRequest(leg.model, messages, opts, env);
+      break;
   }
   const aigMetadata = buildAigMetadata(opts);
   if (aigMetadata) req.headers["cf-aig-metadata"] = aigMetadata;
@@ -565,14 +930,20 @@ async function callOne(leg, messages, opts, env, fetchImpl, logger, nowFn) {
       return { parsed: parseOpenAi(json), gatewayRequestId, attempts };
     case "deepseek":
       return { parsed: parseOpenAi(json), gatewayRequestId, attempts };
+    case "local":
+      return {
+        parsed: leg.model === MODELS.local.fast ? parseOllamaChat(json) : parseOpenAi(json),
+        gatewayRequestId,
+        attempts
+      };
   }
 }
 async function complete(messages, env, opts = {}, deps = {}) {
   if (messages.length === 0) {
-    throw new ValidationError("messages must not be empty");
+    throw new ValidationError2("messages must not be empty");
   }
   if (!env.AI_GATEWAY_BASE_URL) {
-    throw new ValidationError("AI_GATEWAY_BASE_URL is required in 0.3.0");
+    throw new ValidationError2("AI_GATEWAY_BASE_URL is required in 0.3.0");
   }
   const fetchImpl = deps.fetch ?? fetch;
   const now = deps.now ?? (() => Date.now());
@@ -581,7 +952,13 @@ async function complete(messages, env, opts = {}, deps = {}) {
   const tier = opts.tier ?? "balanced";
   const system = systemText(opts, messages);
   const tokenEstimate = estimateTokens(messages, system);
-  const route = plan(tier, opts, tokenEstimate);
+  let route = plan(tier, opts, tokenEstimate);
+  if (env.LLM_LOCAL_FIRST && tier === "fast" && !opts.model && env.GPU_LLM_API_TOKEN) {
+    route = { primary: { provider: "local", model: MODELS.local.fast }, fallback: route.primary };
+  }
+  if (env.LLM_LOCAL_WORKBENCH && tier === "workbench" && !opts.model && env.GPU_LLM_API_TOKEN) {
+    route = { primary: { provider: "local", model: MODELS.local.workbench }, fallback: route.primary };
+  }
   const kv = env.LLM_COST_KV;
   const todayKey = `llm:daily-cost:${isoDate(now())}`;
   const monthKey = `llm:monthly-cost:${isoMonth(now())}`;
@@ -616,7 +993,7 @@ async function complete(messages, env, opts = {}, deps = {}) {
   if (opts.tools && opts.tools.length > 0) {
     routeLegs = routeLegs.filter((l) => TOOL_CAPABLE_PROVIDERS.has(l.provider));
     if (routeLegs.length === 0) {
-      throw new ValidationError(
+      throw new ValidationError2(
         `tool-calling requires a tool-capable provider (${[...TOOL_CAPABLE_PROVIDERS].join(", ")}); tier '${tier}' has none \u2014 use tier fast/balanced/smart or a claude-* model override`
       );
     }
@@ -629,7 +1006,7 @@ async function complete(messages, env, opts = {}, deps = {}) {
     }
     if (opts.signal?.aborted) {
       return toErrorResponse(
-        new InternalError("llm call aborted", { provider: leg.provider, model: leg.model })
+        new InternalError3("llm call aborted", { provider: leg.provider, model: leg.model })
       );
     }
     try {
@@ -704,7 +1081,7 @@ async function complete(messages, env, opts = {}, deps = {}) {
     } catch (e) {
       if (e instanceof DOMException && e.name === "AbortError") {
         return toErrorResponse(
-          new InternalError("llm call aborted", { provider: leg.provider, model: leg.model })
+          new InternalError3("llm call aborted", { provider: leg.provider, model: leg.model })
         );
       }
       if (isProviderError(e)) {
@@ -721,15 +1098,15 @@ async function complete(messages, env, opts = {}, deps = {}) {
     }
   }
   return toErrorResponse(
-    new InternalError("LLM_ALL_PROVIDERS_FAILED", { attempts: attemptLog, tier, tokenEstimate })
+    new InternalError3("LLM_ALL_PROVIDERS_FAILED", { attempts: attemptLog, tier, tokenEstimate })
   );
 }
 async function* completionStream(messages, env, opts = {}) {
   if (messages.length === 0) {
-    throw new ValidationError("messages must not be empty");
+    throw new ValidationError2("messages must not be empty");
   }
   if (!env.AI_GATEWAY_BASE_URL) {
-    throw new ValidationError("AI_GATEWAY_BASE_URL is required in 0.3.0");
+    throw new ValidationError2("AI_GATEWAY_BASE_URL is required in 0.3.0");
   }
   const deps = opts.deps ?? {};
   const fetchImpl = deps.fetch ?? fetch;
@@ -744,7 +1121,7 @@ async function* completionStream(messages, env, opts = {}) {
   if (streamLeg.provider !== "anthropic") {
     const result = await complete(messages, env, opts, deps);
     if (result.error !== null || result.data === null) {
-      throw new InternalError("LLM_ALL_PROVIDERS_FAILED", { error: result.error });
+      throw new InternalError3("LLM_ALL_PROVIDERS_FAILED", { error: result.error });
     }
     yield result.data.content;
     return result.data;
@@ -753,7 +1130,7 @@ async function* completionStream(messages, env, opts = {}) {
     logger?.warn?.("llm.provider.coolingDown", { provider: streamLeg.provider });
     const result = await complete(messages, env, opts, deps);
     if (result.error !== null || result.data === null) {
-      throw new InternalError("LLM_ALL_PROVIDERS_FAILED", { error: result.error });
+      throw new InternalError3("LLM_ALL_PROVIDERS_FAILED", { error: result.error });
     }
     yield result.data.content;
     return result.data;
@@ -773,12 +1150,12 @@ async function* completionStream(messages, env, opts = {}) {
     });
   } catch (e) {
     if (e instanceof DOMException && e.name === "AbortError") {
-      throw new InternalError("llm call aborted", {
+      throw new InternalError3("llm call aborted", {
         provider: streamLeg.provider,
         model: streamLeg.model
       });
     }
-    throw new InternalError("llm stream fetch failed", {
+    throw new InternalError3("llm stream fetch failed", {
       message: e instanceof Error ? e.message : String(e)
     });
   }
@@ -790,7 +1167,7 @@ async function* completionStream(messages, env, opts = {}) {
     }
     const result = await complete(messages, env, opts, deps);
     if (result.error !== null || result.data === null) {
-      throw new InternalError("LLM_ALL_PROVIDERS_FAILED", {
+      throw new InternalError3("LLM_ALL_PROVIDERS_FAILED", {
         streamError: `${streamLeg.provider} ${String(response.status)}: ${text.slice(0, 300)}`,
         error: result.error
       });
@@ -799,7 +1176,7 @@ async function* completionStream(messages, env, opts = {}) {
     return result.data;
   }
   if (!response.body) {
-    throw new InternalError("llm stream response body is null", {
+    throw new InternalError3("llm stream response body is null", {
       provider: streamLeg.provider
     });
   }
@@ -917,15 +1294,20 @@ function assertGrounding(response, sources) {
 export {
   BASE_BACKOFF_MS,
   DEFAULT_EMBEDDING_MODEL,
+  LOCAL_EMBEDDING_MODEL,
   MODELS,
   MODEL_PRICE_PER_1M,
   PROVIDER_COOLDOWN_MS,
   assertGrounding,
+  clearGcpTokenCache,
   clearProviderCooldown,
   complete,
   completionStream,
   embed,
+  embedLocal,
   isProviderCoolingDown,
-  markProviderCoolingDown
+  markProviderCoolingDown,
+  mintGcpAccessToken,
+  serviceAccountProjectId
 };
 //# sourceMappingURL=index.mjs.map