@latimer-woods-tech/llm 0.5.0 → 0.6.0

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 0.6.0 — 2026-08-03
+
+### Added — local Qwen workbench and tool protocol
+
+- `LLM_LOCAL_WORKBENCH` routes workbench tasks to `qwen3.6:27b` first, with
+  DeepSeek retained as an independent cloud fallback.
+- The local OpenAI-compatible provider now forwards normalized tool schemas,
+  tool choice, assistant tool calls, and tool results.
+- Explicit `qwen*` model overrides route locally, and both local models are
+  priced at zero marginal token cost in the canonical ledger table.
+
 ## 0.4.4 — 2026-06-03
 
 ### Added — streaming tool-calls (Phase 1c; Anthropic)

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Adrian Perry / Latimer Woods Technology
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/index.d.mts

@@ -5,13 +5,17 @@ import { Logger } from '@latimer-woods-tech/logger';
  * Workers AI embedding API.
  *
  * Pins to bge-base-en-v1.5 (768-dim cosine) — the platform standard.
+ * @remarks Local-rail errors use @latimer-woods-tech/errors (observability ratchet).
  * model_version is returned with every result so callers can record it
  * for provenance-aware re-embed when the model is swapped.
  *
  * Contract: inject the Workers AI binding (env.AI) — never import a vendor SDK.
  */
+/** The platform-standard embedding model (768-dim cosine) used when no override is given. */
 declare const DEFAULT_EMBEDDING_MODEL = "@cf/baai/bge-base-en-v1.5";
+/** Supported embedding model identifiers — widen deliberately, since dims are part of the contract. */
 type EmbeddingModel = '@cf/baai/bge-base-en-v1.5';
+/** One embedding call's output: the vectors plus the provenance needed to re-embed later. */
 interface EmbedResult {
     vectors: number[][];
     model: string;
@@ -38,6 +42,54 @@ interface AiBinding {
 declare function embed(ai: AiBinding, input: string | string[], opts?: {
     model?: EmbeddingModel;
 }): Promise<EmbedResult>;
+/**
+ * Self-hosted GPU embedding model (nomic-embed-text, 768-dim), reached through
+ * the same CF AI Gateway custom provider as the local chat provider.
+ *
+ * ⚠️ VECTOR-SPACE WARNING: nomic-embed-text and bge-base-en-v1.5 are BOTH 768-dim
+ * but occupy DIFFERENT embedding spaces. Their vectors are NOT comparable. A store
+ * built with one model must be QUERIED — and, when migrating, RE-EMBEDDED — with
+ * the SAME model. Never write vectors from both models into one store.
+ */
+declare const LOCAL_EMBEDDING_MODEL = "nomic-embed-text";
+/** Env subset needed to reach the local embedding rail (shares GPU_LLM_* with the chat provider). */
+interface LocalEmbedEnv {
+    AI_GATEWAY_BASE_URL: string;
+    GPU_LLM_API_TOKEN?: string;
+    GPU_LLM_ACCESS_CLIENT_ID?: string;
+    GPU_LLM_ACCESS_CLIENT_SECRET?: string;
+}
+/**
+ * Embeds text on the self-hosted GPU rail (nomic-embed-text) via the AI Gateway
+ * custom provider — the $0 alternative to Workers AI {@link embed}.
+ *
+ * Unlike the local CHAT provider, this has NO cross-provider fallback ON PURPOSE:
+ * falling back to Workers AI (bge) would silently write an incompatible-space
+ * vector into a nomic store and corrupt retrieval. If the rail is unavailable
+ * this THROWS and the caller decides (retry local, or defer the write) — it must
+ * NOT substitute a different model.
+ *
+ * @param env   - Gateway base URL + GPU_LLM_* auth (same secrets as the chat provider).
+ * @param input - Single string or array of strings to embed.
+ * @returns Embedding vectors, model `nomic-embed-text`, and dimension count.
+ * @throws  On any transport/HTTP/empty-response error (no silent fallback).
+ */
+declare function embedLocal(env: LocalEmbedEnv, input: string | string[]): Promise<EmbedResult>;
+
+/**
+ * Exchange a service-account key for a cloud-platform access token, reusing a
+ * cached token until it is within {@link EXPIRY_SKEW_MS} of expiring.
+ *
+ * @param gcpSaKey - Service-account JSON key, base64-encoded or raw.
+ * @param fetchImpl - Fetch implementation (injectable for tests).
+ * @returns A bearer token valid for Vertex AI.
+ * @throws ValidationError when the key is malformed; InternalError when the exchange fails.
+ */
+declare function mintGcpAccessToken(gcpSaKey: string, fetchImpl: typeof fetch): Promise<string>;
+/** Reset the per-isolate token cache. Tests only. */
+declare function clearGcpTokenCache(): void;
+/** The project the service account belongs to — the Vertex project by default. */
+declare function serviceAccountProjectId(gcpSaKey: string): string;
 
 /**
  * A tool the model may call. `parameters` is a JSON Schema object describing
@@ -94,7 +146,7 @@ interface LLMMessage {
  * Quality tier selected by the caller. Routing is workload-split:
  *  - `fast`      → Grok 4.3 with Anthropic Haiku fallback (routine drafts/small jobs)
  *  - `balanced`  → Anthropic Sonnet (default)
- *  - `smart`     → Anthropic Opus OR Gemini 2.5 Pro if input is long-context (>150k tokens estimated)
+ *  - `smart`     → Anthropic Opus OR Gemini 2.5 Flash if input is long-context (>150k tokens estimated)
  *  - `verifier`  → Groq Llama (cheap second opinion; only used from verifier code path)
  *  - `workbench` → DeepSeek Chat with Groq fallback (boring, reviewable, non-sensitive batch work)
  */
@@ -173,7 +225,7 @@ interface LLMOptions {
 /**
  * Provider that produced an LLM response.
  */
-type LLMProvider = 'anthropic' | 'gemini' | 'groq' | 'grok' | 'deepseek';
+type LLMProvider = 'anthropic' | 'gemini' | 'groq' | 'grok' | 'deepseek' | 'local';
 /**
  * Result returned by a successful completion.
  */
@@ -220,13 +272,47 @@ interface LLMEnv {
     /** Optional — only required when caller passes `{ model: 'grok-*' }` override. */
     GROK_API_KEY?: string;
     /**
-     * Google Cloud short-lived access token with `aiplatform.endpoints.predict`.
-     * Callers mint this via the JWT-bearer flow (service account → token exchange);
-     * see `docs/runbooks/rotate-gcp-sa.md`. Token must be valid for ≥ 5 minutes.
+     * Cost optimization: when true, `fast`-tier calls route to the self-hosted GPU
+     * (qwen3 via the `custom-local-gpu` AI Gateway provider) FIRST, with the normal
+     * fast route (Grok→Haiku) as automatic fallback on any error. Off by default →
+     * fully dormant. Requires {@link LLMEnv.GPU_LLM_API_TOKEN}.
+     */
+    LLM_LOCAL_FIRST?: boolean;
+    /**
+     * Agentic-work opt-in: route the `workbench` tier to local Qwen 27B first,
+     * with DeepSeek as the independent cloud fallback.
      */
-    VERTEX_ACCESS_TOKEN: string;
-    VERTEX_PROJECT: string;
-    VERTEX_LOCATION: string;
+    LLM_LOCAL_WORKBENCH?: boolean;
+    /** Bearer for the local GPU proxy (`GPU_LLM_API_TOKEN` in Secret Manager). Required to enable local. */
+    GPU_LLM_API_TOKEN?: string;
+    /** Cloudflare Access service-token id for the local GPU app; forwarded by the gateway to the origin. */
+    GPU_LLM_ACCESS_CLIENT_ID?: string;
+    /** Cloudflare Access service-token secret for the local GPU app. */
+    GPU_LLM_ACCESS_CLIENT_SECRET?: string;
+    /**
+     * GCP service-account JSON key (base64-encoded or raw) with `roles/aiplatform.user`.
+     * The credential for the `gemini` leg in environments WITHOUT a metadata server
+     * (Cloudflare Workers): the package exchanges it for a fresh access token per
+     * isolate and caches it (see `./gcp-token.ts`).
+     *
+     * Optional. When neither this nor {@link LLMEnv.VERTEX_ACCESS_TOKEN} is set, the
+     * leg falls back to Application Default Credentials — the GCP metadata server's
+     * ambient service-account token (keyless; the default on Cloud Run / GCE). Off
+     * GCP with no static credential, the metadata probe fails and the chain falls
+     * through to its next leg.
+     */
+    GCP_SA_KEY?: string;
+    /**
+     * Pre-minted Google Cloud access token with `aiplatform.endpoints.predict`.
+     * Explicit override, used only when {@link LLMEnv.GCP_SA_KEY} is absent and
+     * before ADC is attempted. Valid for ~1 hour from minting, so it cannot be a
+     * durable Worker secret.
+     */
+    VERTEX_ACCESS_TOKEN?: string;
+    /** Vertex project. Defaults to the `GCP_SA_KEY`'s own `project_id`. */
+    VERTEX_PROJECT?: string;
+    /** Vertex location. Defaults to `us-central1`. */
+    VERTEX_LOCATION?: string;
     /**
      * Optional KV store for org-level daily/monthly cost tracking and enforcement.
      * When provided alongside {@link LLMOptions.dailyCapUsd} or {@link LLMOptions.monthlyCapUsd},
@@ -289,15 +375,15 @@ interface LLMDeps {
 }
 declare const MODELS: {
     readonly anthropic: {
-        readonly fast: "claude-haiku-4-20250514";
+        readonly fast: "claude-haiku-4-5";
         readonly balanced: "claude-sonnet-4-6";
         readonly smart: "claude-opus-4-7";
     };
     readonly gemini: {
-        readonly smart: "gemini-2.5-pro";
+        readonly smart: "gemini-2.5-flash";
     };
     readonly groq: {
-        readonly verifier: "llama-4-maverick";
+        readonly verifier: "llama-3.3-70b-versatile";
     };
     readonly grok: {
         readonly fast: "grok-4.3";
@@ -305,6 +391,10 @@ declare const MODELS: {
     readonly deepseek: {
         readonly workbench: "deepseek-chat";
     };
+    readonly local: {
+        readonly fast: "qwen3:8b";
+        readonly workbench: "qwen3.6:27b";
+    };
 };
 /** Cooldown duration in ms after a provider exhausts all retries. */
 declare const PROVIDER_COOLDOWN_MS = 30000;
@@ -343,8 +433,8 @@ declare const BASE_BACKOFF_MS = 250;
  *
  * Routing summary (0.3.0):
  *   - `fast`     → Grok 4.3; Anthropic Haiku fallback when Grok is unavailable
- *   - `balanced` → Anthropic Sonnet; Gemini 2.5 Pro if `longContextThreshold` exceeded
- *   - `smart`    → Anthropic Opus; Gemini 2.5 Pro if long-context
+ *   - `balanced` → Anthropic Sonnet; Gemini 2.5 Flash if `longContextThreshold` exceeded
+ *   - `smart`    → Anthropic Opus; Gemini 2.5 Flash if long-context
  *   - `verifier` → Groq Llama 3.3 70B (no fallback — verifier is inherently cheap/best-effort)
  *   - `workbench` → DeepSeek Chat; Groq fallback for boring/reviewable internal batch jobs
  *
@@ -414,4 +504,4 @@ declare function completionStream(messages: LLMMessage[], env: LLMEnv, opts?: LL
  */
 declare function assertGrounding(response: string, sources: string[]): boolean;
 
-export { type AiBinding, BASE_BACKOFF_MS, type CostKvStore, DEFAULT_EMBEDDING_MODEL, type EmbedResult, type EmbeddingModel, type LLMContentBlock, type LLMDeps, type LLMEnv, type LLMMessage, type LLMOptions, type LLMProvider, type LLMRecordContext, type LLMRecordRow, type LLMResult, type LLMTier, type LLMTool, type LLMToolCall, MODELS, MODEL_PRICE_PER_1M, PROVIDER_COOLDOWN_MS, assertGrounding, clearProviderCooldown, complete, completionStream, embed, isProviderCoolingDown, markProviderCoolingDown };
+export { type AiBinding, BASE_BACKOFF_MS, type CostKvStore, DEFAULT_EMBEDDING_MODEL, type EmbedResult, type EmbeddingModel, type LLMContentBlock, type LLMDeps, type LLMEnv, type LLMMessage, type LLMOptions, type LLMProvider, type LLMRecordContext, type LLMRecordRow, type LLMResult, type LLMTier, type LLMTool, type LLMToolCall, LOCAL_EMBEDDING_MODEL, type LocalEmbedEnv, MODELS, MODEL_PRICE_PER_1M, PROVIDER_COOLDOWN_MS, assertGrounding, clearGcpTokenCache, clearProviderCooldown, complete, completionStream, embed, embedLocal, isProviderCoolingDown, markProviderCoolingDown, mintGcpAccessToken, serviceAccountProjectId };