@latimer-woods-tech/llm 0.4.2 → 0.4.4

package/CHANGELOG.md

@@ -1,5 +1,40 @@
 # Changelog
 
+## 0.4.4 — 2026-06-03
+
+### Added — streaming tool-calls (Phase 1c; Anthropic)
+
+- `completionStream` now accumulates Anthropic tool-use stream events
+  (`content_block_start` + `input_json_delta` fragments) into the same
+  normalized `LLMResult.toolCalls`, and captures `stopReason` from the
+  `message_delta`. The returned result of a streamed tool-use turn matches the
+  non-streaming shape. Text deltas still stream incrementally as before.
+
+---
+
+## 0.4.3 — 2026-06-03
+
+### Added — tool-calling for OpenAI-style providers (Phase 1b: Grok, DeepSeek)
+
+- **Grok and DeepSeek now support tool-calling.** Requests carry OpenAI-format
+  `tools` + `tool_choice`; responses parse `tool_calls` + `finish_reason` into
+  the same normalized `LLMResult.toolCalls` / `stopReason` as Anthropic.
+- Anthropic-shaped tool blocks are converted to OpenAI wire format:
+  `tool_use` → an assistant message with `tool_calls`; `tool_result` → a
+  standalone `tool` message keyed by `tool_call_id`.
+- **`TOOL_CAPABLE_PROVIDERS`** widened to `anthropic, grok, deepseek`, so the
+  `fast` (Grok→Haiku) and `workbench` (DeepSeek) tiers support tool loops while
+  still failing closed for `verifier` (Groq Llama).
+- Malformed tool-call argument JSON is tolerated (billed as `{}`, never throws).
+
+### Not yet
+
+- **Gemini** tool-calling is deferred to a focused follow-up — its
+  `tool_use_id`↔function-name correlation and schema constraints need dedicated
+  handling. Gemini stays out of `TOOL_CAPABLE_PROVIDERS` until then.
+
+---
+
 ## 0.4.2 — 2026-06-03
 
 ### Added — tool-calling (Agent Runtime Phase 1a; Anthropic)

package/dist/index.d.mts

@@ -0,0 +1,379 @@
+import { FactoryResponse } from '@latimer-woods-tech/errors';
+import { Logger } from '@latimer-woods-tech/logger';
+
+/**
+ * A tool the model may call. `parameters` is a JSON Schema object describing
+ * the tool's input. Provider-agnostic; normalized per provider at request time.
+ */
+interface LLMTool {
+    name: string;
+    description?: string;
+    /** JSON Schema for the tool's input arguments. */
+    parameters: Record<string, unknown>;
+}
+/**
+ * A tool invocation requested by the model, normalized across providers.
+ */
+interface LLMToolCall {
+    /** Provider-assigned call id; echo it back in the matching tool_result. */
+    id: string;
+    name: string;
+    /** Parsed argument object the model passed to the tool. */
+    arguments: Record<string, unknown>;
+}
+/**
+ * Structured content block for tool-calling conversations. The field shapes
+ * mirror the Anthropic Messages wire format so they pass through unchanged.
+ */
+type LLMContentBlock = {
+    type: 'text';
+    text: string;
+} | {
+    type: 'tool_use';
+    id: string;
+    name: string;
+    input: Record<string, unknown>;
+} | {
+    type: 'tool_result';
+    tool_use_id: string;
+    content: string;
+    is_error?: boolean;
+};
+/**
+ * Single chat message exchanged with an LLM provider.
+ *
+ * `content` is a plain string in the common case. For tool-calling
+ * conversations it may be an array of {@link LLMContentBlock}s (e.g. an
+ * assistant turn carrying `tool_use` blocks, or a user turn carrying
+ * `tool_result` blocks). Providers that don't support tool-calling receive
+ * the text projection of the content (see `contentToText`).
+ */
+interface LLMMessage {
+    role: 'user' | 'assistant' | 'system';
+    content: string | LLMContentBlock[];
+}
+/**
+ * Quality tier selected by the caller. Routing is workload-split:
+ *  - `fast`      → Grok 4.3 with Anthropic Haiku fallback (routine drafts/small jobs)
+ *  - `balanced`  → Anthropic Sonnet (default)
+ *  - `smart`     → Anthropic Opus OR Gemini 2.5 Pro if input is long-context (>150k tokens estimated)
+ *  - `verifier`  → Groq Llama (cheap second opinion; only used from verifier code path)
+ *  - `workbench` → DeepSeek Chat with Groq fallback (boring, reviewable, non-sensitive batch work)
+ */
+type LLMTier = 'fast' | 'balanced' | 'smart' | 'verifier' | 'workbench';
+/**
+ * Options that influence LLM completion behaviour.
+ */
+interface LLMOptions {
+    /** Quality tier; see {@link LLMTier}. Defaults to `balanced`. */
+    tier?: LLMTier;
+    /** Explicit model override. Takes precedence over tier. */
+    model?: string;
+    maxTokens?: number;
+    temperature?: number;
+    system?: string;
+    /** Token budget above which we force long-context routing (Gemini). */
+    longContextThreshold?: number;
+    /** Per-call cancellation signal. Aborts the in-flight provider request. */
+    signal?: AbortSignal;
+    /** Optional run identifier stamped on ledger rows + logs. */
+    runId?: string;
+    /** Optional project identifier stamped on ledger rows + logs. */
+    project?: string;
+    /** Optional actor identifier (supervisor / worker / human). */
+    actor?: string;
+    /** Optional workload label used in logs and cost-policy call sites. */
+    workload?: string;
+    /** Grok reasoning effort. Defaults to `none` for cost-controlled fast/draft calls. */
+    reasoningEffort?: 'none' | 'low' | 'medium' | 'high';
+    /** Anthropic prompt-cache control. Defaults to `true` for `system` prompts ≥ 1024 tokens. */
+    promptCache?: boolean;
+    /**
+     * Maximum estimated cost in USD for this completion.
+     * This cap is enforced after the provider returns because it uses actual
+     * response token counts to compute the final cost.
+     * If the post-call estimated cost exceeds this cap, `complete` returns a
+     * {@link RateLimitError} with code `LLM_COST_CAP_EXCEEDED` and
+     * `completionStream` throws the same error.
+     * Pricing is based on {@link MODEL_PRICE_PER_1M}; unknown models default to
+     * Opus rates (conservative upper bound).
+     */
+    maxCostUsd?: number;
+    /**
+     * Org-level daily cost cap in USD. Requires `env.LLM_COST_KV` to be set.
+     * When today's cumulative spend read from KV is >= this value, `complete`
+     * returns a {@link RateLimitError} with code `LLM_DAILY_CAP_EXCEEDED`
+     * without making any provider call. After a successful call the daily
+     * accumulator is updated in KV (TTL: 48 h).
+     */
+    dailyCapUsd?: number;
+    /**
+     * Org-level monthly cost cap in USD. Requires `env.LLM_COST_KV` to be set.
+     * Same enforcement pattern as {@link dailyCapUsd} but keyed by YYYY-MM.
+     * KV TTL: 40 days.
+     */
+    monthlyCapUsd?: number;
+    /**
+     * Metering context. When supplied and `deps.onRecord` is set, a {@link LLMRecordRow}
+     * is emitted after every successful completion. Errors are swallowed.
+     */
+    ledger?: LLMRecordContext;
+    /**
+     * Tools the model may call. When present, routing **fails closed** to
+     * tool-capable providers — failover never falls back to a provider that
+     * can't honour the tool schema. See {@link LLMResult.toolCalls}.
+     */
+    tools?: LLMTool[];
+    /**
+     * Tool-selection policy. `'auto'` (default when `tools` is set) lets the
+     * model decide; `'none'` forbids tool use; `{ name }` forces a specific tool.
+     */
+    toolChoice?: 'auto' | 'none' | {
+        name: string;
+    };
+}
+/**
+ * Provider that produced an LLM response.
+ */
+type LLMProvider = 'anthropic' | 'gemini' | 'groq' | 'grok' | 'deepseek';
+/**
+ * Result returned by a successful completion.
+ */
+interface LLMResult {
+    content: string;
+    provider: LLMProvider;
+    model: string;
+    tier: LLMTier;
+    tokens: {
+        input: number;
+        output: number;
+        cacheRead?: number;
+        cacheWrite?: number;
+    };
+    latency: number;
+    /** Number of attempts before success (1 = primary succeeded). */
+    attempts: number;
+    /** Monotonic request id from AI Gateway, if present in headers. */
+    gatewayRequestId?: string;
+    /**
+     * Why generation stopped, normalized across providers. `'tool_use'` means
+     * the model is requesting one or more tool calls (see {@link toolCalls}).
+     */
+    stopReason?: 'end' | 'tool_use' | 'max_tokens' | 'other';
+    /**
+     * Tool calls the model requested, normalized across providers. Present
+     * (non-empty) when `stopReason === 'tool_use'`.
+     */
+    toolCalls?: LLMToolCall[];
+}
+/**
+ * Environment bindings required by {@link complete}.
+ *
+ * `AI_GATEWAY_BASE_URL` is REQUIRED in 0.3.0. All provider calls flow through the
+ * Cloudflare AI Gateway for unified logging, rate limiting, and cost telemetry.
+ * In test/dev the caller may pass a custom fetch impl that short-circuits this.
+ */
+interface LLMEnv {
+    AI_GATEWAY_BASE_URL: string;
+    ANTHROPIC_API_KEY: string;
+    GROQ_API_KEY: string;
+    /** Optional — only required for `{ tier: 'workbench' }` or `deepseek-*` model overrides. */
+    DEEPSEEK_API_KEY?: string;
+    /** Optional — only required when caller passes `{ model: 'grok-*' }` override. */
+    GROK_API_KEY?: string;
+    /**
+     * Google Cloud short-lived access token with `aiplatform.endpoints.predict`.
+     * Callers mint this via the JWT-bearer flow (service account → token exchange);
+     * see `docs/runbooks/rotate-gcp-sa.md`. Token must be valid for ≥ 5 minutes.
+     */
+    VERTEX_ACCESS_TOKEN: string;
+    VERTEX_PROJECT: string;
+    VERTEX_LOCATION: string;
+    /**
+     * Optional KV store for org-level daily/monthly cost tracking and enforcement.
+     * When provided alongside {@link LLMOptions.dailyCapUsd} or {@link LLMOptions.monthlyCapUsd},
+     * `complete` will block calls that would exceed the declared cap.
+     * Any KV-like store satisfying `get`/`put` works (e.g. Cloudflare KV, in-memory stub).
+     */
+    LLM_COST_KV?: CostKvStore;
+}
+/**
+ * Minimal KV store interface for org-level LLM cost tracking.
+ * Cloudflare KV satisfies this. An in-memory stub is sufficient for tests.
+ */
+interface CostKvStore {
+    get(key: string): Promise<string | null>;
+    put(key: string, value: string, options?: {
+        expirationTtl?: number;
+    }): Promise<void>;
+}
+/**
+ * Caller-supplied context stamped on every metering row.
+ * Mirrors the `LLMRecordContext` in `@latimer-woods-tech/llm-meter`; kept inline
+ * to avoid a circular dependency (llm-meter imports llm).
+ */
+interface LLMRecordContext {
+    project: string;
+    actor: string;
+    runId?: string;
+    workload?: string;
+    tenantId?: string;
+}
+/**
+ * Row shape passed to the optional {@link LLMDeps.onRecord} callback.
+ * Callers can wire this directly to `recordCall` from `@latimer-woods-tech/llm-meter`.
+ */
+interface LLMRecordRow extends LLMRecordContext {
+    model: string;
+    provider: LLMProvider;
+    tier: LLMTier;
+    inputTokens: number;
+    outputTokens: number;
+    cacheReadTokens: number;
+    cacheWriteTokens: number;
+    latencyMs: number;
+    costUsd: number;
+    yyyyMm: string;
+}
+/**
+ * Optional dependencies for {@link complete}.
+ */
+interface LLMDeps {
+    fetch?: typeof fetch;
+    logger?: Logger;
+    now?: () => number;
+    /**
+     * Optional metering callback. Called after every successful completion.
+     * Errors are swallowed so metering never blocks the caller.
+     * Wire to `recordCall` from `@latimer-woods-tech/llm-meter`.
+     */
+    onRecord?: (row: LLMRecordRow) => Promise<void>;
+}
+declare const MODELS: {
+    readonly anthropic: {
+        readonly fast: "claude-haiku-4-20250514";
+        readonly balanced: "claude-sonnet-4-6";
+        readonly smart: "claude-opus-4-7";
+    };
+    readonly gemini: {
+        readonly smart: "gemini-2.5-pro";
+    };
+    readonly groq: {
+        readonly verifier: "llama-4-maverick";
+    };
+    readonly grok: {
+        readonly fast: "grok-4.3";
+    };
+    readonly deepseek: {
+        readonly workbench: "deepseek-chat";
+    };
+};
+/** Cooldown duration in ms after a provider exhausts all retries. */
+declare const PROVIDER_COOLDOWN_MS = 30000;
+/**
+ * Returns `true` if the provider is currently in its cooldown window.
+ * Uses the injected `now` function (or `Date.now`) for testability.
+ */
+declare function isProviderCoolingDown(provider: LLMProvider, now?: () => number): boolean;
+/**
+ * USD cost per 1 million tokens for each model.
+ * Source: Anthropic / Google / xAI pricing pages as of 2026-05.
+ * Keep these model names in sync with the default routing constants in
+ * {@link MODELS}; unknown models fall back to Opus rates (conservative upper bound).
+ *
+ * CANONICAL pricing source for the platform. `@latimer-woods-tech/llm-meter`
+ * derives its cents-denominated rates from this table and a drift-guard test
+ * there fails CI if they diverge — make all rate changes here.
+ */
+declare const MODEL_PRICE_PER_1M: Record<string, {
+    input: number;
+    output: number;
+    cacheRead: number;
+    cacheWrite: number;
+}>;
+/**
+ * Marks a provider as cooling down for {@link PROVIDER_COOLDOWN_MS} milliseconds.
+ */
+declare function markProviderCoolingDown(provider: LLMProvider, now?: () => number): void;
+/**
+ * Clears the cooldown state for a provider after a successful call.
+ */
+declare function clearProviderCooldown(provider: LLMProvider): void;
+declare const BASE_BACKOFF_MS = 250;
+/**
+ * Run a completion through the routing plan for the requested tier.
+ *
+ * Routing summary (0.3.0):
+ *   - `fast`     → Grok 4.3; Anthropic Haiku fallback when Grok is unavailable
+ *   - `balanced` → Anthropic Sonnet; Gemini 2.5 Pro if `longContextThreshold` exceeded
+ *   - `smart`    → Anthropic Opus; Gemini 2.5 Pro if long-context
+ *   - `verifier` → Groq Llama 3.3 70B (no fallback — verifier is inherently cheap/best-effort)
+ *   - `workbench` → DeepSeek Chat; Groq fallback for boring/reviewable internal batch jobs
+ *
+ * All provider traffic flows through Cloudflare AI Gateway at `AI_GATEWAY_BASE_URL`.
+ *
+ * Per-provider reliability guarantees (0.4.0):
+ *   - Exponential backoff with jitter on 429 / 5xx (base 500ms, cap 8s, up to 2 retries).
+ *   - Provider cooldown: after exhausting retries the provider is marked cooling down
+ *     for 30 seconds; subsequent calls skip it and go straight to the fallback leg.
+ *
+ * @param messages - Ordered chat history.
+ * @param env - API key + gateway bindings.
+ * @param opts - Optional tier/model/parameters override.
+ * @param deps - Optional fetch/logger/clock injection (for testing).
+ * @returns A {@link FactoryResponse} carrying either an {@link LLMResult} or
+ *   an error (`LLM_ALL_PROVIDERS_FAILED`, `LLM_RATE_LIMITED`, or `INTERNAL_ERROR`).
+ */
+declare function complete(messages: LLMMessage[], env: LLMEnv, opts?: LLMOptions, deps?: LLMDeps): Promise<FactoryResponse<LLMResult>>;
+/**
+ * Streams a completion from the primary Anthropic provider, yielding text chunks
+ * as they arrive. Falls back to the non-streaming {@link complete} function when
+ * the provider does not support streaming (i.e. a non-Anthropic primary is selected).
+ *
+ * The generator's **return value** (accessible via `gen.return()` or by consuming
+ * the full iteration) is an {@link LLMResult} with the same shape as {@link complete}.
+ *
+ * Usage pattern:
+ * ```ts
+ * const gen = completionStream(messages, env, opts);
+ * for await (const chunk of gen) {
+ *   // stream chunk to client
+ * }
+ * const result = (await gen.return(undefined)).value; // LLMResult
+ * ```
+ *
+ * @param messages - Ordered chat history.
+ * @param env - API key + gateway bindings.
+ * @param opts - Optional tier/model/parameters override. Accepts `deps` as nested field.
+ * @returns An async generator that yields `string` chunks and returns an {@link LLMResult}.
+ */
+declare function completionStream(messages: LLMMessage[], env: LLMEnv, opts?: LLMOptions & {
+    deps?: LLMDeps;
+}): AsyncGenerator<string, LLMResult, unknown>;
+/**
+ * Returns `true` if `response` contains at least one verbatim phrase of at
+ * least 5 consecutive whitespace-delimited tokens that also appears in one of
+ * the `sources` strings.
+ *
+ * Returns `true` unconditionally when `sources` is empty (no grounding
+ * documents means grounding cannot be violated).
+ *
+ * This is a lightweight guard for RAG pipelines — it detects obvious
+ * hallucinations where the model generates content not present in any
+ * retrieved source. It is NOT a semantic similarity check.
+ *
+ * @param response - The LLM-generated text to inspect.
+ * @param sources - Retrieved source documents to check against.
+ * @returns `true` if the response is grounded, `false` if hallucination detected.
+ *
+ * @example
+ * ```ts
+ * const grounded = assertGrounding(llmAnswer, retrievedDocs);
+ * if (!grounded) {
+ *   // flag or re-rank the response
+ * }
+ * ```
+ */
+declare function assertGrounding(response: string, sources: string[]): boolean;
+
+export { BASE_BACKOFF_MS, type CostKvStore, type LLMContentBlock, type LLMDeps, type LLMEnv, type LLMMessage, type LLMOptions, type LLMProvider, type LLMRecordContext, type LLMRecordRow, type LLMResult, type LLMTier, type LLMTool, type LLMToolCall, MODELS, MODEL_PRICE_PER_1M, PROVIDER_COOLDOWN_MS, assertGrounding, clearProviderCooldown, complete, completionStream, isProviderCoolingDown, markProviderCoolingDown };