@spendgraph/llms 0.2.0

package/README.md

@@ -0,0 +1,354 @@
+# @spendgraph/llms
+
+Call any LLM provider, get one shape back, and record what it consumed. Takes
+plain objects and calls your client by duck typing, so it has zero dependencies
+and never imports a provider SDK.
+
+There are no prompts here. Prompt storage, rendering and rollouts live in
+`@spendgraph/prompt`; this package is the model call and its token usage.
+
+```sh
+npm install @spendgraph/llms
+```
+
+## Llm
+
+```ts
+import { Llm } from "@spendgraph/llms";
+
+const llm = new Llm({
+  client: new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY }),
+  model: "claude-opus-5",
+  spendgraph: {
+    apiKey: process.env.SPENDGRAPH_API_KEY,
+    baseUrl: "https://your-spendgraph.example.com",
+  },
+});
+
+const reply = await llm.call([{ role: "user", content: "Summarise this." }]);
+
+reply.output;
+reply.inputTokens;
+reply.outputTokens;
+reply.eventId;
+```
+
+`call` sends the messages, normalises the reply, and — when `spendgraph` is set
+and `trace` is not `false` — posts the token counts to `POST /api/v1/ingest`.
+Nothing about the conversation is sent, only what it consumed. Pricing stays on
+the server, where the catalogue lives.
+
+Leave `spendgraph` off and nothing is recorded; the reply is still normalised.
+
+A recorded call that carries no tokens prices as nothing, so the first one warns
+once. The usual cause is a stream whose usage was never requested.
+
+### Streaming
+
+```ts
+const reply = await llm.stream(messages, { onText: (delta) => res.write(delta) });
+```
+
+Usage arrives in the last events of a stream, so the recording happens once it
+drains. On OpenAI the driver sets `stream_options: { include_usage: true }` for
+you — without it the final chunk carries no usage at all.
+
+`stream` does not drive tool loops. Passing tools throws rather than silently
+streaming without them; use `call`, or `{ tools: undefined }` for this one.
+
+### Tools
+
+Pass a bus and `call` becomes a loop: it sends the declarations, runs each tool
+call, feeds the results back in the provider's own envelope, and sums the
+tokens across every call in the loop.
+
+```ts
+const reply = await llm.call(messages, { tools: bus });
+```
+
+A bus is anything with `anthropic()`, `openai()` and `invoke()` — which is what
+harness's `toolbus()` and its per-turn `trace` handle both return.
+
+### Defaults, and overriding them
+
+Everything except `client`, `provider` and `spendgraph` is a default you set
+once and override per call.
+
+```ts
+const llm = new Llm({ client, model: "claude-opus-5", tools: bus, maxTokens: 8_000 });
+
+await llm.call(messages);
+await llm.call(messages, { model: "claude-haiku-4-5" });
+await llm.call(messages, { tools: undefined });
+await llm.call(messages, { trace: false });
+```
+
+`with()` derives a second client, merging `params` and `metadata` rather than
+replacing them.
+
+| Option | Where | |
+| --- | --- | --- |
+| `client` | client only | your provider SDK instance |
+| `provider` | client only | pricing slug, for OpenAI-compatible gateways |
+| `spendgraph` | client only | `{ apiKey, baseUrl }`, or `{ via }` to send usage yourself |
+| `model` | both | the id to send |
+| `trace` | both | `false` calls the model and records nothing |
+| `tools` | both | a bus; its presence turns `call` into a loop |
+| `maxTokens` | both | defaults to 16000 on Anthropic, unset elsewhere |
+| `maxSteps` | both | tool-loop ceiling, default 8 |
+| `params` | both | merged into the request body, call over client |
+| `metadata` | both | merged onto the usage event, call over client |
+| `onText` | both | per-delta callback for `stream` |
+| `eventId` | call only | supply your own so a retried job counts once |
+
+### A shape, not prose
+
+`schema` makes the reply an object. The parsed value comes back on `data`.
+
+```ts
+const reply = await llm.call(messages, {
+  schema: {
+    type: "object",
+    properties: { sentiment: { type: "string" }, score: { type: "number" } },
+    required: ["sentiment", "score"],
+  },
+});
+
+reply.data; // { sentiment: "warm", score: 4 }
+```
+
+The three providers say this three different ways and it is the same request:
+Anthropic has no JSON mode, so it gets a **forced tool call** — `tool_choice`
+pins it, which is the difference between "you may" and "you will"; OpenAI gets a
+`json_schema` with `strict: true`; Gemini gets a `responseSchema` and the mime
+type, because the schema alone still returns prose.
+
+A reply that is not the shape you asked for **throws**. Coming back with `data`
+quietly missing is how an empty case gets written into your data.
+
+Pass a bare JSON Schema, or `{ schema, name, description }` to name it. The
+schema is the wire format on purpose — `@spendgraph/tools` turns a `FieldSpec[]`
+into one, zod emits one, and hand-writing one is three lines. Accepting it is
+what keeps this package dependency-free.
+
+### When a provider is having a bad day
+
+```ts
+new Llm({
+  client,
+  model: "claude-opus-5",
+  fallbacks: ["claude-sonnet-5", "gpt-5"],
+  onFallback: ({ from, to, error }) => log.warn({ from, to }, String(error)),
+});
+```
+
+Only for failures about the provider — 429, 529, 5xx. A 400 is about your
+request and is the same answer everywhere, so trying three models spends three
+round trips to be told so three times. The usage event records the model that
+actually answered, not the one you asked for.
+
+### With the harness
+
+`LlmReply` is structurally the `TraceOutcome` that `prompt.trace()` accepts, so
+the two compose in one line — harness records the rollout, so leave `trace`
+off here or the same call is counted twice.
+
+```ts
+const llm = new Llm({ client, model, trace: false });
+
+await prompt.trace(values, ({ messages }) => llm.call(messages));
+await prompt.trace(values, ({ messages, turn }) => llm.call(messages, { tools: turn }), {
+  tools: bus,
+});
+```
+
+### Sending usage yourself
+
+`via` replaces the built-in sender — buffer, batch, or push onto a queue. The
+route takes 100 events per request and the built-in sender chunks to that.
+
+```ts
+spendgraph: { via: { send: (events) => queue.push(events) } }
+```
+
+## Affording the ceiling
+
+`max_tokens` is a request for headroom, not a bill — output is charged on what
+is written. A provider that reserves the ceiling against a prepaid balance
+refuses the whole call over headroom that was never going to be used:
+
+```
+402 You requested up to 4096 tokens, but can only afford 4000.
+```
+
+`affordable` wraps any client and retries that once, at the number the provider
+named:
+
+```ts
+import { affordable, Llm } from "@spendgraph/llms";
+
+const llm = new Llm({
+  client: affordable(anthropic, { floor: 1024 }),
+  model: "claude-sonnet-5",
+  maxTokens: 4096,
+});
+```
+
+It is a decorator rather than something inside `Llm` because how a client copes
+with a provider's billing is the client's business. The wrapper mirrors whatever
+shape it is given, so `driverFor` still recognises an Anthropic, OpenAI or
+Gemini client through it, and it shrinks whichever field that provider carries
+the ceiling in — `max_tokens`, `max_completion_tokens` or a nested
+`generationConfig.maxOutputTokens`.
+
+`floor` is worth setting where the reply is a forced tool call. A schema
+truncated mid-arguments parses to nothing, which is a worse answer than the
+refusal it replaced.
+
+A `402` naming no number is passed through untouched: that one is an empty
+account, and no ceiling will fix it.
+
+## Lower level
+
+`read` and `stream` normalise a reply you fetched yourself — a client `Llm`
+does not drive, a raw `fetch`, a framework callback.
+
+```ts
+import { anthropicMessages, read, stream } from "@spendgraph/llms";
+
+const reply = read(await client.messages.create({ ...anthropicMessages(messages), model }));
+
+const reading = stream(source, { provider: "anthropic" });
+for await (const delta of reading) res.write(delta);
+const finished = await reading.reply();
+```
+
+`reply()` is only correct once the stream drains, since usage arrives in its
+last events; it drains for you if you never iterated. `collect(source, { onText })`
+is the same thing without the loop.
+
+### Shapes
+
+Six wire families cover the providers `model_pricing` knows:
+
+| Shape | Speaks it |
+| --- | --- |
+| `anthropic` | Anthropic Messages, Bedrock Mantle, Vertex Anthropic |
+| `openai` | OpenAI chat completions, and every gateway that copies it — OpenRouter, z.ai, Groq, Together, DeepSeek, Fireworks, Azure, Ollama, vLLM |
+| `responses` | OpenAI Responses |
+| `bedrock` | Bedrock Converse |
+| `gemini` | Gemini `generateContent`, Vertex Gemini |
+| `perplexity` | Perplexity Sonar |
+
+`read` sniffs the shape, but cannot tell OpenAI-compatible gateways apart —
+they are byte-identical — so pass `provider` for anything that is not OpenAI:
+
+```ts
+read(reply, { provider: "zai" });        // zai/glm-5
+read(reply, { provider: "openrouter" }); // openrouter/anthropic/claude-opus-4
+```
+
+**Bedrock Converse never echoes the model id**, so pass `model` there or the
+reply comes back with none and prices at zero:
+
+```ts
+read(converseReply, { provider: "bedrock", model: "anthropic.claude-opus-5" });
+```
+
+### Perplexity Sonar
+
+Sonar speaks OpenAI chat completions and adds `search_results`, which is enough
+to tell the two apart, so a Sonar reply is sniffed without `provider`. Its
+sources come back on `citations`, newest field first and the legacy `citations`
+array as the fallback.
+
+```ts
+const reply = read(sonarReply, { provider: "perplexity" });
+reply.output;
+reply.citations; // ["https://…", "https://…"]
+```
+
+Point an OpenAI client at `https://api.perplexity.ai` and `Llm` drives it, as
+long as you name the provider so the reply is priced as Sonar and not as GPT:
+
+```ts
+new Llm({ client, model: "sonar-pro", provider: "perplexity" });
+```
+
+Sonar bills citation tokens and reasoning tokens as their own line items, so
+they are read and recorded separately rather than folded into output, which
+would price them at the wrong rate. Deep Research is the model that charges
+both.
+
+```ts
+reply.citationTokens;
+reply.reasoningTokens;
+```
+
+They ride the usage event only when non-zero, and the server bills them at the
+input and output rate — no pricing row publishes a rate of their own, so the
+choice is a known rate or $0. The per-request search fee is priced from
+`model_pricing.per_request_micros` and needs nothing on the event. Sonar does
+not cache, so both cache counts are zero.
+
+Read them only off the top level of `usage`. OpenAI reports its own reasoning
+count under `completion_tokens_details`, where it is already inside
+`completion_tokens` — recording that one would bill it twice.
+
+## Model ids
+
+`model_pricing` follows litellm. `anthropic` and `openai` rows are keyed bare
+(`claude-opus-5`, `gpt-5`), Bedrock keeps the lab-namespaced native id
+(`anthropic.claude-3-5-haiku-20241022-v1:0`), and everything else carries its
+slug (`zai/glm-5`). The rule is applied for you, and never prefixes twice.
+
+An id matching no row is not an error — the event records and prices at zero,
+and the dashboard calls it unpriced.
+
+## What it removes
+
+Six ways to record a plausible-looking call that prices wrong, five of them
+silent:
+
+| Mistake | Was |
+| --- | --- |
+| `prompt_tokens` vs `input_tokens` | zero cost |
+| model id missing its pricing slug | zero cost |
+| not summing tokens across a tool loop | an agent priced as one call |
+| dropping cache token counts | cache reads billed at full rate |
+| `content[0].text` on a thinking model | a thinking block as the answer |
+| a refusal read as an empty success | a completed record with no output |
+
+`toolCalls` is normalised too — `{ id, name, args }`, with `args` already
+parsed from whatever JSON string the provider sent. Malformed JSON reads as
+`{}`, so tool validation reports the missing field rather than the turn
+throwing.
+
+## Testing against a real provider
+
+Every other test here uses a fake client, which proves the request body is the
+one this code meant to build and nothing about whether the provider agrees. The
+three structured-output shapes in particular were written from documentation.
+
+`src/tests/live` calls the real APIs. It **skips** unless the key is set, so a
+clone with no credentials still passes.
+
+```sh
+ANTHROPIC_API_KEY=… OPENAI_API_KEY=… GEMINI_API_KEY=… npm run test:live
+```
+
+The clients there are built on `fetch` rather than the provider SDKs, on
+purpose: a smoke test exists to find out whether the body this package builds is
+one the real API accepts, and an SDK in between would reshape it before it left.
+
+It costs real money — cheap models, one-word prompts, a 64-token ceiling — and
+what it checks is the part fakes cannot:
+
+| | |
+| --- | --- |
+| tokens | a real reply reports non-zero input and output |
+| schema | Anthropic's forced tool call, OpenAI's strict `json_schema`, Gemini's `responseSchema` |
+| tools | the loop calls, feeds the result back, and answers |
+| streaming | deltas arrive and usage still lands |
+| gemini `maxTokens` | lands as `generationConfig.maxOutputTokens`, where it is actually read |
+| history | an assistant turn survives as `model` on Gemini, rather than being replayed as the caller |

package/dist/adapters/anthropic.d.ts

@@ -0,0 +1,2 @@
+import type { Adapter } from "../types.js";
+export declare const anthropic: Adapter;

package/dist/adapters/anthropic.js

@@ -0,0 +1,101 @@
+import { pricingId } from "../model.js";
+import { count, parseArgs, reply, setUsage } from "../usage.js";
+function usageOf(raw) {
+    return {
+        inputTokens: count(raw?.input_tokens),
+        outputTokens: count(raw?.output_tokens),
+        cacheReadTokens: count(raw?.cache_read_input_tokens),
+        cacheWriteTokens: count(raw?.cache_creation_input_tokens),
+        citationTokens: 0,
+        reasoningTokens: 0,
+    };
+}
+function textOf(blocks) {
+    let text = "";
+    for (const block of blocks) {
+        if (block.type === "text" && typeof block.text === "string")
+            text += block.text;
+    }
+    return text;
+}
+function toolCallsOf(blocks) {
+    const calls = [];
+    for (const block of blocks) {
+        if (block.type !== "tool_use")
+            continue;
+        calls.push({
+            id: block.id ?? "",
+            name: block.name ?? "",
+            args: parseArgs(block.input),
+        });
+    }
+    return calls;
+}
+function refusalOf(raw) {
+    if (raw.stop_reason !== "refusal")
+        return undefined;
+    const { category, explanation } = raw.stop_details ?? {};
+    return explanation ?? `declined: ${category ?? "unspecified"}`;
+}
+export const anthropic = {
+    provider: "anthropic",
+    detect(value) {
+        const raw = value;
+        return Boolean(raw && Array.isArray(raw.content) && "stop_reason" in raw);
+    },
+    read(value, opts = {}) {
+        const raw = value;
+        const blocks = raw.content ?? [];
+        const error = refusalOf(raw);
+        return reply({
+            output: textOf(blocks),
+            model: opts.model ?? pricingId(opts.provider ?? "anthropic", raw.model ?? ""),
+            usage: usageOf(raw.usage),
+            toolCalls: toolCallsOf(blocks),
+            stopReason: raw.stop_reason,
+            status: error ? "failed" : "completed",
+            error,
+        });
+    },
+    chunk(event, state) {
+        const e = event;
+        switch (e.type) {
+            case "message_start": {
+                state.model = e.message?.model ?? state.model;
+                setUsage(state.usage, usageOf(e.message?.usage));
+                return "";
+            }
+            case "content_block_start": {
+                const block = e.content_block;
+                if (block?.type === "tool_use") {
+                    state.tools.set(String(e.index), {
+                        id: block.id ?? "",
+                        name: block.name ?? "",
+                        json: "",
+                    });
+                }
+                return "";
+            }
+            case "content_block_delta": {
+                const delta = e.delta ?? {};
+                if (delta.type === "text_delta" && typeof delta.text === "string") {
+                    state.text += delta.text;
+                    return delta.text;
+                }
+                if (delta.type === "input_json_delta" && typeof delta.partial_json === "string") {
+                    const pending = state.tools.get(String(e.index));
+                    if (pending)
+                        pending.json += delta.partial_json;
+                }
+                return "";
+            }
+            case "message_delta": {
+                state.stopReason = e.delta?.stop_reason ?? state.stopReason;
+                setUsage(state.usage, usageOf(e.usage));
+                return "";
+            }
+            default:
+                return "";
+        }
+    },
+};

package/dist/adapters/bedrock.d.ts

@@ -0,0 +1,2 @@
+import type { Adapter } from "../types.js";
+export declare const bedrock: Adapter;

package/dist/adapters/bedrock.js

@@ -0,0 +1,88 @@
+import { count, parseArgs, reply, setUsage } from "../usage.js";
+function usageOf(raw) {
+    return {
+        inputTokens: count(raw?.inputTokens),
+        outputTokens: count(raw?.outputTokens),
+        cacheReadTokens: count(raw?.cacheReadInputTokens),
+        cacheWriteTokens: count(raw?.cacheWriteInputTokens),
+        citationTokens: 0,
+        reasoningTokens: 0,
+    };
+}
+function blocksOf(raw) {
+    return raw.output?.message?.content ?? [];
+}
+function textOf(blocks) {
+    let text = "";
+    for (const block of blocks) {
+        if (typeof block.text === "string")
+            text += block.text;
+    }
+    return text;
+}
+function toolCallsOf(blocks) {
+    const calls = [];
+    for (const { toolUse } of blocks) {
+        if (!toolUse)
+            continue;
+        calls.push({
+            id: toolUse.toolUseId ?? "",
+            name: toolUse.name ?? "",
+            args: parseArgs(toolUse.input),
+        });
+    }
+    return calls;
+}
+export const bedrock = {
+    provider: "bedrock",
+    detect(value) {
+        const raw = value;
+        return Boolean(raw?.output && typeof raw.output === "object" && "message" in raw.output);
+    },
+    read(value, opts = {}) {
+        const raw = value;
+        const blocks = blocksOf(raw);
+        const filtered = raw.stopReason === "content_filtered";
+        const modelId = opts.model ?? "";
+        return reply({
+            output: textOf(blocks),
+            model: modelId,
+            usage: usageOf(raw.usage),
+            toolCalls: toolCallsOf(blocks),
+            stopReason: raw.stopReason,
+            status: filtered ? "failed" : "completed",
+            error: filtered ? "declined by a content filter" : undefined,
+        });
+    },
+    chunk(event, state) {
+        const e = event;
+        if (e.contentBlockStart?.start?.toolUse) {
+            const { toolUseId, name } = e.contentBlockStart.start.toolUse;
+            state.tools.set(String(e.contentBlockStart.contentBlockIndex ?? state.tools.size), {
+                id: toolUseId ?? "",
+                name: name ?? "",
+                json: "",
+            });
+            return "";
+        }
+        if (e.contentBlockDelta?.delta) {
+            const { text, toolUse } = e.contentBlockDelta.delta;
+            if (typeof toolUse?.input === "string") {
+                const key = String(e.contentBlockDelta.contentBlockIndex ?? 0);
+                const pending = state.tools.get(key);
+                if (pending)
+                    pending.json += toolUse.input;
+            }
+            if (typeof text === "string" && text) {
+                state.text += text;
+                return text;
+            }
+            return "";
+        }
+        if (e.messageStop?.stopReason)
+            state.stopReason = e.messageStop.stopReason;
+        if (e.metadata?.usage)
+            setUsage(state.usage, usageOf(e.metadata.usage));
+        return "";
+    },
+};

package/dist/adapters/gemini.d.ts

@@ -0,0 +1,2 @@
+import type { Adapter } from "../types.js";
+export declare const gemini: Adapter;

package/dist/adapters/gemini.js

@@ -0,0 +1,87 @@
+import { pricingId } from "../model.js";
+import { count, reply, setUsage } from "../usage.js";
+function usageOf(raw) {
+    return {
+        inputTokens: count(raw?.promptTokenCount),
+        outputTokens: count(raw?.candidatesTokenCount) + count(raw?.thoughtsTokenCount),
+        cacheReadTokens: count(raw?.cachedContentTokenCount),
+        cacheWriteTokens: 0,
+        citationTokens: 0,
+        reasoningTokens: 0,
+    };
+}
+function partsOf(raw) {
+    return raw.candidates?.[0]?.content?.parts ?? [];
+}
+function textOf(parts) {
+    let text = "";
+    for (const part of parts) {
+        if (!part.thought && typeof part.text === "string")
+            text += part.text;
+    }
+    return text;
+}
+function toolCallsOf(parts) {
+    const calls = [];
+    for (const part of parts) {
+        const call = part.functionCall;
+        if (!call)
+            continue;
+        calls.push({
+            id: `${call.name ?? "call"}-${calls.length}`,
+            name: call.name ?? "",
+            args: call.args ?? {},
+        });
+    }
+    return calls;
+}
+export const gemini = {
+    provider: "gemini",
+    detect(value) {
+        const raw = value;
+        return Boolean(raw && (Array.isArray(raw.candidates) || raw.usageMetadata));
+    },
+    read(value, opts = {}) {
+        const raw = value;
+        const parts = partsOf(raw);
+        const blocked = raw.promptFeedback?.blockReason;
+        const finish = raw.candidates?.[0]?.finishReason;
+        const refused = Boolean(blocked) || finish === "SAFETY" || finish === "PROHIBITED_CONTENT";
+        return reply({
+            output: textOf(parts),
+            model: opts.model ?? pricingId(opts.provider ?? "gemini", raw.modelVersion ?? ""),
+            usage: usageOf(raw.usageMetadata),
+            toolCalls: toolCallsOf(parts),
+            stopReason: finish,
+            status: refused ? "failed" : "completed",
+            error: refused ? `blocked: ${blocked ?? finish}` : undefined,
+        });
+    },
+    chunk(event, state) {
+        const raw = event;
+        if (raw.modelVersion)
+            state.model = raw.modelVersion;
+        if (raw.usageMetadata)
+            setUsage(state.usage, usageOf(raw.usageMetadata));
+        const finish = raw.candidates?.[0]?.finishReason;
+        if (finish)
+            state.stopReason = finish;
+        const parts = partsOf(raw);
+        for (const part of parts) {
+            const call = part.functionCall;
+            if (!call)
+                continue;
+            const key = String(state.tools.size);
+            state.tools.set(key, {
+                id: `${call.name ?? "call"}-${key}`,
+                name: call.name ?? "",
+                json: JSON.stringify(call.args ?? {}),
+            });
+        }
+        const text = textOf(parts);
+        if (!text)
+            return "";
+        state.text += text;
+        return text;
+    },
+};

package/dist/adapters/index.d.ts

@@ -0,0 +1,5 @@
+import type { Adapter, Provider } from "../types.js";
+export type Shape = "anthropic" | "openai" | "responses" | "bedrock" | "gemini" | "perplexity";
+export declare function adapterFor(provider: Provider): Adapter;
+export declare function adapterOfShape(shape: Shape): Adapter;
+export declare function detect(value: unknown): Adapter | undefined;

package/dist/adapters/index.js

@@ -0,0 +1,36 @@
+import { anthropic } from "./anthropic.js";
+import { bedrock } from "./bedrock.js";
+import { gemini } from "./gemini.js";
+import { openai } from "./openai.js";
+import { perplexity } from "./perplexity.js";
+import { responses } from "./responses.js";
+const BY_SHAPE = {
+    anthropic,
+    openai,
+    responses,
+    bedrock,
+    gemini,
+    perplexity,
+};
+/** Sniffed in this order, with OpenAI last because its shape is the loosest. */
+const ORDERED = [anthropic, bedrock, responses, gemini, perplexity, openai];
+/**
+ * Which wire shape a provider speaks. Everything absent is OpenAI-compatible,
+ * which is what most gateways and self-hosted servers expose.
+ */
+const SHAPE_OF = {
+    anthropic: "anthropic",
+    bedrock: "bedrock",
+    gemini: "gemini",
+    vertex_ai: "gemini",
+    perplexity: "perplexity",
+};
+export function adapterFor(provider) {
+    return BY_SHAPE[SHAPE_OF[provider] ?? "openai"];
+}
+export function adapterOfShape(shape) {
+    return BY_SHAPE[shape];
+}
+export function detect(value) {
+    return ORDERED.find((adapter) => adapter.detect(value));
+}

package/dist/adapters/openai.d.ts

@@ -0,0 +1,2 @@
+import type { Adapter } from "../types.js";
+export declare const openai: Adapter;