@north-light/sdk 0.3.624 → 0.3.625

package/dist/index.d.ts

@@ -33,7 +33,7 @@ export type { AuthorizeNonce, Connection, ConnectionStore, Identity, OAuth2Optio
 export type { IdTokenClaims } from '@north-light/identity';
 export { cursorFromRequest, forwardRunEvents } from './resources/forward.js';
 export { INFERENCE_SCOPE, inferenceEndpoint } from './resources/inference-endpoint.js';
-export type { InferenceEndpointOptions, InferenceProvider, InferenceRung, InferenceUsage } from './resources/inference-endpoint.js';
+export type { InferenceEndpointOptions, InferenceRung, InferenceSettlement, SettlementStore } from './resources/inference-endpoint.js';
 export { RUN_NOTICE_SCOPE, RunNoticeError, verifyRunNotice } from './resources/run-notice.js';
 export type { RunNotice, VerifyRunNoticeOptions } from './resources/run-notice.js';
 export type { NodeStreamEvent, NodeStreamEventListener, NodeStreamEventType } from './resources/node-stream.js';

package/dist/resources/inference-endpoint.d.ts

@@ -1,50 +1,75 @@
+import { type Units } from './inference-pricing.js';
 /** The scope an inference token carries. */
 export declare const INFERENCE_SCOPE = "crtr:llm";
 export type InferenceRung = 'ultra' | 'strong' | 'medium' | 'light';
-export interface InferenceProvider {
-    /** The vendor's own API base URL, for example `https://api.anthropic.com`. */
-    baseURL: string;
-    /** The app's key for that vendor. It replaces the token on the way upstream and is never logged or returned. */
-    apiKey: string;
-    /** The header the vendor's API carries its key in: `x-api-key` or `Authorization: Bearer`. Defaults to `x-api-key` for the provider id `anthropic` and to `bearer` for every other id. */
-    keyHeader?: 'x-api-key' | 'bearer';
+/** One model call's usage report to the directory (`POST /v1/billing/settle-model`). `list_usd` is the Anthropic list price of the served model; `cost_usd` is what Northlight paid for the call: 0 when the team Claude pool served it, `list_usd` when the app's API key did. */
+export interface InferenceSettlement {
+    admission_id: string;
+    served_model: string;
+    units: Units;
+    list_usd: number;
+    cost_usd: number;
+    vendor_ref: string;
 }
-export interface InferenceUsage {
-    /** The person the call ran for, `user:<id>` (the token's `sub`). */
-    sub: string;
-    /** The provider id, the first path segment after the mount. */
-    provider: string;
-    /** The model the request named. */
-    model: string;
-    /** The vendor's status code (always 2xx; a refused or failed request does not report). */
-    status: number;
-    /** The vendor's own `usage` object, merged across stream events; `undefined` when the answer carried none. */
-    usage: Record<string, unknown> | undefined;
+/**
+ * The app's durable queue of usage reports the directory has not yet accepted, kept in the app's own database. The helper saves a
+ * report before its first post and removes it on the directory's 2xx. `settle-model` is idempotent, so a repeat, or a second
+ * instance sweeping the same row, is harmless.
+ */
+export interface SettlementStore {
+    /** Stores a new report, due at once. */
+    save(settlement: InferenceSettlement): Promise<void>;
+    /** Deletes the report for `admissionId`; no error when it is already gone. */
+    remove(admissionId: string): Promise<void>;
+    /** Up to `limit` stored reports whose next attempt is at or before `now`, earliest first, with the posts already tried. */
+    due(now: Date, limit: number): Promise<Array<{
+        settlement: InferenceSettlement;
+        attempts: number;
+    }>>;
+    /** Sets the report's attempt count to `attempts` and its next attempt to `at`. */
+    retryLater(admissionId: string, at: Date, attempts: number): Promise<void>;
 }
 export interface InferenceEndpointOptions {
-    /** The directory issuer. */
+    /** The directory issuer. Admission and settlement are posted to its `/v1/billing` routes. */
     issuer: string;
-    /** The endpoint base URL the app registered, for example `https://context.northlight.ai/v1/inference`. The token's `aud` must equal it exactly; its path is where the handler is mounted. */
+    /** The endpoint base URL the app registered, for example `https://context.northlight.ai/v1/inference`. The token's `aud` must equal it exactly; its path is where `handle` is mounted. The endpoint serves `POST <baseURL>/anthropic/v1/messages`. */
     baseURL: string;
-    /** The app's own principal, `app:<id>`; the token's grantee must be it. */
+    /** The app's own principal, `app:<id>`; the token's actor chain must contain it. */
     app: string;
-    /** Upstream base URL and key per provider id (a model catalog provider id such as `anthropic`). */
-    providers: Record<string, InferenceProvider>;
-    /** Strength to model: the catalog model ids (`<provider id>/<model>`) the endpoint serves. A request naming any other model is refused. */
+    /** The app's Anthropic API key. It is used only for a call the directory admits as card-paid (no `pool` in the admission), and is never logged or returned. */
+    apiKey: string;
+    /** Strength to model: `anthropic/<model id>` for each rung the endpoint serves. A request naming any other model is refused. Every model needs a list price in this SDK; the constructor throws for one that has none. */
     rungs: Partial<Record<InferenceRung, string>>;
-    /** Called once per completed request, after its answer has been streamed to the caller. A throw is written to `console.error`. */
-    onUsage?: (usage: InferenceUsage) => void | Promise<void>;
-    /** The `fetch` used for the upstream call and the directory's keys (default: the global `fetch`). */
+    /** The app's `OAuth2Client.directoryToken`: a directory API token as the app, re-minted when called with the token the directory just refused. */
+    directoryToken: (rejected?: string) => Promise<string>;
+    /** Where usage reports wait until the directory accepts them. */
+    settlements: SettlementStore;
+    /** Called with ids, model and the vendor's token counts only: never the prompt, the answer or a key. */
+    log?: (entry: Record<string, unknown>) => void;
+    /** The `fetch` used for the directory, its keys and the upstream call (default: the global `fetch`). */
     fetch?: typeof fetch;
 }
 /**
- * The server side of an app-paid app: a Fetch handler that serves a person's runtime the vendors' own APIs with the
- * app's keys. Mount it at the path of `baseURL`. A request to `<baseURL>/<provider id>/<vendor path>` carries a
- * directory token in the header that vendor's API carries its key in; the handler verifies it (issuer, `aud` =
- * `baseURL`, expiry, scope `crtr:llm`, the app itself as grantee) and answers 401, sending nothing upstream, when it
- * does not hold. A provider id the app did not configure is 404. A request whose JSON body names a model outside
- * `rungs` is refused. Otherwise the request goes upstream unchanged with the app's key in place of the token, and the
- * vendor's answer comes back unchanged, status codes and `retry-after` included. The handler keeps no state, counts
- * no spend and applies no limit.
+ * The server side of an app whose model calls Northlight pays for or bills: a handler for `POST <baseURL>/anthropic/v1/messages`
+ * that serves a person's runtime the Anthropic Messages API. Mount `handle` at the path of `baseURL`; call `sweep` on a timer.
+ *
+ * The request carries a directory token (`x-api-key` or `Authorization: Bearer`) that the handler verifies (issuer, `aud` =
+ * `baseURL`, expiry, scope `crtr:llm`, the app in the actor chain); a token that does not hold is 401 and nothing is sent upstream.
+ * A server-side tool or service tier is 400, a model outside `rungs` 404, a method other than POST 405. Otherwise the directory
+ * decides, per call and from current state, whether the call is admitted: a refusal reaches the runtime as a 402 carrying the
+ * directory's `payment` body, and a directory that cannot be reached or answers anything unexpected is 503 with nothing sent upstream.
+ *
+ * The upstream comes from the admission answer. An answer carrying `pool` (Northlight pays) sends the call to the pool at
+ * `pool.url` with `pool.token`, and the call settles at list price with `cost_usd` 0. An answer without it (the person pays) sends
+ * the call to Anthropic with the app's `apiKey`, and `cost_usd` equals the list price. There is no other route: an unreachable or
+ * refusing pool answers the caller 502 or the pool's own status (a 429 keeps its `Retry-After`) and never reaches the API key.
+ *
+ * The answer streams back with `Crtr-Charge-Id` and calculated `crtr_charge` on its terminal event (or JSON body), without waiting
+ * for settlement. Usage is read off the stream and reported to the directory at the list price of the model the vendor reports it
+ * served (the requested one when it reports none), through `settlements`. A failed upstream answer reports nothing, and neither does
+ * an answer from a served model with no list price (logged with its usage): its admission stays unsettled at zero.
  */
-export declare function inferenceEndpoint(options: InferenceEndpointOptions): (request: Request) => Promise<Response>;
+export declare function inferenceEndpoint(options: InferenceEndpointOptions): {
+    handle: (request: Request) => Promise<Response>;
+    sweep: (now?: Date) => Promise<void>;
+};

package/dist/resources/inference-endpoint.js

@@ -1,149 +1,384 @@
 import { JwksCache, verifyCallToken } from '@north-light/identity';
+import { hasListPrice, listUsd, longCacheWriteOf, pricedModel, unitsOf } from './inference-pricing.js';
 /** The scope an inference token carries. */
 export const INFERENCE_SCOPE = 'crtr:llm';
-const jwksUrl = (issuer) => `${issuer.replace(/\/+$/, '')}/.well-known/jwks.json`;
+const VENDOR_URL = 'https://api.anthropic.com';
+const MESSAGES_PATH = 'v1/messages';
+const POOL_MESSAGES_PATH = 'v1/llm/anthropic/messages';
+const DIRECTORY_TIMEOUT_MS = 10_000;
+/** Posts made in the request's own lifetime before the stored report is left to `sweep`. */
+const IMMEDIATE_ATTEMPTS = 3;
+const SWEEP_BATCH = 50;
 const STRIPPED_REQUEST_HEADERS = ['host', 'connection', 'keep-alive', 'transfer-encoding', 'te', 'upgrade', 'content-length', 'authorization', 'x-api-key', 'cookie', 'proxy-authorization'];
 // The body is decoded and re-streamed, so these describe the upstream wire form, not what we send.
 const STRIPPED_RESPONSE_HEADERS = ['connection', 'keep-alive', 'transfer-encoding', 'content-encoding', 'content-length'];
-function refuse(status, code, message) {
-    return new Response(JSON.stringify({ type: 'error', error: { type: code, code, message } }), { status, headers: { 'content-type': 'application/json' } });
+const object = (value) => typeof value === 'object' && value !== null && !Array.isArray(value) ? value : undefined;
+const refuse = (status, type, message) => Response.json({ type: 'error', error: { type, message } }, { status });
+const backoffMs = (attempts) => Math.min(2 ** attempts * 1000, 60 * 60 * 1000);
+/** The bearer token of an Anthropic-style request. */
+const bearerOf = (request) => request.headers.get('x-api-key') ?? request.headers.get('authorization')?.replace(/^Bearer\s+/i, '');
+/** Whether a request asks for something the vendor bills beyond token counts, which settlement does not price: a server-side tool
+ *  (any tool with a type other than `custom`) or a service tier or speed setting. */
+function hasBilledFeature(body) {
+    if (body === undefined)
+        return false;
+    if (body.speed !== undefined)
+        return true;
+    if (body.service_tier !== undefined && body.service_tier !== 'auto')
+        return true;
+    return Array.isArray(body.tools) && body.tools.some((tool) => {
+        const type = object(tool)?.type;
+        return type !== undefined && type !== 'custom';
+    });
 }
-function keyHeaderOf(id, provider) {
-    return provider?.keyHeader ?? (id === 'anthropic' ? 'x-api-key' : 'bearer');
-}
-function tokenOf(request, header) {
-    if (header === 'x-api-key')
-        return request.headers.get('x-api-key') ?? undefined;
-    return /^Bearer ([^\s]+)$/.exec(request.headers.get('authorization') ?? '')?.[1];
-}
-/** The vendor's `usage` of a completed answer: the body's for JSON, the events' merged for a server-sent stream. */
-async function usageOf(body, contentType) {
-    const object = (value) => typeof value === 'object' && value !== null && !Array.isArray(value) ? value : undefined;
+/** What the vendor reported for a finished answer. `usage` is the merged `usage` objects of a server-sent stream (or the JSON body's),
+ *  undefined when the answer carried none; a stream cut off keeps what arrived. */
+async function reportOf(body, contentType) {
     const reader = body.getReader();
     const decoder = new TextDecoder();
-    if (contentType.includes('text/event-stream')) {
-        let merged;
-        let pending = '';
-        const take = (line) => {
-            if (!line.startsWith('data:'))
-                return;
-            let event;
-            try {
-                event = JSON.parse(line.slice(5).trim());
-            }
-            catch {
-                return;
-            }
-            const found = object(object(event)?.usage) ?? object(object(object(event)?.message)?.usage);
-            if (found !== undefined)
-                merged = { ...merged, ...found };
-        };
+    const stream = contentType.includes('text/event-stream');
+    let usage;
+    let model;
+    let pending = '';
+    let stopped = false;
+    const take = (value) => {
+        if (object(value)?.type === 'message_stop')
+            stopped = true;
+        const found = object(object(value)?.usage) ?? object(object(object(value)?.message)?.usage);
+        if (found !== undefined)
+            usage = { ...usage, ...found };
+        const named = object(value)?.model ?? object(object(value)?.message)?.model;
+        if (typeof named === 'string')
+            model = named;
+    };
+    const takeLine = (line) => {
+        if (!line.startsWith('data:'))
+            return;
+        try {
+            take(JSON.parse(line.slice(5).trim()));
+        }
+        catch { /* A line that is not JSON carries no usage. */ }
+    };
+    try {
         for (;;) {
             const { done, value } = await reader.read();
             if (done)
                 break;
             pending += decoder.decode(value, { stream: true });
-            const lines = pending.split('\n');
-            pending = lines.pop();
-            lines.forEach(take);
+            if (stream) {
+                const lines = pending.split('\n');
+                pending = lines.pop();
+                lines.forEach(takeLine);
+            }
         }
-        take(pending);
-        return merged;
     }
-    let text = '';
-    for (;;) {
-        const { done, value } = await reader.read();
-        if (done)
-            break;
-        text += decoder.decode(value, { stream: true });
-    }
-    if (!contentType.includes('json'))
-        return undefined;
-    try {
-        return object(object(JSON.parse(text))?.usage);
-    }
-    catch {
-        return undefined;
+    catch { /* The upstream stream broke: the caller's side going away does not end this read. */ }
+    if (stream)
+        takeLine(pending);
+    else if (contentType.includes('json')) {
+        try {
+            take(JSON.parse(pending));
+        }
+        catch { /* No usable body, no usage. */ }
     }
+    return { usage, model, complete: stream ? stopped : usage !== undefined };
+}
+/** Only the terminal SSE frame waits for pricing; text and tools keep streaming. */
+function chargedBody(body, contentType, charge) {
+    const decoder = new TextDecoder();
+    const encoder = new TextEncoder();
+    const stream = contentType.includes('text/event-stream');
+    if (!stream && !contentType.includes('json'))
+        return body;
+    let pending = '';
+    const frame = async (text) => {
+        const lines = text.split('\n');
+        for (let i = 0; i < lines.length; i++) {
+            if (!lines[i].startsWith('data:'))
+                continue;
+            let value;
+            try {
+                value = object(JSON.parse(lines[i].slice(5).trim()));
+            }
+            catch {
+                continue;
+            }
+            if (value?.type !== 'message_stop')
+                continue;
+            const calculated = await charge;
+            if (calculated !== undefined)
+                lines[i] = `data: ${JSON.stringify({ ...value, crtr_charge: calculated })}`;
+        }
+        return lines.join('\n');
+    };
+    return body.pipeThrough(new TransformStream({
+        async transform(chunk, controller) {
+            pending += decoder.decode(chunk, { stream: true });
+            if (!stream)
+                return;
+            for (;;) {
+                const boundary = /\r?\n\r?\n/.exec(pending);
+                if (boundary === null)
+                    break;
+                const end = boundary.index + boundary[0].length;
+                const text = pending.slice(0, end);
+                pending = pending.slice(end);
+                controller.enqueue(encoder.encode(await frame(text)));
+            }
+        },
+        async flush(controller) {
+            pending += decoder.decode();
+            if (stream)
+                pending = await frame(pending);
+            else if (contentType.includes('json')) {
+                const calculated = await charge;
+                if (calculated !== undefined)
+                    pending = JSON.stringify({ ...object(JSON.parse(pending)), crtr_charge: calculated });
+            }
+            controller.enqueue(encoder.encode(pending));
+        },
+    }));
 }
 /**
- * The server side of an app-paid app: a Fetch handler that serves a person's runtime the vendors' own APIs with the
- * app's keys. Mount it at the path of `baseURL`. A request to `<baseURL>/<provider id>/<vendor path>` carries a
- * directory token in the header that vendor's API carries its key in; the handler verifies it (issuer, `aud` =
- * `baseURL`, expiry, scope `crtr:llm`, the app itself as grantee) and answers 401, sending nothing upstream, when it
- * does not hold. A provider id the app did not configure is 404. A request whose JSON body names a model outside
- * `rungs` is refused. Otherwise the request goes upstream unchanged with the app's key in place of the token, and the
- * vendor's answer comes back unchanged, status codes and `retry-after` included. The handler keeps no state, counts
- * no spend and applies no limit.
+ * The server side of an app whose model calls Northlight pays for or bills: a handler for `POST <baseURL>/anthropic/v1/messages`
+ * that serves a person's runtime the Anthropic Messages API. Mount `handle` at the path of `baseURL`; call `sweep` on a timer.
+ *
+ * The request carries a directory token (`x-api-key` or `Authorization: Bearer`) that the handler verifies (issuer, `aud` =
+ * `baseURL`, expiry, scope `crtr:llm`, the app in the actor chain); a token that does not hold is 401 and nothing is sent upstream.
+ * A server-side tool or service tier is 400, a model outside `rungs` 404, a method other than POST 405. Otherwise the directory
+ * decides, per call and from current state, whether the call is admitted: a refusal reaches the runtime as a 402 carrying the
+ * directory's `payment` body, and a directory that cannot be reached or answers anything unexpected is 503 with nothing sent upstream.
+ *
+ * The upstream comes from the admission answer. An answer carrying `pool` (Northlight pays) sends the call to the pool at
+ * `pool.url` with `pool.token`, and the call settles at list price with `cost_usd` 0. An answer without it (the person pays) sends
+ * the call to Anthropic with the app's `apiKey`, and `cost_usd` equals the list price. There is no other route: an unreachable or
+ * refusing pool answers the caller 502 or the pool's own status (a 429 keeps its `Retry-After`) and never reaches the API key.
+ *
+ * The answer streams back with `Crtr-Charge-Id` and calculated `crtr_charge` on its terminal event (or JSON body), without waiting
+ * for settlement. Usage is read off the stream and reported to the directory at the list price of the model the vendor reports it
+ * served (the requested one when it reports none), through `settlements`. A failed upstream answer reports nothing, and neither does
+ * an answer from a served model with no list price (logged with its usage): its admission stays unsettled at zero.
  */
 export function inferenceEndpoint(options) {
     const upstreamFetch = options.fetch ?? ((input, init) => fetch(input, init));
+    const log = (entry) => options.log?.(entry);
     const mount = new URL(options.baseURL).pathname.replace(/\/+$/, '');
-    const served = new Set(Object.values(options.rungs));
+    const messagesPath = `${mount}/anthropic/${MESSAGES_PATH}`;
+    const served = new Set();
+    for (const spec of Object.values(options.rungs)) {
+        const model = spec.startsWith('anthropic/') ? spec.slice('anthropic/'.length) : undefined;
+        if (model === undefined || !hasListPrice(model))
+            throw new Error(`inferenceEndpoint: rung model ${spec} must be anthropic/<model> with a list price`);
+        served.add(model);
+    }
     if (served.size === 0)
         throw new Error('inferenceEndpoint: rungs must name at least one model');
-    for (const model of served) {
-        const provider = model.split('/')[0];
-        if (!model.includes('/') || options.providers[provider] === undefined)
-            throw new Error(`inferenceEndpoint: rung model ${model} must be <provider id>/<model> for a configured provider`);
-    }
-    const jwks = new JwksCache(options.issuer, (input, init) => upstreamFetch(input, init), jwksUrl(options.issuer));
-    return async (request) => {
-        const url = new URL(request.url);
-        if (!url.pathname.startsWith(`${mount}/`))
+    const issuerBase = options.issuer.replace(/\/+$/, '');
+    const jwks = new JwksCache(options.issuer, (input, init) => upstreamFetch(input, init), `${issuerBase}/.well-known/jwks.json`);
+    const vendorRefPrefix = options.app.replace(/^app:/, '');
+    /** A POST to the directory as the app; a 401 hands the refused token back and retries once with a fresh one. */
+    const directoryPost = async (path, body) => {
+        const send = (token) => upstreamFetch(new URL(path, issuerBase), {
+            method: 'POST',
+            headers: { accept: 'application/json', 'content-type': 'application/json', authorization: `Bearer ${token}` },
+            body: JSON.stringify(body),
+            signal: AbortSignal.timeout(DIRECTORY_TIMEOUT_MS),
+        });
+        const token = await options.directoryToken();
+        let response = await send(token);
+        if (response.status === 401)
+            response = await send(await options.directoryToken(token));
+        return { status: response.status, json: await response.json().catch(() => undefined) };
+    };
+    /** Asks the directory to admit one model call. Anything but an explicit admission or a payment refusal is a refusal to serve. */
+    const admit = async (token, model) => {
+        let answer;
+        try {
+            answer = await directoryPost('/v1/billing/admit-model', { token, model });
+        }
+        catch {
+            return { kind: 'unavailable' };
+        }
+        const body = object(answer.json);
+        if (answer.status === 200) {
+            const { status, admission_id, charge_id, pool } = body ?? {};
+            if (status !== 'ok' || typeof admission_id !== 'string' || typeof charge_id !== 'string')
+                return { kind: 'unavailable' };
+            if (pool === undefined)
+                return { kind: 'admitted', admissionId: admission_id, chargeId: charge_id, pool: undefined };
+            const { url, token: poolToken } = object(pool) ?? {};
+            // A pool the helper cannot read is not an admission for the API key.
+            return typeof url === 'string' && url !== '' && typeof poolToken === 'string' && poolToken !== ''
+                ? { kind: 'admitted', admissionId: admission_id, chargeId: charge_id, pool: { url: url.replace(/\/+$/, ''), token: poolToken } }
+                : { kind: 'unavailable' };
+        }
+        const error = object(body?.error);
+        if (answer.status === 402) {
+            const payment = object(object(error?.details)?.payment);
+            return payment && typeof payment.reason === 'string' && typeof payment.message === 'string'
+                ? { kind: 'refused', payment: {
+                        reason: payment.reason,
+                        message: payment.message,
+                        pay_url: typeof payment.pay_url === 'string' ? payment.pay_url : null,
+                        reset_at: typeof payment.reset_at === 'string' ? payment.reset_at : null,
+                    } }
+                : { kind: 'unavailable' };
+        }
+        return answer.status === 403 && typeof error?.message === 'string' ? { kind: 'forbidden', message: error.message } : { kind: 'unavailable' };
+    };
+    /** True once the directory answered 2xx (settled or duplicate) and the stored report is removed. */
+    const deliver = async (settlement) => {
+        let status;
+        try {
+            status = (await directoryPost('/v1/billing/settle-model', settlement)).status;
+        }
+        catch (error) {
+            log({ settle_error: String(error), admission: settlement.admission_id });
+            return false;
+        }
+        // Only a 2xx means the directory priced the charge. Anything else keeps the report, which `sweep` posts again.
+        if (status < 200 || status >= 300) {
+            log({ settle_status: status, admission: settlement.admission_id });
+            return false;
+        }
+        await options.settlements.remove(settlement.admission_id);
+        return true;
+    };
+    const record = async (settlement) => {
+        await options.settlements.save(settlement);
+        for (let attempt = 0; attempt < IMMEDIATE_ATTEMPTS; attempt++) {
+            if (await deliver(settlement))
+                return;
+            if (attempt < IMMEDIATE_ATTEMPTS - 1)
+                await new Promise((resolve) => setTimeout(resolve, 200 * 2 ** attempt));
+        }
+    };
+    const sweep = async (now = new Date()) => {
+        for (const { settlement, attempts } of await options.settlements.due(now, SWEEP_BATCH)) {
+            if (await deliver(settlement))
+                continue;
+            await options.settlements.retryLater(settlement.admission_id, new Date(now.getTime() + backoffMs(attempts + 1)), attempts + 1);
+        }
+    };
+    const handle = async (request) => {
+        if (new URL(request.url).pathname !== messagesPath)
             return refuse(404, 'not_found', 'No such path.');
-        const [providerId, ...restSegments] = url.pathname.slice(mount.length + 1).split('/');
-        const provider = Object.hasOwn(options.providers, providerId) ? options.providers[providerId] : undefined;
-        const token = tokenOf(request, keyHeaderOf(providerId, provider));
+        const token = bearerOf(request);
         if (token === undefined)
             return refuse(401, 'unauthorized', 'The request carries no directory token.');
-        let verified;
+        let sub;
         try {
-            verified = await verifyCallToken({ token, jwks, issuer: options.issuer, audience: options.baseURL });
+            const verified = await verifyCallToken({ token, jwks, issuer: options.issuer, audience: options.baseURL });
+            if (!(verified.scope ?? '').split(' ').includes(INFERENCE_SCOPE) || !verified.chain.some((link) => link.sub === options.app))
+                throw new Error('scope');
+            sub = verified.sub;
         }
         catch {
             return refuse(401, 'unauthorized', 'The token is not a valid directory token for this endpoint.');
         }
-        if (!(verified.scope ?? '').split(' ').includes(INFERENCE_SCOPE) || !verified.chain.some((link) => link.sub === options.app)) {
-            return refuse(401, 'unauthorized', 'The token was not minted for this app to serve inference.');
-        }
-        if (provider === undefined)
-            return refuse(404, 'not_found', `This endpoint serves no provider ${providerId}.`);
-        const body = request.method === 'GET' || request.method === 'HEAD' ? undefined : await request.arrayBuffer();
+        if (request.method !== 'POST')
+            return refuse(405, 'invalid_request_error', 'Only POST is served.');
+        const body = await request.arrayBuffer();
         let model;
+        let maxTokens = 0;
+        let billedFeature = false;
         try {
-            model = body === undefined ? undefined : JSON.parse(new TextDecoder().decode(body)).model;
+            const parsed = object(JSON.parse(new TextDecoder().decode(body)));
+            model = parsed?.model;
+            if (hasBilledFeature(parsed))
+                billedFeature = true;
+            if (typeof parsed?.max_tokens === 'number' && parsed.max_tokens > 0)
+                maxTokens = parsed.max_tokens;
         }
         catch {
             model = undefined;
         }
+        if (billedFeature)
+            return refuse(400, 'invalid_request', 'Server-side tools and service tiers are not served here.');
         if (typeof model !== 'string')
             return refuse(400, 'invalid_request', 'The request body must be JSON naming a model.');
-        if (!served.has(`${providerId}/${model}`))
-            return refuse(404, 'model_unavailable', `This endpoint does not serve ${providerId}/${model}.`);
+        if (!served.has(model))
+            return refuse(404, 'model_unavailable', `This endpoint does not serve ${model}.`);
+        const admission = await admit(token, model);
+        if (admission.kind === 'refused') {
+            return Response.json({ type: 'error', error: { type: 'invalid_request_error', message: admission.payment.message }, payment: admission.payment }, { status: 402 });
+        }
+        if (admission.kind === 'forbidden')
+            return refuse(403, 'permission_error', admission.message);
+        if (admission.kind === 'unavailable')
+            return refuse(503, 'admission_unavailable', 'Northlight could not confirm payment for this call. Try again shortly.');
         const headers = new Headers(request.headers);
         for (const name of STRIPPED_REQUEST_HEADERS)
             headers.delete(name);
-        if (keyHeaderOf(providerId, provider) === 'x-api-key')
-            headers.set('x-api-key', provider.apiKey);
+        const pool = admission.pool;
+        if (pool !== undefined)
+            headers.set('authorization', `Bearer ${pool.token}`);
         else
-            headers.set('authorization', `Bearer ${provider.apiKey}`);
-        let upstream;
+            headers.set('x-api-key', options.apiKey);
+        const upstreamUrl = pool !== undefined ? `${pool.url}/${POOL_MESSAGES_PATH}` : `${VENDOR_URL}/${MESSAGES_PATH}`;
+        let answer;
         try {
-            upstream = await upstreamFetch(`${provider.baseURL.replace(/\/+$/, '')}/${restSegments.join('/')}${url.search}`, { method: request.method, headers, body, redirect: 'manual', signal: request.signal });
+            // Not tied to the caller's connection: the vendor bills the whole answer, so the usage branch reads it to its end and the settlement is complete.
+            answer = await upstreamFetch(upstreamUrl, { method: 'POST', headers, body, redirect: 'manual' });
         }
         catch {
-            return refuse(502, 'upstream_unreachable', `${providerId} could not be reached.`);
+            return refuse(502, 'upstream_unreachable', `${pool !== undefined ? 'The Claude pool' : 'anthropic'} could not be reached.`);
         }
-        const responseHeaders = new Headers(upstream.headers);
+        const responseHeaders = new Headers(answer.headers);
         for (const name of STRIPPED_RESPONSE_HEADERS)
             responseHeaders.delete(name);
-        if (upstream.body === null || options.onUsage === undefined || !upstream.ok) {
-            return new Response(upstream.body, { status: upstream.status, statusText: upstream.statusText, headers: responseHeaders });
+        if (answer.body === null || !answer.ok) {
+            return new Response(answer.body, { status: answer.status, statusText: answer.statusText, headers: responseHeaders });
         }
-        const [toCaller, toUsage] = upstream.body.tee();
-        const report = options.onUsage;
-        void usageOf(toUsage, upstream.headers.get('content-type') ?? '').then((usage) => report({ sub: verified.sub, provider: providerId, model, status: upstream.status, usage }), () => undefined).catch((error) => { console.error('inferenceEndpoint onUsage failed:', error); });
-        return new Response(toCaller, { status: upstream.status, statusText: upstream.statusText, headers: responseHeaders });
+        responseHeaders.set('Crtr-Charge-Id', admission.chargeId);
+        const [toCaller, toUsage] = answer.body.tee();
+        const vendorRef = answer.headers.get('request-id') ?? `${vendorRefPrefix}:${admission.admissionId}`;
+        const contentType = answer.headers.get('content-type') ?? '';
+        const calculated = reportOf(toUsage, contentType).then(({ usage, model: servedByVendor, complete }) => {
+            if (usage === undefined) {
+                log({ sub, model, admission: admission.admissionId, unsettled: 'no_usage' });
+                return undefined;
+            }
+            if (complete && ![
+                usage.input_tokens,
+                usage.output_tokens,
+                usage.cache_read_input_tokens === undefined ? 0 : usage.cache_read_input_tokens,
+                usage.cache_creation_input_tokens === undefined ? 0 : usage.cache_creation_input_tokens,
+            ].every((count) => typeof count === 'number' && Number.isSafeInteger(count) && count >= 0)) {
+                log({ sub, model, admission: admission.admissionId, usage, unsettled: 'invalid_usage' });
+                return undefined;
+            }
+            const units = unitsOf(usage);
+            // A stream that ended before its final usage event reports only the prompt side: price the output the request allowed instead.
+            if (!complete)
+                units.output = Math.max(units.output, maxTokens);
+            // The vendor may serve another model than the one asked for: the charge is that model's, at its own rates.
+            const servedModel = servedByVendor ?? model;
+            const priced = pricedModel(servedModel);
+            if (priced === undefined) {
+                log({ sub, model, served_model: servedModel, admission: admission.admissionId, usage, unsettled: 'unpriced_served_model' });
+                return undefined;
+            }
+            const list = listUsd(priced, units, longCacheWriteOf(usage, units));
+            const settlement = {
+                admission_id: admission.admissionId,
+                served_model: servedModel,
+                units,
+                list_usd: list,
+                cost_usd: pool !== undefined ? 0 : list,
+                vendor_ref: vendorRef,
+            };
+            return { settlement, usage };
+        });
+        void calculated.then(async (result) => {
+            if (result === undefined)
+                return;
+            const { settlement, usage } = result;
+            await record(settlement);
+            log({ sub, model, served_model: settlement.served_model, admission: admission.admissionId, upstream: pool !== undefined ? 'pool' : 'api_key', list_usd: settlement.list_usd, cost_usd: settlement.cost_usd, usage });
+        }).catch((error) => { log({ sub, admission: admission.admissionId, settle_failed: String(error) }); });
+        const charge = calculated.then((result) => result === undefined ? undefined : { id: admission.chargeId, served_model: result.settlement.served_model, list_usd: result.settlement.list_usd }, () => undefined);
+        return new Response(chargedBody(toCaller, contentType, charge), { status: answer.status, statusText: answer.statusText, headers: responseHeaders });
     };
+    return { handle, sweep };
 }

package/dist/resources/inference-pricing.d.ts

@@ -0,0 +1,18 @@
+/** Whether the bare model id has a list price. */
+export declare const hasListPrice: (model: string) => boolean;
+/** The priced model a vendor-reported model id is: the id itself, or a dated snapshot of it (`claude-sonnet-5-5-20261001`).
+ *  Undefined for a model with no list price, which must never be settled at another model's rates. */
+export declare function pricedModel(reported: string): string | undefined;
+export interface Units {
+    input: number;
+    output: number;
+    cache_read: number;
+    cache_write: number;
+}
+/** The vendor's own `usage` object (`input_tokens`, `output_tokens`, `cache_read_input_tokens`, `cache_creation_input_tokens`) as settlement units. */
+export declare const unitsOf: (usage: Record<string, unknown>) => Units;
+/** The one-hour cache-write tokens of the vendor's `usage` (`cache_creation.ephemeral_1h_input_tokens`), never more than the call's cache writes. */
+export declare function longCacheWriteOf(usage: Record<string, unknown>, units: Units): number;
+/** The list price in dollars of one call's units. `cacheWrite1h` of the units' cache writes were one-hour writes, which Anthropic
+ *  bills at twice the base input rate; the rest are five-minute writes. Throws for a model with no entry: the handler serves only priced models. */
+export declare function listUsd(model: string, units: Units, cacheWrite1h?: number): number;

package/dist/resources/inference-pricing.js

@@ -0,0 +1,48 @@
+const PRICES = {
+    'claude-fable-5': { input: 10, output: 50, cacheRead: 1, cacheWrite: 12.5 },
+    'claude-fable-5-1': { input: 10, output: 50, cacheRead: 0.25, cacheWrite: 12.5 },
+    'claude-opus-5-5': { input: 4, output: 20, cacheRead: 0.2, cacheWrite: 5 },
+    'claude-sonnet-5-5': { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5 },
+    'claude-haiku-5-5': {
+        input: 0.1, output: 0.5, cacheRead: 0.01, cacheWrite: 0.125,
+        tier: { above: 100_000, rates: { input: 0.5, output: 2.5, cacheRead: 0.05, cacheWrite: 0.625 } },
+    },
+};
+/** Whether the bare model id has a list price. */
+export const hasListPrice = (model) => Object.hasOwn(PRICES, model);
+/** The priced model a vendor-reported model id is: the id itself, or a dated snapshot of it (`claude-sonnet-5-5-20261001`).
+ *  Undefined for a model with no list price, which must never be settled at another model's rates. */
+export function pricedModel(reported) {
+    if (hasListPrice(reported))
+        return reported;
+    const undated = reported.replace(/-\d{8}$/, '');
+    return undated !== reported && hasListPrice(undated) ? undated : undefined;
+}
+const tokens = (usage, key) => {
+    const value = usage[key];
+    return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0;
+};
+/** The vendor's own `usage` object (`input_tokens`, `output_tokens`, `cache_read_input_tokens`, `cache_creation_input_tokens`) as settlement units. */
+export const unitsOf = (usage) => ({
+    input: tokens(usage, 'input_tokens'),
+    output: tokens(usage, 'output_tokens'),
+    cache_read: tokens(usage, 'cache_read_input_tokens'),
+    cache_write: tokens(usage, 'cache_creation_input_tokens'),
+});
+/** The one-hour cache-write tokens of the vendor's `usage` (`cache_creation.ephemeral_1h_input_tokens`), never more than the call's cache writes. */
+export function longCacheWriteOf(usage, units) {
+    const creation = usage['cache_creation'];
+    const nested = typeof creation === 'object' && creation !== null ? creation['ephemeral_1h_input_tokens'] : undefined;
+    return typeof nested === 'number' && Number.isFinite(nested) && nested > 0 ? Math.min(nested, units.cache_write) : 0;
+}
+/** The list price in dollars of one call's units. `cacheWrite1h` of the units' cache writes were one-hour writes, which Anthropic
+ *  bills at twice the base input rate; the rest are five-minute writes. Throws for a model with no entry: the handler serves only priced models. */
+export function listUsd(model, units, cacheWrite1h = 0) {
+    const price = PRICES[model];
+    if (price === undefined)
+        throw new Error(`No list price for ${model}`);
+    const rates = price.tier !== undefined && units.input + units.cache_read + units.cache_write > price.tier.above ? price.tier.rates : price;
+    const micro = units.input * rates.input + units.output * rates.output + units.cache_read * rates.cacheRead + (units.cache_write - cacheWrite1h) * rates.cacheWrite + cacheWrite1h * rates.input * 2;
+    // dollars per million tokens is micro-dollars per token
+    return Math.round(micro) / 1_000_000;
+}

package/dist/skill/northlight/SKILL.md

@@ -2,7 +2,7 @@
 name: northlight
 description: "Use when building, debugging or shipping an app that signs people in with Northlight, starts runs on their Northlight runtime, or calls Northlight providers (sms, search, integrations, context, device) with @north-light/sdk."
 metadata:
-  sdk-version: "0.3.624"
+  sdk-version: "0.3.625"
 ---
 
 # Building a Northlight app
@@ -27,7 +27,7 @@ Never text a person first: Comms refuses a send to anyone who has not texted you
 
 ## First: check this skill matches the SDK
 
-This skill is for SDK version `0.3.624`. If the project has `node_modules/@north-light/sdk/package.json`, compare its `version` with that number. If they differ, run `npx @north-light/sdk skill` in the project, then read the rewritten `.agents/skills/northlight/SKILL.md` (or `.claude/skills/northlight/SKILL.md`) instead of this copy. It needs no account. A project that has no SDK yet starts with `npx @north-light/sdk@latest init <dir>`.
+This skill is for SDK version `0.3.625`. If the project has `node_modules/@north-light/sdk/package.json`, compare its `version` with that number. If they differ, run `npx @north-light/sdk skill` in the project, then read the rewritten `.agents/skills/northlight/SKILL.md` (or `.claude/skills/northlight/SKILL.md`) instead of this copy. It needs no account. A project that has no SDK yet starts with `npx @north-light/sdk@latest init <dir>`.
 
 ## Where things run
 

package/dist/version.d.ts

@@ -1,5 +1,5 @@
 /** This SDK's release version. Informational only: compatibility is the contract level, not this number. */
-export declare const SDK_VERSION = "0.3.624";
+export declare const SDK_VERSION = "0.3.625";
 /** The release sent in `Crouter-Minimum-Runtime-Version` on every request. It is frozen, so a runtime from before the
  *  contract level existed (which only knows this header) refuses this SDK, while every later runtime accepts it. */
 export declare const LEGACY_FLOOR = "0.3.423";

package/dist/version.js

@@ -1,6 +1,6 @@
 // SDK_VERSION is written by scripts/release-version.mjs on every release; do not edit by hand.
 /** This SDK's release version. Informational only: compatibility is the contract level, not this number. */
-export const SDK_VERSION = '0.3.624';
+export const SDK_VERSION = '0.3.625';
 // LEGACY_FLOOR is written once by `scripts/contract.mjs release`, at the first release that carries the contract level
 // (the `0.0.0` placeholder is replaced there and never again); do not edit by hand.
 /** The release sent in `Crouter-Minimum-Runtime-Version` on every request. It is frozen, so a runtime from before the

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@north-light/sdk",
-  "version": "0.3.624",
-  "crouterContract": "1.49",
+  "version": "0.3.625",
+  "crouterContract": "1.50",
   "description": "Typed Node and browser client for running crouter agents through the crtrd /v1 API.",
   "type": "module",
   "main": "./dist/index.js",
@@ -44,9 +44,9 @@
     "prepublishOnly": "node -e \"if (process.env.npm_config_tag !== 'next') { console.error('@north-light/sdk publishes only with --tag next; scripts/promote-sdk.mjs is the only mover of latest.'); process.exit(1); }\""
   },
   "dependencies": {
-    "@north-light/api": "0.3.624",
-    "@north-light/identity": "0.3.624",
-    "@north-light/plugin": "0.3.624",
+    "@north-light/api": "0.3.625",
+    "@north-light/identity": "0.3.625",
+    "@north-light/plugin": "0.3.625",
     "jose": "^6.2.1"
   },
   "peerDependencies": {