@plurnk/plurnk-providers 1.5.0 → 1.6.1

package/src/capacity.ts

@@ -0,0 +1,140 @@
+import type {
+    PromptTokenMeasurement,
+    ProviderRequestCapacity,
+} from "./types.ts";
+import { assertPromptTokenMeasurement } from "./promptTokens.ts";
+
+const positiveOrNull = (value: number | null, name: string): number | null => {
+    if (value === null) return null;
+    if (!Number.isSafeInteger(value) || value <= 0) {
+        throw new TypeError(`${name} must be a positive safe integer or null`);
+    }
+    return value;
+};
+
+export const effectiveOutputBudget = ({
+    requested,
+    configured,
+    maxOutputTokens,
+    contextWindow,
+}: {
+    requested?: number;
+    configured: number | null;
+    maxOutputTokens: number | null;
+    contextWindow: number | null;
+}): number | null => {
+    if (requested !== undefined && (!Number.isSafeInteger(requested) || requested <= 0)) {
+        throw new TypeError("maxOutputTokens must be a positive safe integer");
+    }
+    const policies = [requested ?? null, configured].filter((value): value is number => value !== null);
+    if (policies.length === 0) return null;
+    const physical = [
+        positiveOrNull(maxOutputTokens, "maxOutputTokens"),
+        positiveOrNull(contextWindow, "contextWindow"),
+    ].filter((value): value is number => value !== null);
+    return Math.min(...policies, ...physical);
+};
+
+export const effectiveReasoningBudget = ({
+    configured,
+    outputBudget,
+}: {
+    configured: number | null;
+    outputBudget: number | null;
+}): number | null => {
+    positiveOrNull(configured, "reasoningBudget");
+    positiveOrNull(outputBudget, "outputBudget");
+    if (configured === null) return null;
+    if (outputBudget === null) {
+        throw new TypeError("a reasoning budget requires a resolved total output budget");
+    }
+    if (outputBudget < 2) {
+        throw new TypeError("maxOutputTokens must leave at least one token outside the reasoning budget");
+    }
+    return Math.min(configured, outputBudget - 1);
+};
+
+export const effectiveInputCapacity = ({
+    contextWindow,
+    maxInputTokens,
+    outputBudget,
+}: {
+    contextWindow: number | null;
+    maxInputTokens: number | null;
+    outputBudget: number | null;
+}): number | null => {
+    positiveOrNull(contextWindow, "contextWindow");
+    positiveOrNull(maxInputTokens, "maxInputTokens");
+    positiveOrNull(outputBudget, "outputBudget");
+    const combinedCapacity = contextWindow !== null && outputBudget !== null
+        ? contextWindow - outputBudget
+        : null;
+    if (combinedCapacity !== null && combinedCapacity <= 0) {
+        throw new TypeError(
+            `outputBudget (${outputBudget}) must leave positive input capacity inside contextWindow (${contextWindow})`,
+        );
+    }
+    const capacities = [
+        maxInputTokens,
+        combinedCapacity,
+    ].filter((value): value is number => value !== null);
+    return capacities.length === 0 ? null : Math.min(...capacities);
+};
+
+export const requestCapacityDecision = (
+    inputCapacity: number | null,
+    measurement: PromptTokenMeasurement,
+): ProviderRequestCapacity["decision"] => {
+    const prompt = assertPromptTokenMeasurement(measurement, "provider capacity");
+    // An upper bound can prove fit when it is below the limit, but exceeding
+    // the limit proves nothing about the unknown exact count. Estimates never
+    // authorize or reject; the provider remains the capacity oracle.
+    return inputCapacity === null
+        || prompt.kind === "estimate"
+        || prompt.kind === "unavailable"
+        ? "defer"
+        : prompt.tokens <= inputCapacity
+            ? "admit"
+            : prompt.kind === "exact"
+                ? "reject"
+                : "defer";
+};
+
+export const assessRequestCapacity = ({
+    contextWindow,
+    maxInputTokens,
+    maxOutputTokens,
+    outputBudget,
+    reasoningBudget,
+    measurement,
+}: {
+    contextWindow: number | null;
+    maxInputTokens: number | null;
+    maxOutputTokens: number | null;
+    outputBudget: number | null;
+    reasoningBudget: number | null;
+    measurement: PromptTokenMeasurement;
+}): ProviderRequestCapacity => {
+    positiveOrNull(contextWindow, "contextWindow");
+    positiveOrNull(maxInputTokens, "maxInputTokens");
+    positiveOrNull(maxOutputTokens, "maxOutputTokens");
+    positiveOrNull(outputBudget, "outputBudget");
+    positiveOrNull(reasoningBudget, "reasoningBudget");
+    if (reasoningBudget !== null
+        && (outputBudget === null || reasoningBudget >= outputBudget)) {
+        throw new TypeError("reasoningBudget must be a strict subset of outputBudget");
+    }
+    const prompt = assertPromptTokenMeasurement(measurement, "provider capacity");
+    const inputCapacity = effectiveInputCapacity({ contextWindow, maxInputTokens, outputBudget });
+
+    return {
+        decision: requestCapacityDecision(inputCapacity, prompt),
+        contextWindow,
+        maxInputTokens,
+        maxOutputTokens,
+        outputBudget,
+        reasoningBudget,
+        inputCapacity,
+        prompt,
+    };
+};

package/src/catalogProvider.test.ts

@@ -1,23 +1,26 @@
 import test, { mock } from "node:test";
 import { strict as assert } from "node:assert";
+import { once } from "node:events";
+import { createServer } from "node:http";
 import { catalogProviderFromEnv } from "./catalogProvider.ts";
-import { providerCostFor } from "./cost.ts";
 import { resetEmittedWarnings } from "./warnings.ts";
 
 const env = {
     OPENAI_API_KEY: "test-key",
     OPENAI_BASE_URL: "https://api.openai.com/v1",
     PLURNK_PROVIDERS_FETCH_TIMEOUT: "1000",
+    PLURNK_PROVIDERS_OPERATION_TIMEOUT: "3000",
+    PLURNK_PROVIDERS_FIRST_CONTENT_TIMEOUT: "1000",
     PLURNK_PROVIDERS_STREAM_IDLE_TIMEOUT: "0",
     PLURNK_PROVIDERS_REASONING: "off",
     PLURNK_PROVIDERS_TEMPERATURE: "0.2",
     PLURNK_PROVIDERS_REPEAT_PENALTY: "1.15",
     PLURNK_PROVIDERS_FREQUENCY_PENALTY: "0",
-    PLURNK_PROVIDERS_REASONING_RESERVE: "10%",
-    PLURNK_PROVIDERS_COMPLETION_RESERVE: "25%",
+    PLURNK_PROVIDERS_OUTPUT_BUDGET: "35%",
     PLURNK_PROVIDERS_RETRY_ATTEMPTS: "0",
     PLURNK_PROVIDERS_ERROR_DETAIL_LIMIT: "512",
-    PLURNK_PROVIDERS_PROMPT_CACHE_KEY: "1",
+    PLURNK_PROVIDERS_CACHE_AFFINITY: "1",
+    PLURNK_PROVIDERS_CACHE_WRITE_POLICY: "stable-system",
 };
 
 test.afterEach(() => {
@@ -30,25 +33,20 @@ test("catalog provider resolves model physics and Models.dev USD rates", () => {
     assert.notEqual(provider, null);
     assert.equal(provider?.model, "gpt-4.1-mini");
     assert.equal(provider?.contextWindow, 1_047_576);
-    assert.equal(provider?.reasoningReserve, 16_384);
-    assert.equal(provider?.completionReserve, 32_768);
-    assert.ok((provider?.calculateCost({
-        prompt: 1_000_000,
-        completion: 1_000_000,
-        reasoning: 0,
-        cached: 0,
-        total: 2_000_000,
-    }) ?? 0) > 0);
+    assert.equal(provider?.maxInputTokens, null);
+    assert.equal(provider?.maxOutputTokens, 32_768);
+    assert.equal(provider?.outputBudget, 32_768);
+    assert.equal(provider?.reasoningBudget, null);
 });
 
-test("an operator context window caps catalog physics and percentage reserves derive from the cap", () => {
+test("an operator context window caps catalog physics and percentage output policy", () => {
     const provider = catalogProviderFromEnv("openai", {
         ...env,
         PLURNK_PROVIDERS_CONTEXT_WINDOW: "128000",
     }, "gpt-4.1-mini");
     assert.equal(provider?.contextWindow, 128_000);
-    assert.equal(provider?.reasoningReserve, 16_000);
-    assert.equal(provider?.completionReserve, 32_000);
+    assert.equal(provider?.outputBudget, 32_768, "the model output maximum remains the tighter cap");
+    assert.equal(provider?.reasoningBudget, null);
 
     const oversized = catalogProviderFromEnv("openai", {
         ...env,
@@ -92,12 +90,16 @@ test("official AI SDK provider owns the native request while PLURNK owns call se
     const result = await provider?.generate({
         workerId: "worker",
         messages: [{ role: "user", content: "hello" }],
-        maxTokens: 64,
+        maxOutputTokens: 64,
         sampling: { top_p: 0.8, seed: 7 },
     });
 
     assert.equal(result?.assistant.content, "done");
-    assert.equal(result?.assistant.usage.total, 3);
+    assert.equal(result?.accounting[0]?.usage?.totalTokens, 3);
+    assert.deepEqual(result?.accounting[0]?.cost, {
+        kind: "unknown",
+        reason: "the provider response omitted a token category with a distinct Models.dev rate",
+    });
     assert.equal(calls.length, 1);
     assert.equal(calls[0]?.url, "https://api.openai.com/v1/chat/completions");
     assert.equal(calls[0]?.body.model, "gpt-4.1-mini");
@@ -105,6 +107,299 @@ test("official AI SDK provider owns the native request while PLURNK owns call se
     assert.equal(calls[0]?.body.top_p, 0.8);
     assert.equal(calls[0]?.body.seed, 7);
     assert.equal(calls[0]?.body.max_tokens, 64);
+    assert.equal(calls[0]?.body.prompt_cache_key, "worker", "the official OpenAI SDK projects the documented affinity key");
+});
+
+test("xAI's native chat contract caps the complete reasoning response", async () => {
+    let call: { headers: Headers; body: Record<string, unknown> } | undefined;
+    mock.method(globalThis, "fetch", async (_input: string | URL | Request, init?: RequestInit) => {
+        call = {
+            headers: new Headers(init?.headers),
+            body: JSON.parse(String(init?.body)) as Record<string, unknown>,
+        };
+        return new Response([
+            `data: ${JSON.stringify({
+                id: "response-xai",
+                object: "chat.completion.chunk",
+                created: 1,
+                model: "grok-build-0.1",
+                choices: [{ index: 0, delta: { reasoning_content: "consider" }, finish_reason: null }],
+            })}`,
+            `data: ${JSON.stringify({
+                id: "response-xai",
+                object: "chat.completion.chunk",
+                created: 2,
+                model: "grok-build-0.1",
+                choices: [{ index: 0, delta: { content: "OK" }, finish_reason: "stop" }],
+            })}`,
+            `data: ${JSON.stringify({
+                id: "response-xai",
+                object: "chat.completion.chunk",
+                created: 3,
+                model: "grok-build-0.1",
+                choices: [],
+                usage: {
+                    prompt_tokens: 5,
+                    completion_tokens: 4,
+                    total_tokens: 9,
+                    prompt_tokens_details: { cached_tokens: 2 },
+                    completion_tokens_details: { reasoning_tokens: 3 },
+                    cost_in_usd_ticks: 1_230_000,
+                },
+            })}`,
+            "data: [DONE]",
+        ].join("\n\n"), { headers: { "content-type": "text/event-stream" } });
+    });
+
+    const provider = catalogProviderFromEnv("xai", {
+        ...env,
+        XAI_API_KEY: "test-key",
+        PLURNK_PROVIDERS_REASONING: "adaptive",
+    }, "grok-build-0.1");
+    const result = await provider?.generate({
+        workerId: "xai-worker",
+        messages: [{ role: "user", content: "hello" }],
+        maxOutputTokens: 16,
+    });
+
+    assert.equal(call?.body.max_completion_tokens, 16);
+    assert.equal("max_tokens" in (call?.body ?? {}), false);
+    assert.equal(call?.headers.get("x-grok-conv-id"), "xai-worker");
+    assert.equal(result?.assistant.reasoning, "consider");
+    assert.equal(result?.assistant.content, "OK");
+    assert.deepEqual(result?.accounting[0]?.usage, {
+        inputTokens: 5,
+        outputTokens: 4,
+        totalTokens: 9,
+        inputTokenDetails: { cacheReadTokens: 2 },
+        outputTokenDetails: { textTokens: 1, reasoningTokens: 3 },
+    });
+    assert.deepEqual(result?.accounting[0]?.cost, {
+        kind: "charged",
+        amount: { amount: "1230000", currency: "USDTICK" },
+        usdEquivalent: "0.000123",
+        source: "xAI response usage.cost_in_usd_ticks",
+    });
+});
+
+test("Cerebras explicit reasoning activation needs no operator effort or token budget", async () => {
+    let body: Record<string, unknown> | undefined;
+    mock.method(globalThis, "fetch", async (_input: string | URL | Request, init?: RequestInit) => {
+        body = JSON.parse(String(init?.body)) as Record<string, unknown>;
+        return new Response([
+            `data: ${JSON.stringify({
+                id: "chatcmpl-cerebras",
+                object: "chat.completion.chunk",
+                created: 1,
+                model: "gemma-4-31b",
+                choices: [{ index: 0, delta: { reasoning: "consider" }, finish_reason: null }],
+            })}`,
+            `data: ${JSON.stringify({
+                id: "chatcmpl-cerebras",
+                object: "chat.completion.chunk",
+                created: 2,
+                model: "gemma-4-31b",
+                choices: [{ index: 0, delta: { content: "done" }, finish_reason: "stop" }],
+            })}`,
+            `data: ${JSON.stringify({
+                id: "chatcmpl-cerebras",
+                object: "chat.completion.chunk",
+                created: 3,
+                model: "gemma-4-31b",
+                choices: [],
+                usage: {
+                    prompt_tokens: 2,
+                    completion_tokens: 2,
+                    total_tokens: 4,
+                    completion_tokens_details: { reasoning_tokens: 1 },
+                },
+            })}`,
+            "data: [DONE]",
+        ].join("\n\n"), { headers: { "content-type": "text/event-stream" } });
+    });
+
+    const provider = catalogProviderFromEnv("cerebras", {
+        ...env,
+        CEREBRAS_API_KEY: "test-key",
+        PLURNK_PROVIDERS_REASONING: "on",
+    }, "gemma-4-31b");
+    const result = await provider?.generate({
+        workerId: "worker",
+        messages: [{ role: "user", content: "hello" }],
+    });
+
+    assert.equal(body?.reasoning_effort, "medium", "the native SDK projects unqualified on to its enabled posture");
+    assert.equal("thinking_budget_tokens" in (body ?? {}), false, "activation does not invent a token budget");
+    assert.equal(result?.assistant.reasoning, "consider");
+    assert.equal(result?.accounting[0]?.usage?.outputTokenDetails?.reasoningTokens, 1);
+});
+
+test("Google adaptive reasoning requests and preserves readable thought summaries", async () => {
+    const bodies: Array<{
+        generationConfig?: { thinkingConfig?: { includeThoughts?: boolean } };
+    }> = [];
+    mock.method(globalThis, "fetch", async (_input: string | URL | Request, init?: RequestInit) => {
+        bodies.push(JSON.parse(String(init?.body)) as typeof bodies[number]);
+        return new Response(`data: ${JSON.stringify({
+            responseId: "response-gemini",
+            candidates: [{
+                content: {
+                    role: "model",
+                    parts: [
+                        { text: "consider", thought: true },
+                        { text: "done" },
+                    ],
+                },
+                finishReason: "STOP",
+            }],
+            usageMetadata: {
+                promptTokenCount: 2,
+                candidatesTokenCount: 1,
+                thoughtsTokenCount: 1,
+                totalTokenCount: 4,
+            },
+        })}\n\n`, {
+            headers: { "content-type": "text/event-stream" },
+        });
+    });
+
+    const provider = catalogProviderFromEnv("google", {
+        ...env,
+        GEMINI_API_KEY: "test-key",
+        PLURNK_PROVIDERS_REASONING: "adaptive",
+    }, "gemini-3.7-flash");
+    const result = await provider?.generate({
+        workerId: "worker",
+        messages: [{ role: "user", content: "hello" }],
+    });
+
+    assert.deepEqual(bodies[0]?.generationConfig?.thinkingConfig, {
+        includeThoughts: true,
+    }, "adaptive leaves thinking depth to Google while requesting its readable summary");
+    assert.equal(result?.assistant.reasoning, "consider");
+    assert.equal(result?.assistant.content, "done");
+    assert.equal(result?.accounting[0]?.usage?.outputTokenDetails?.reasoningTokens, 1);
+
+    const disabled = catalogProviderFromEnv("google", {
+        ...env,
+        GEMINI_API_KEY: "test-key",
+        PLURNK_PROVIDERS_REASONING: "off",
+    }, "gemini-3.7-flash");
+    await disabled?.generate({
+        workerId: "worker",
+        messages: [{ role: "user", content: "hello" }],
+    });
+    assert.equal(
+        bodies[1]?.generationConfig?.thinkingConfig?.includeThoughts,
+        undefined,
+        "off does not request readable thoughts",
+    );
+});
+
+test("native provider routes project their documented cache controls through the actual SDK request", async (t) => {
+    const calls: Array<{ url: string; headers: Headers; body: Record<string, unknown> }> = [];
+    mock.method(globalThis, "fetch", async (input: string | URL | Request, init?: RequestInit) => {
+        calls.push({
+            url: String(input),
+            headers: new Headers(init?.headers),
+            body: JSON.parse(String(init?.body)) as Record<string, unknown>,
+        });
+        return new Response([
+            `data: ${JSON.stringify({
+                id: "response",
+                object: "chat.completion.chunk",
+                created: 1,
+                model: "served",
+                choices: [{ index: 0, delta: { content: "ok" }, finish_reason: "stop" }],
+            })}`,
+            `data: ${JSON.stringify({
+                id: "response",
+                object: "chat.completion.chunk",
+                created: 2,
+                model: "served",
+                choices: [],
+                usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 },
+            })}`,
+            "data: [DONE]",
+        ].join("\n\n"), { headers: { "content-type": "text/event-stream" } });
+    });
+
+    await t.test("DeepInfra native options become prompt_cache_key", async () => {
+        const provider = catalogProviderFromEnv("deepinfra", {
+            ...env,
+            DEEPINFRA_API_KEY: "test-key",
+        }, "zai-org/GLM-5.2");
+        await provider?.generate({
+            workerId: "deepinfra-worker",
+            messages: [{ role: "user", content: "hello" }],
+        });
+        assert.equal(calls.at(-1)?.body.prompt_cache_key, "deepinfra-worker");
+    });
+
+    await t.test("OpenRouter carries session affinity and an Anthropic system breakpoint", async () => {
+        let call: { headers: Headers; body: Record<string, unknown> } | undefined;
+        const server = createServer(async (request, response) => {
+            const chunks: Buffer[] = [];
+            for await (const chunk of request) chunks.push(Buffer.from(chunk));
+            call = {
+                headers: new Headers(request.headers as Record<string, string>),
+                body: JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record<string, unknown>,
+            };
+            response.writeHead(200, { "content-type": "text/event-stream" });
+            response.end([
+                `data: ${JSON.stringify({
+                    id: "response",
+                    object: "chat.completion.chunk",
+                    created: 1,
+                    model: "served",
+                    choices: [{ index: 0, delta: { content: "ok" }, finish_reason: "stop" }],
+                })}`,
+                `data: ${JSON.stringify({
+                    id: "response",
+                    object: "chat.completion.chunk",
+                    created: 2,
+                    model: "served",
+                    choices: [],
+                    usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 },
+                })}`,
+                "data: [DONE]",
+            ].join("\n\n"));
+        });
+        server.listen(0, "127.0.0.1");
+        await once(server, "listening");
+        t.after(() => new Promise<void>((resolve, reject) => {
+            server.close((error) => error === undefined ? resolve() : reject(error));
+        }));
+        const address = server.address();
+        if (address === null || typeof address === "string") {
+            throw new Error("OpenRouter request-capture server did not bind a TCP address");
+        }
+
+        const provider = catalogProviderFromEnv("openrouter", {
+            ...env,
+            OPENROUTER_API_KEY: "test-key",
+        }, "anthropic/claude-sonnet-4.6", `http://127.0.0.1:${address.port}/api/v1`);
+        await provider?.generate({
+            workerId: "openrouter-worker",
+            messages: [
+                { role: "system", content: "stable system packet" },
+                { role: "user", content: "changing user packet" },
+            ],
+        });
+        assert.equal(call?.headers.get("x-session-id"), "openrouter-worker");
+        assert.deepEqual((call?.body.messages as unknown[] | undefined)?.[0], {
+            role: "system",
+            content: [{
+                type: "text",
+                text: "stable system packet",
+                cache_control: { type: "ephemeral" },
+            }],
+        });
+        assert.deepEqual((call?.body.messages as unknown[] | undefined)?.[1], {
+            role: "user",
+            content: "changing user packet",
+        });
+    });
 });
 
 test("cataloged unknown model fails unless its context is explicit", () => {
@@ -120,22 +415,35 @@ test("cataloged unknown model fails unless its context is explicit", () => {
     assert.equal(provider?.contextWindow, 8192);
 });
 
-test("Models.dev is the only fallback rate table", () => {
-    const usage = {
-        prompt: 1_000,
-        cached: 400,
-        completion: 100,
-        reasoning: 50,
-        total: 1_150,
-    };
+test("Models.dev is the only fallback rate table", async () => {
+    mock.method(globalThis, "fetch", async () => new Response([
+        `data: ${JSON.stringify({
+            id: "response",
+            model: "served",
+            choices: [{ index: 0, delta: { content: "ok" }, finish_reason: "stop" }],
+        })}`,
+        `data: ${JSON.stringify({
+            id: "response",
+            model: "served",
+            choices: [],
+            usage: {
+                prompt_tokens: 1_000,
+                prompt_tokens_details: { cached_tokens: 400 },
+                completion_tokens: 100,
+                total_tokens: 1_150,
+            },
+        })}`,
+        "data: [DONE]",
+    ].join("\n\n"), { headers: { "content-type": "text/event-stream" } }));
     const cataloged = catalogProviderFromEnv("deepseek", {
         ...env,
         DEEPSEEK_API_KEY: "test-key",
     }, "deepseek-v4-flash");
     assert.notEqual(cataloged, null);
-    assert.deepEqual(providerCostFor(cataloged!, usage), {
+    const catalogedResponse = await cataloged!.generate({ workerId: "cataloged", messages: [] });
+    assert.deepEqual(catalogedResponse.accounting[0]?.cost, {
         kind: "estimated",
-        usd: "0.00012712",
+        amount: { amount: "0.00012712", currency: "USD" },
         source: "Models.dev catalog rates",
     });
 
@@ -144,8 +452,9 @@ test("Models.dev is the only fallback rate table", () => {
         XAI_API_KEY: "test-key",
         PLURNK_PROVIDERS_CONTEXT_WINDOW: "8192",
     }, "not-in-the-catalog");
-    assert.deepEqual(providerCostFor(uncataloged!, usage), {
+    const uncatalogedResponse = await uncataloged!.generate({ workerId: "uncataloged", messages: [] });
+    assert.deepEqual(uncatalogedResponse.accounting[0]?.cost, {
         kind: "unknown",
-        reason: "the response reported no cost and Models.dev has no rate for this model",
+        reason: "Models.dev has no complete rate for this model",
     });
 });