@plurnk/plurnk-providers 1.14.2 → 1.16.0

package/src/inputModalities.test.ts

@@ -0,0 +1,53 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import Mock from "./Mock.ts";
+import Pool from "./Pool.ts";
+import { inputModalitiesOf } from "./catalogProvider.ts";
+import { estimatePromptTokens } from "./promptTokens.ts";
+import { chatMessageText, type ChatMessage, type PromptTokenMeasurement } from "./types.ts";
+
+const tokensOf = (measurement: PromptTokenMeasurement): number => "tokens" in measurement ? measurement.tokens : -1;
+
+const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47]);
+const PDF = new Uint8Array([0x25, 0x50, 0x44, 0x46]);
+const parts: ChatMessage = {
+    role: "user",
+    content: [
+        { type: "text", text: "look at this" },
+        { type: "image", image: PNG, mediaType: "image/png" },
+        { type: "file", data: PDF, mediaType: "application/pdf" },
+    ],
+};
+
+test("{§provider-input-modalities} message text is the text parts alone; image and file parts count nothing here", () => {
+    assert.equal(chatMessageText(parts), "look at this");
+    assert.equal(chatMessageText({ content: "plain" }), "plain");
+    assert.equal(tokensOf(estimatePromptTokens([parts])), Math.ceil("look at this".length / 2));
+});
+
+test("{§provider-input-modalities} the catalog's modalities are kept to the wire vocabulary; unknown models declare none", () => {
+    assert.deepEqual([...inputModalitiesOf(["text", "image", "video", "pdf"])], ["image", "video", "pdf"]);
+    assert.deepEqual([...inputModalitiesOf(["text", "audio", "hologram"])], ["audio"]);
+    assert.deepEqual([...inputModalitiesOf(undefined)], []);
+});
+
+test("{§provider-input-modalities} the Mock declares modalities by option and records what it received", async () => {
+    const blind = new Mock({ contextWindow: 200000, responses: [] });
+    assert.deepEqual([...blind.inputModalities], []);
+    const seeing = new Mock({ contextWindow: 200000, responses: [], inputModalities: ["image", "pdf"] });
+    assert.ok(seeing.inputModalities.has("image") && seeing.inputModalities.has("pdf") && !seeing.inputModalities.has("audio"));
+    assert.equal(tokensOf(await seeing.countPromptTokens([parts])), Math.ceil("look at this".length / 2));
+    await assert.rejects(seeing.generate({ messages: [parts], maxOutputTokens: 16, workerId: "1", primaryWorkerId: "1", callKind: "bare" } as never));
+    assert.equal(seeing.received.length, 1, "the request was recorded before the empty queue refused it");
+    const received = seeing.received[0]?.[0]?.content;
+    assert.ok(Array.isArray(received) && received[1]?.type === "image" && received[2]?.type === "file" && received[2].mediaType === "application/pdf");
+});
+
+test("{§provider-input-modalities} a pool accepts a modality only when every backend does", () => {
+    const both = new Mock({ contextWindow: 200000, responses: [], inputModalities: ["image", "pdf"] });
+    const imageOnly = new Mock({ contextWindow: 200000, responses: [], inputModalities: ["image"] });
+    const blind = new Mock({ contextWindow: 200000, responses: [] });
+    assert.deepEqual([...new Pool([both, both]).inputModalities], ["image", "pdf"]);
+    assert.deepEqual([...new Pool([both, imageOnly]).inputModalities], ["image"]);
+    assert.deepEqual([...new Pool([both, blind]).inputModalities], []);
+});

package/src/promptTokens.ts

@@ -1,3 +1,4 @@
+import { chatMessageText } from "./types.ts";
 import type { ChatMessage, PromptTokenMeasurement } from "./types.ts";
 
 const KINDS = new Set<PromptTokenMeasurement["kind"]>([
@@ -38,7 +39,7 @@ export const estimatePromptTokens = (
     detail = "chars/2 over message content; provider request framing is unknown",
 ): PromptTokenMeasurement => ({
     kind: "estimate",
-    tokens: Math.ceil(messages.reduce((sum, { content }) => sum + content.length, 0) / 2),
+    tokens: Math.ceil(messages.reduce((sum, message) => sum + chatMessageText(message).length, 0) / 2),
     source: "heuristic:chars2",
     detail,
 });

package/src/reasoning-effort.ts

@@ -0,0 +1,15 @@
+// A fixed reasoning effort from a reasoning policy, and its native SDK projection.
+import type { ReasoningPolicy } from "./types.ts";
+
+export const fixedEffort = (mode: ReasoningPolicy): "low" | "medium" | "high" | "xhigh" | "max" => {
+    if (mode === "low" || mode === "medium" || mode === "high" || mode === "xhigh" || mode === "max") return mode;
+    throw new TypeError(`reasoning policy '${mode}' is not a fixed effort`);
+};
+
+// The native SDK effort surface tops at xhigh; admission never grants a native
+// route "max", so reaching it here is a contract violation, not a fallback site.
+export const nativeFixedEffort = (mode: ReasoningPolicy): "low" | "medium" | "high" | "xhigh" => {
+    const effort = fixedEffort(mode);
+    if (effort === "max") throw new TypeError(`reasoning policy 'max' has no native SDK effort surface`);
+    return effort;
+};

package/src/types.ts

@@ -47,10 +47,25 @@ export class UnsupportedReasoningPolicyError extends Error {
     }
 }
 
+// {§provider-input-modalities} — the native non-text inputs a route's model accepts, in the
+// catalog's vocabulary (Models.dev `modalities.input` minus `text`).
+export type InputModality = "image" | "pdf" | "audio" | "video";
+export const INPUT_MODALITIES: readonly InputModality[] = Object.freeze(["image", "pdf", "audio", "video"]);
+// A user message may carry parts: text beside native images and files for a model that declares
+// the matching modality. System and assistant messages stay text.
+export type ChatContentPart =
+    | { readonly type: "text"; readonly text: string }
+    | { readonly type: "image"; readonly image: Uint8Array; readonly mediaType: string }
+    | { readonly type: "file"; readonly data: Uint8Array; readonly mediaType: string };
 export interface ChatMessage {
     role: "system" | "user" | "assistant";
-    content: string;
+    content: string | readonly ChatContentPart[];
 }
+// The text of a message, for counting and for transports that take only text.
+export const chatMessageText = (message: Pick<ChatMessage, "content">): string =>
+    typeof message.content === "string"
+        ? message.content
+        : message.content.map((part) => part.type === "text" ? part.text : "").join("");
 
 // {§provider-call-kind} The caller-owned semantic contract for one logical model call. Providers do
 // not infer this from messages or grammar presence: an emission expects PLURNK
@@ -299,6 +314,8 @@ export interface Provider {
     readonly reasoningBudget: number | null;
     // Exact durable policies this adapter can represent without coercion.
     readonly supportedReasoningPolicies: readonly ReasoningPolicy[];
+    // {§provider-input-modalities} — the native non-text parts the route's model accepts.
+    readonly inputModalities: ReadonlySet<InputModality>;
     readonly inputCapacity: number | null;
     readonly model: string;
     // Optional: the backend's self-reported served model id, from a