decision-api 0.0.0 → 1.2.0

package/README.md

@@ -1,6 +1,6 @@
 # decision-api
 
-One client for decision models: models that answer typed questions with calibrated probabilities instead of text. It speaks TypeSafe's Jev, Cloudflare's Clef and Clef-flash (Workers AI) and OpenAI's gpt-6-luna, and returns the same answer shape from all of them. It depends only on `zod`.
+One client for decision models: models that answer typed questions with calibrated probabilities instead of text. It speaks TypeSafe's Jev, Cloudflare's Clef and Clef-flash (Workers AI), OpenAI's gpt-6-luna and the decision models on OpenRouter (such as `microsoft/microsoft-decision-1` and `typesafe/jev-1.13`), and returns the same answer shape from all of them. It depends only on `zod`.
 
 ```sh
 npm install decision-api
@@ -13,7 +13,8 @@ const decide = createDecisionApi({
 	keys: {
 		typesafe: '…',
 		cloudflare: { apiToken: '…', accountId: '…' },
-		openai: '…'
+		openai: '…',
+		openrouter: '…' // every OpenRouter model, named by its slug
 	},
 	defaultModel: 'jev',
 	registerUsage: (usage, context: { customerId: string }) => {
@@ -44,11 +45,25 @@ answers?.team.choice // 'billing' | 'other'
 
 TypeScript enforces the setup:
 
-- **Keys select models.** Only models whose provider key you passed can be named, as `defaultModel` or per call. A key typed `string | undefined` (a raw `process.env` read) does not count; narrow it first.
+- **Keys select models.** Only models whose provider key you passed can be named, as `defaultModel` or per call. A key typed `string | undefined` (a raw `process.env` read) does not count; narrow it first. One `openrouter` key opens every OpenRouter model at once; a new one is added as its slug and price in `DECISION_MODELS`.
 - **The model is optional only with a default.** Without `defaultModel`, every call must name one.
 - **Usage context is required when typed.** The type of `registerUsage`'s second parameter is required on every call and passed back to `registerUsage` and `onError`.
 
-A call answers `null` instead of throwing, and `onError` says why: `timeout`, `aborted` (the call's own signal), `request-failed`, `state-too-large`, `unreadable-response` or `answer-mismatch`; a throwing `registerUsage` reports `usage-failed` without failing the call. Retryable statuses (429 and 5xx) are retried once, after the provider's `Retry-After` when it asks for 2 s or less, and not at all when it asks for more. `fetch` replaces the transport, for example to route calls through your own rate limits, and `defaultSignal` supplies the signal for calls that pass none. A call with no questions answers `{}` without a request. The one throw is a call with no model and no `defaultModel`, which TypeScript already rejects.
+A call answers `null` instead of throwing, and `onError` says why: `timeout`, `aborted` (the call's own signal), `unauthorized` (401/403, a bad key), `out-of-credits` (402), `rate-limited` (429 after the retry), `request-failed` (any other status, or a transport error), `state-too-large`, `unreadable-response` or `answer-mismatch`. Every error carries its `model` and `provider`, and `status` when the provider answered; a throwing `registerUsage` reports `usage-failed` without failing the call. Retryable statuses (429 and 5xx) are retried once, after the provider's `Retry-After` when it asks for 2 s or less, and not at all when it asks for more. `fetch` replaces the transport, for example to route calls through your own rate limits, and `defaultSignal` supplies the signal for calls that pass none. A call with no questions answers `{}` without a request. The one throw is a call with no model and no `defaultModel`, which TypeScript already rejects.
+
+## Benchmark
+
+`bun run bench` asks every model with a key the fixed questions in [`bench/cases.ts`](bench/cases.ts) and writes `.artifacts/bench/report.html`: accuracy against the expected answers, probability on the right answer, latency p50/p90 and cost per model, every answer side by side, and a closing leaderboard that scores each model out of 100 (one point per right answer, a failed call scores zero). Keys come from the environment or `.env`: copy [`.env.example`](.env.example) to `.env` and fill in the providers you have; `--runs=5` and `--models=jev,clef` narrow or widen a run.
+
+Results from 2026-10-10 (12 cases, 21 questions, 3 runs per case):
+
+| # | Model | Score | Points | p(correct) | Latency p50 / p90 | Cost / 1k calls |
+| --- | --- | --- | --- | --- | --- | --- |
+| 1 | `jev` | 100 | 21 / 21 | 96% | 262 / 292 ms | $0.017 |
+| 2 | `clef` | 100 | 21 / 21 | 94% | 292 / 550 ms | $0.067 |
+| 3 | `microsoft/microsoft-decision-1` | 95.2 | 20 / 21 | 95% | 354 / 819 ms | $0.005 |
+| 4 | `gpt-6-luna` | 95.2 | 20 / 21 | 94% | 149 / 264 ms | $0.029 |
+| 5 | `clef-flash` | 95.2 | 20 / 21 | 85% | 227 / 662 ms | $0.025 |
 
 ## Releasing
 

package/dist/create-decision-api.d.ts

@@ -21,11 +21,24 @@ export type DecisionErrorKind = 'state-too-large'
 /** No answer within `timeoutMs`. */
  | 'timeout'
 /** The call's own signal (or `defaultSignal`) aborted it. */
- | 'aborted' | 'request-failed' | 'unreadable-response' | 'answer-mismatch' | 'usage-failed';
+ | 'aborted'
+/** 401 or 403: the provider rejected the key. Retrying will not help. */
+ | 'unauthorized'
+/** 402: the provider's account is out of credits. Retrying will not help. */
+ | 'out-of-credits'
+/** 429 that outlasted the retry, or asked to wait longer than a call can. */
+ | 'rate-limited'
+/** Any other refused request (see `status`), or a transport error (no `status`). */
+ | 'request-failed' | 'unreadable-response' | 'answer-mismatch' | 'usage-failed';
 export declare class DecisionError extends Error {
     readonly kind: DecisionErrorKind;
     readonly model: DecisionModel;
-    constructor(kind: DecisionErrorKind, model: DecisionModel, message: string);
+    /** The HTTP status, when the provider answered. */
+    readonly status?: number | undefined;
+    readonly provider: DecisionProvider;
+    constructor(kind: DecisionErrorKind, model: DecisionModel, message: string, 
+    /** The HTTP status, when the provider answered. */
+    status?: number | undefined);
 }
 /** Sends one request. Replace it to route calls through your own budget or proxy. */
 export type DecisionFetch = (request: {

package/dist/create-decision-api.js

@@ -24,13 +24,26 @@ const MAX_RETRY_DELAY_MS = 2_000;
 export class DecisionError extends Error {
     kind;
     model;
-    constructor(kind, model, message) {
+    status;
+    provider;
+    constructor(kind, model, message, 
+    /** The HTTP status, when the provider answered. */
+    status) {
         super(message);
         this.kind = kind;
         this.model = model;
+        this.status = status;
         this.name = 'DecisionError';
+        this.provider = DECISION_MODELS[model].provider;
     }
 }
+/** The kind a refused response maps to; anything unlisted is `request-failed`. */
+const KIND_BY_STATUS = {
+    401: 'unauthorized',
+    402: 'out-of-credits',
+    403: 'unauthorized',
+    429: 'rate-limited'
+};
 export function createDecisionApi(options) {
     const send = options.fetch ?? ((request) => fetch(request.url, request.init));
     const maxStateChars = options.maxStateChars ?? DECISION_MAX_STATE_CHARS;
@@ -40,10 +53,11 @@ export function createDecisionApi(options) {
         const model = requestedModel ?? options.defaultModel;
         if (!model)
             throw new Error('No model given and no defaultModel set');
-        const fail = (kind, message) => {
-            options.onError?.(new DecisionError(kind, model, message), context);
+        const report = (error) => {
+            options.onError?.(error, context);
             return null;
         };
+        const fail = (kind, message) => report(new DecisionError(kind, model, message));
         if (Object.keys(questions).length === 0) {
             return {};
         }
@@ -65,7 +79,7 @@ export function createDecisionApi(options) {
         }
         catch (error) {
             if (error instanceof DecisionError)
-                return fail(error.kind, error.message);
+                return report(error);
             return fail('request-failed', messageOf(error));
         }
         let json;
@@ -145,7 +159,7 @@ async function postWithRetry(input) {
                 !RETRYABLE_STATUSES.has(response.status) ||
                 delayMs > MAX_RETRY_DELAY_MS) {
                 const body = (await response.text()).slice(0, 200);
-                throw failure('request-failed', `${input.model} ${response.status}: ${body}`);
+                throw new DecisionError(KIND_BY_STATUS[response.status] ?? 'request-failed', input.model, `${input.model} ${response.status}: ${body}`, response.status);
             }
             // An unread body holds its connection until garbage collection.
             await response.body?.cancel().catch(() => { });

package/dist/models.d.ts

@@ -8,6 +8,8 @@ export type DecisionProviderKeys = {
         accountId: string;
     };
     openai?: string;
+    /** One key for every model OpenRouter hosts (the `vendor/model` names). */
+    openrouter?: string;
 };
 export type DecisionProvider = keyof DecisionProviderKeys;
 /**
@@ -33,6 +35,14 @@ export declare const DECISION_MODELS: {
         readonly provider: "openai";
         readonly inputUsdPerMTok: 0.1;
     };
+    readonly 'microsoft/microsoft-decision-1': {
+        readonly provider: "openrouter";
+        readonly inputUsdPerMTok: 0.042;
+    };
+    readonly 'typesafe/jev-1.13': {
+        readonly provider: "openrouter";
+        readonly inputUsdPerMTok: 0.042;
+    };
 };
 export type DecisionModel = keyof typeof DECISION_MODELS;
 type ModelsOf<Provider> = {

package/dist/models.js

@@ -13,7 +13,15 @@ export const DECISION_MODELS = {
     jev: { provider: 'typesafe', inputUsdPerMTok: 0.042 },
     clef: { provider: 'cloudflare', inputUsdPerMTok: 0.24 },
     'clef-flash': { provider: 'cloudflare', inputUsdPerMTok: 0.09 },
-    'gpt-6-luna': { provider: 'openai', inputUsdPerMTok: 0.1 }
+    'gpt-6-luna': { provider: 'openai', inputUsdPerMTok: 0.1 },
+    // OpenRouter models are named by their OpenRouter slug, so adding one is
+    // one line here: https://openrouter.ai/rankings/decisions
+    'microsoft/microsoft-decision-1': {
+        provider: 'openrouter',
+        inputUsdPerMTok: 0.042
+    },
+    // TypeSafe's Jev 1.13 on OpenRouter, served on OpenRouter's zero-data-retention list
+    'typesafe/jev-1.13': { provider: 'openrouter', inputUsdPerMTok: 0.042 }
 };
 /** One POST: where it goes, and the headers and body it carries. */
 export function buildDecisionRequest(model, input, keys) {
@@ -40,6 +48,14 @@ export function buildDecisionRequest(model, input, keys) {
             body: JSON.stringify({ ...input, model })
         };
     }
+    if (provider === 'openrouter') {
+        // https://openrouter.ai/docs/api/api-reference/alphadecisions/submit-a-decisions-request
+        return {
+            url: 'https://openrouter.ai/api/alpha/decisions',
+            headers: headers(keys.openrouter),
+            body: JSON.stringify({ ...input, model })
+        };
+    }
     return {
         url: 'https://api.openai.com/v1/decisions',
         headers: headers(keys.openai),
@@ -152,6 +168,8 @@ function fromOpenAiResponse(response) {
 }
 const responseSchemas = {
     typesafe: DecisionResponseSchema,
+    // OpenRouter's Decisions API speaks System One's shape for every model.
+    openrouter: DecisionResponseSchema,
     // Workers AI wraps every model's output in its own envelope.
     cloudflare: z
         .object({ result: DecisionResponseSchema })

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "decision-api",
-	"version": "0.0.0",
-	"description": "One typed client for decision models (TypeSafe Jev, Cloudflare Clef, OpenAI gpt-6-luna) that answer questions with calibrated probabilities",
+	"version": "1.2.0",
+	"description": "One typed client for decision models (TypeSafe Jev, Cloudflare Clef, OpenAI gpt-6-luna, OpenRouter models such as Microsoft-Decision-1) that answer questions with calibrated probabilities",
 	"license": "MIT",
 	"author": "Kristoffer Hansen",
 	"repository": {
@@ -36,7 +36,8 @@
 		"zod",
 		"openai",
 		"cloudflare",
-		"typesafe"
+		"typesafe",
+		"openrouter"
 	],
 	"scripts": {
 		"build": "rm -rf dist && tsc -p tsconfig.build.json",
@@ -46,6 +47,7 @@
 		"lint:write": "bun run typecheck && biome check --write",
 		"prepublishOnly": "bun run build",
 		"verify:package": "node scripts/verify-package.mjs",
+		"bench": "bun bench/run.ts",
 		"prepare": "husky"
 	},
 	"dependencies": {