decision-api 1.3.0 → 2.0.0

package/README.md

@@ -18,7 +18,8 @@ const decide = createDecisionApi({
 	},
 	defaultModel: 'jev',
 	registerUsage: (usage, context: { customerId: string }) => {
-		// usage: model, provider, inputTokens, outputTokens, costUsd, latencyMs
+		// usage: model, resolvedModel, provider, inputTokens, outputTokens,
+		// costUsd, latencyMs, attempts
 	},
 	onError: (error, context) => console.warn(error.kind, error.message)
 })
@@ -49,7 +50,50 @@ TypeScript enforces the setup:
 - **The model is optional only with a default.** Without `defaultModel`, every call must name one.
 - **Usage context is required when typed.** The type of `registerUsage`'s second parameter is required on every call and passed back to `registerUsage` and `onError`.
 
-A call answers `null` instead of throwing, and `onError` says why: `timeout`, `aborted` (the call's own signal), `unauthorized` (401/403, a bad key), `out-of-credits` (402), `rate-limited` (429 after the retry), `request-failed` (any other status, or a transport error), `state-too-large`, `images-unsupported`, `image-detached` (see [Images](#images)), `unreadable-response` or `answer-mismatch`. Every error carries its `model` and `provider`, and `status` when the provider answered; a throwing `registerUsage` reports `usage-failed` without failing the call. Retryable statuses (429 and 5xx) are retried once, after the provider's `Retry-After` when it asks for 2 s or less, and not at all when it asks for more. `fetch` replaces the transport, for example to route calls through your own rate limits, and `defaultSignal` supplies the signal for calls that pass none. A call with no questions answers `{}` without a request. The one throw is a call with no model and no `defaultModel`, which TypeScript already rejects.
+A call answers `null` instead of throwing, and `onError` says why: `timeout`, `aborted` (the call's own signal: your own doing, so a logging `onError` may skip it), `unauthorized` (401/403, a bad key), `out-of-credits` (402), `rate-limited` (429 after the retry), `request-failed` (any other status, or a transport error), `request-too-large` (see below), `images-unsupported`, `image-detached` (see [Images](#images)), `unreadable-response` or `answer-mismatch`. Every error carries its `model` and `provider`, `status` when the provider answered, `latencyMs` and `attempts`; a throwing `registerUsage` reports `usage-failed` without failing the call. `fetch` replaces the transport, for example to route calls through your own rate limits, and `defaultSignal` supplies the signal for calls that pass none. A call with no questions answers `{}` without a request. The one throw is a call with no model and no `defaultModel`, which TypeScript already rejects.
+
+**Timeouts and retries.** `timeoutMs` (default 4 s, per client or per call) bounds the whole call: every attempt, the pause between them and the body read. A status of 429, 500, 502, 503, 504 or 529 and a transport error are retried once, after the provider's `Retry-After` when it asks for 2 s or less and the deadline leaves time for it; otherwise the call fails at once. `attemptTimeoutMs` (off by default) abandons a request that has not answered by then and sends one more while time remains, so a voice turn can set `timeoutMs: 1_200, attemptTimeoutMs: 500` and still get a second try. An abandoned request may still be billed.
+
+**Size.** The state and questions together are measured before sending: past `maxRequestChars` (default `DECISION_MAX_REQUEST_CHARS`, 90k characters, under Jev's ~32.5k-token wall) the call fails with `request-too-large` and nothing is sent. So does a call over its model's `maxQuestions`, `maxChoices` (options in one choice) or `maxLevels` (levels in one score) in `DECISION_MODELS`, and a provider's own token-limit refusal maps to the same kind. Extra questions add next to nothing to latency, but every question's instructions and criteria are billed as input tokens.
+
+## Usage and errors per call
+
+`decide.result()` takes the same arguments as `decide()` and answers what happened, so a caller can store the model, tokens and latency next to its own record, or the reason there is no answer, without a side channel through the usage context:
+
+```ts
+const outcome = await decide.result({ state, questions, customerId: 'c_1' })
+if (outcome.ok) {
+	outcome.answers // as decide() answers them
+	outcome.usage // DecisionUsage; null for a call with no questions
+} else {
+	outcome.error // DecisionError: kind, status, latencyMs, attempts
+}
+```
+
+`registerUsage` and `onError` still run. `usage.resolvedModel` is the model the provider says answered, when it says: TypeSafe resolves `jev` (sent as `jev-latest`) to a release there, so a new Jev release shows up in your rows.
+
+## One question per item
+
+`perItem` asks the same question about each item of a list under keys `prefix_0`, `prefix_1`… and reads the answers back typed, with no key helpers and no `undefined` to default:
+
+```ts
+import { perItem } from 'decision-api'
+
+const perBlock = perItem(blocks, 'block', (block, index) => ({
+	type: 'noul',
+	instructions: `Does \`blocks[${index}]\` hold private material?`
+}))
+const answers = await decide({
+	state: { blocks },
+	questions: { ...perBlock.questions, rules },
+	customerId: 'c_1'
+})
+for (const { item, answer } of answers ? perBlock.read(answers) : []) {
+	if (answer.noul >= 0.35) flagged.push(item.id)
+}
+```
+
+The single answer types are exported too: `DecisionNoulAnswer`, `DecisionChoiceAnswer`, `DecisionScoreAnswer` and their union `DecisionAnswer`.
 
 ## Images
 
@@ -86,11 +130,43 @@ Each image is sent once, base64-encoded under a unique reference, however often
 - **A plain template loses the image.** `` `Same as ${badge}?` `` writes only the reference; the call answers `null` with `image-detached` instead of asking about a bare string.
 - **Prepare images first.** `image()` throws for anything but PNG or JPEG. Jev rejects images, and never resizes them, outside 280 to 1568 px a side or over 16 MiB, and ignores EXIF orientation, so scale and rotate before sending ([limits](https://docs.typesafe.ai/images#limits)). TypeSafe calls images a preview: the format and accuracy may change between Jev releases.
 
+## Testing
+
+`decision-api/testing` has a fake client typed as the real one, so a test needs no cast. A number answers a noul or a score, a string a choice, or give the full answer. An answer that does not fit its question, or a missing one, throws in the test, since the real client would never return it:
+
+```ts
+import { fakeDecide } from 'decision-api/testing'
+
+const ask = fakeDecide<typeof decide>(() => ({ agreed: 0.9, team: 'billing' }))
+const down = fakeDecide<typeof decide>(() => null) // fails as request-failed
+ask.calls[0]?.questions // what the code under test sent
+```
+
+## Evaluating thresholds
+
+`decision-api/eval` runs labelled cases against a real client and reports each question at the threshold your code ships. The app brings the cases; a case is a call plus the answers a careful human would give:
+
+```ts
+import { report, runCases } from 'decision-api/eval'
+
+const logs = await runCases({
+	decide,
+	cases: [{ name: 'salary slip', state, questions, expected: { private: true } }],
+	runs: 3,
+	models: ['jev'], // the client's default when left out
+	context: { customerId: 'eval' } // when registerUsage types a context
+})
+const summary = report(logs, { thresholds: { private: 0.35 } })
+if (!summary.ok) process.exitCode = 1
+```
+
+Per model it reports failed calls, latency p50/p90, input tokens and cost, and per question its accuracy at the threshold (0.5 when unset; a noul is positive at or above it). For nouls it adds misses and false positives there and at every threshold from 0.05 to 0.95, the margin between the lowest positive and the highest negative (below zero, no threshold separates them), the Brier score and a 10-bin reliability table. `summary.ok` is false when any call failed or any answer was wrong.
+
 ## Benchmark
 
-`bun run bench` asks every model with a key the fixed questions in [`bench/cases.ts`](bench/cases.ts) and writes `.artifacts/bench/report.html`: accuracy against the expected answers, probability on the right answer, latency p50/p90 and cost per model, every answer side by side, and a closing leaderboard that scores each model out of 100 (one point per right answer, a failed call scores zero). Keys come from the environment or `.env`: copy [`.env.example`](.env.example) to `.env` and fill in the providers you have; `--runs=5` and `--models=jev,clef` narrow or widen a run.
+`bun run bench` asks every model with a key the fixed questions in [`bench/cases.ts`](bench/cases.ts) and writes `.artifacts/bench/report.html`: accuracy against the expected answers, probability on the right answer, latency p50/p90 and cost per model, every answer side by side, and a closing leaderboard; the console also prints each model's Brier score on the nouls that scores each model out of 100 (one point per right answer, a failed call scores zero). Keys come from the environment or `.env`: copy [`.env.example`](.env.example) to `.env` and fill in the providers you have; `--runs=5` and `--models=jev,clef` narrow or widen a run.
 
-Results from 2026-10-10 (12 cases, 21 questions, 3 runs per case):
+Results from 2026-10-10 (12 cases, 21 questions, 3 runs per case; `typesafe/jev-1.13` was added after this run):
 
 | # | Model | Score | Points | p(correct) | Latency p50 / p90 | Cost / 1k calls |
 | --- | --- | --- | --- | --- | --- | --- |

package/dist/create-decision-api.d.ts

@@ -3,29 +3,47 @@ import { type DecisionImageModel, type DecisionModel, type DecisionProvider, typ
 import { type DecisionAnswers, type DecisionQuestion } from './questions.ts';
 /**
  * Jev answers `400 max_tokens_exceeded` past roughly 32.5k input tokens (96k
- * chars passed, 128k did not). A state over this is refused before it leaves
- * the process; keep well under the wall, since tokenization varies. Clef takes
- * 64k tokens but silently truncates long state, so the same guard keeps it
- * from judging half a corpus.
+ * chars of state passed, 128k did not). Every question's instructions and
+ * criteria count too, so a request whose state and questions together are over
+ * this is refused before it leaves the process; keep well under the wall,
+ * since tokenization varies. Clef takes 64k tokens but silently truncates long
+ * state, so the same guard keeps it from judging half a corpus.
  */
-export declare const DECISION_MAX_STATE_CHARS = 90000;
+export declare const DECISION_MAX_REQUEST_CHARS = 90000;
 /** What one answered call cost; `costUsd` is priced from `DECISION_MODELS`. */
 export type DecisionUsage<Model extends DecisionModel = DecisionModel> = {
     model: Model;
+    /**
+     * The model the provider says answered, when it says: an alias such as
+     * `jev` (sent as `jev-latest`) resolves to a release here.
+     */
+    resolvedModel?: string;
     provider: DecisionProvider;
     inputTokens: number;
     outputTokens: number;
     costUsd: number;
+    /** From the first request to the answer, retries included. */
     latencyMs: number;
+    /** Requests sent: 2 when the first was retried. */
+    attempts: number;
 };
-export type DecisionErrorKind = 'state-too-large'
+export type DecisionErrorKind = 
+/**
+ * The state and questions together are over `maxRequestChars`, the call is
+ * over the model's question, option or level limit (`DECISION_MODELS`), or
+ * the provider refused it as too many tokens.
+ */
+'request-too-large'
 /** The call holds an image and the model reads none (`images: false`). */
  | 'images-unsupported'
 /** An image's reference is in the text without the image: build that text with `withImages`. */
  | 'image-detached'
-/** No answer within `timeoutMs`. */
+/** No answer within `timeoutMs`, retries included. */
  | 'timeout'
-/** The call's own signal (or `defaultSignal`) aborted it. */
+/**
+ * The call's own signal (or `defaultSignal`) aborted it: the caller's own
+ * doing, which an `onError` that logs failures may want to skip.
+ */
  | 'aborted'
 /** 401 or 403: the provider rejected the key. Retrying will not help. */
  | 'unauthorized'
@@ -41,6 +59,10 @@ export declare class DecisionError extends Error {
     /** The HTTP status, when the provider answered. */
     readonly status?: number | undefined;
     readonly provider: DecisionProvider;
+    /** From the start of the call to this failure. Set by the client. */
+    latencyMs: number;
+    /** Requests sent before this failure: 0 when it failed before sending. */
+    attempts: number;
     constructor(kind: DecisionErrorKind, model: DecisionModel, message: string, 
     /** The HTTP status, when the provider answered. */
     status?: number | undefined);
@@ -64,7 +86,10 @@ type DecisionCall<Q extends Record<string, DecisionQuestion>, State> = {
     state: State;
     questions: Q;
     signal?: AbortSignal;
+    /** The whole call, retries and pauses included. */
     timeoutMs?: number;
+    /** One request; see `attemptTimeoutMs` on `createDecisionApi`. */
+    attemptTimeoutMs?: number;
 };
 type DecisionApiOptions<Keys extends DecisionProviderKeys, Default, Context> = {
     keys: Keys;
@@ -80,15 +105,34 @@ type DecisionApiOptions<Keys extends DecisionProviderKeys, Default, Context> = {
     /** The signal of a call that passes none. */
     defaultSignal?: () => AbortSignal | undefined;
     fetch?: DecisionFetch;
+    /** The whole call, retries and pauses included. Defaults to 4 s. */
     timeoutMs?: number;
-    maxStateChars?: number;
+    /**
+     * Abandon a request that has not answered by then and send one more, while
+     * `timeoutMs` leaves time. Off by default; the abandoned request may still
+     * be billed.
+     */
+    attemptTimeoutMs?: number;
+    /** State plus questions, in characters. Defaults to `DECISION_MAX_REQUEST_CHARS`. */
+    maxRequestChars?: number;
+};
+/** What `decide.result()` answers: the answers and their usage, or why there are none. */
+export type DecisionResult<Q extends Record<string, DecisionQuestion>, Model extends DecisionModel = DecisionModel> = {
+    ok: true;
+    answers: DecisionAnswers<Q>;
+    /** `null` only for a call with no questions, which sends nothing. */
+    usage: DecisionUsage<Model> | null;
+} | {
+    ok: false;
+    error: DecisionError;
 };
 /**
  * Ask one or more independent judgments about the same state.
  *
  * **Send every independent question in ONE call**: questions are evaluated
- * together, so extra ones are close to free, and latency tracks the size of
- * `state`, not the number of questions.
+ * together, so extra ones add next to nothing to latency, which tracks the
+ * size of `state`. They are not free: every question's instructions and
+ * criteria are billed as input tokens.
  *
  * **Answers `null` rather than throwing** - on a timeout, an abort, a
  * transport error, a rejected request or an answer that does not match its
@@ -96,8 +140,15 @@ type DecisionApiOptions<Keys extends DecisionProviderKeys, Default, Context> = {
  * tagging job treats `null` as "no tags this time", a policy guard as "refuse".
  * It throws only when there is no model at all - a setup bug TypeScript
  * already rejects, which a silent `null` would hide.
+ *
+ * `decide.result()` takes the same arguments and answers the usage with the
+ * answers, or the error instead of `null`.
  */
-export type DecisionApi<Model extends DecisionModel, Default, Context> = <Q extends Record<string, DecisionQuestion>, State = unknown>(input: DecisionCall<Q, State> & Context, ...model: ModelArgument<HasImage<State> extends true ? Extract<Model, DecisionImageModel> : HasImage<Q> extends true ? Extract<Model, DecisionImageModel> : Model, Default>) => Promise<DecisionAnswers<Q> | null>;
+export type DecisionApi<Model extends DecisionModel, Default, Context> = {
+    <Q extends Record<string, DecisionQuestion>, State = unknown>(input: DecisionCall<Q, State> & Context, ...model: DecisionModelArgument<Model, Default, Q, State>): Promise<DecisionAnswers<Q> | null>;
+    result<Q extends Record<string, DecisionQuestion>, State = unknown>(input: DecisionCall<Q, State> & Context, ...model: DecisionModelArgument<Model, Default, Q, State>): Promise<DecisionResult<Q, Model>>;
+};
+type DecisionModelArgument<Model extends DecisionModel, Default, Q, State> = ModelArgument<HasImage<State> extends true ? Extract<Model, DecisionImageModel> : HasImage<Q> extends true ? Extract<Model, DecisionImageModel> : Model, Default>;
 type ModelArgument<Model, Default> = [Default] extends [Model] ? [model?: Model] : [model: Model];
 /**
  * Whether a call's state or questions hold an image, as far as their types

package/dist/create-decision-api.js

@@ -1,21 +1,22 @@
 import { collectImages, detachedReference } from "./images.js";
-import { buildDecisionRequest, DECISION_MODELS, readDecisionResponse } from "./models.js";
+import { buildDecisionRequest, DECISION_MODELS, overModelLimit, readDecisionResponse } from "./models.js";
 import { matchesQuestion } from "./questions.js";
 /**
  * Jev answers `400 max_tokens_exceeded` past roughly 32.5k input tokens (96k
- * chars passed, 128k did not). A state over this is refused before it leaves
- * the process; keep well under the wall, since tokenization varies. Clef takes
- * 64k tokens but silently truncates long state, so the same guard keeps it
- * from judging half a corpus.
+ * chars of state passed, 128k did not). Every question's instructions and
+ * criteria count too, so a request whose state and questions together are over
+ * this is refused before it leaves the process; keep well under the wall,
+ * since tokenization varies. Clef takes 64k tokens but silently truncates long
+ * state, so the same guard keeps it from judging half a corpus.
  */
-export const DECISION_MAX_STATE_CHARS = 90_000;
+export const DECISION_MAX_REQUEST_CHARS = 90_000;
 /**
  * A generous ceiling rather than a target (Jev's p90 is ~280 ms for ten
  * questions): past it the caller is better served by its own fallback.
  */
 const DEFAULT_TIMEOUT_MS = 4_000;
 /** Retried once: the providers' own guidance for these is to back off and retry. */
-const RETRYABLE_STATUSES = new Set([429, 500, 502, 503, 529]);
+const RETRYABLE_STATUSES = new Set([429, 500, 502, 503, 504, 529]);
 const RETRY_DELAY_MS = 250;
 /**
  * A provider asking for a longer pause (`Retry-After`) will not be ready within
@@ -27,6 +28,10 @@ export class DecisionError extends Error {
     model;
     status;
     provider;
+    /** From the start of the call to this failure. Set by the client. */
+    latencyMs = 0;
+    /** Requests sent before this failure: 0 when it failed before sending. */
+    attempts = 0;
     constructor(kind, model, message, 
     /** The HTTP status, when the provider answered. */
     status) {
@@ -38,7 +43,6 @@ export class DecisionError extends Error {
         this.provider = DECISION_MODELS[model].provider;
     }
 }
-/** The kind a refused response maps to; anything unlisted is `request-failed`. */
 const KIND_BY_STATUS = {
     401: 'unauthorized',
     402: 'out-of-credits',
@@ -47,29 +51,35 @@ const KIND_BY_STATUS = {
 };
 export function createDecisionApi(options) {
     const send = options.fetch ?? ((request) => fetch(request.url, request.init));
-    const maxStateChars = options.maxStateChars ?? DECISION_MAX_STATE_CHARS;
-    return async (input, ...[requestedModel]) => {
-        const { state, questions, signal, timeoutMs, ...rest } = input;
+    const maxRequestChars = options.maxRequestChars ?? DECISION_MAX_REQUEST_CHARS;
+    const result = async (input, requestedModel) => {
+        const calledAt = Date.now();
+        const { state, questions, signal, timeoutMs, attemptTimeoutMs, ...rest } = input;
         const context = rest;
         const model = requestedModel ?? options.defaultModel;
         if (!model)
             throw new Error('No model given and no defaultModel set');
         const report = (error) => {
+            error.latencyMs = Date.now() - calledAt;
             options.onError?.(error, context);
-            return null;
+            return { ok: false, error };
         };
         const fail = (kind, message) => report(new DecisionError(kind, model, message));
         if (Object.keys(questions).length === 0) {
-            return {};
+            return { ok: true, answers: {}, usage: null };
         }
         // An image counts as its reference here: its tokens are the provider's to count.
         const stateText = typeof state === 'string' ? state : JSON.stringify(state);
-        const stateChars = stateText.length;
-        if (stateChars > maxStateChars) {
-            return fail('state-too-large', `State is ${stateChars} chars, over ${maxStateChars}`);
+        const questionsText = JSON.stringify(questions);
+        const requestChars = stateText.length + questionsText.length;
+        if (requestChars > maxRequestChars) {
+            return fail('request-too-large', `State and questions are ${requestChars} chars, over ${maxRequestChars}`);
         }
+        const overLimit = overModelLimit(model, questions);
+        if (overLimit)
+            return fail('request-too-large', overLimit);
         const images = collectImages(questions, collectImages(state));
-        const detached = detachedReference(stateText + JSON.stringify(questions), images);
+        const detached = detachedReference(stateText + questionsText, images);
         if (detached) {
             return fail('image-detached', `${detached} is in the text without its image; write that text with withImages`);
         }
@@ -77,14 +87,15 @@ export function createDecisionApi(options) {
             return fail('images-unsupported', `${model} does not read images`);
         }
         const startedAt = Date.now();
-        let text;
+        let sent;
         try {
-            text = await postWithRetry({
+            sent = await postWithRetry({
                 send,
                 model,
                 request: buildDecisionRequest(model, { state, questions, images: [...images.values()] }, options.keys),
                 signal: signal ?? options.defaultSignal?.(),
-                timeoutMs: timeoutMs ?? options.timeoutMs ?? DEFAULT_TIMEOUT_MS
+                timeoutMs: timeoutMs ?? options.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+                attemptTimeoutMs: attemptTimeoutMs ?? options.attemptTimeoutMs
             });
         }
         catch (error) {
@@ -92,39 +103,58 @@ export function createDecisionApi(options) {
                 return report(error);
             return fail('request-failed', messageOf(error));
         }
+        const failAfterSend = (kind, message) => {
+            const error = new DecisionError(kind, model, message);
+            error.attempts = sent.attempts;
+            return report(error);
+        };
         let json;
         try {
-            json = JSON.parse(text);
+            json = JSON.parse(sent.text);
         }
         catch (error) {
-            return fail('unreadable-response', messageOf(error));
+            return failAfterSend('unreadable-response', messageOf(error));
         }
         const parsed = readDecisionResponse(model, json);
         if (!parsed.success) {
-            return fail('unreadable-response', parsed.error.issues[0]?.message ?? 'unknown');
+            return failAfterSend('unreadable-response', parsed.error.issues[0]?.message ?? 'unknown');
         }
-        const { usage, answers } = parsed.data;
-        reportUsage({
+        const { usage: tokens, answers, model: resolvedModel } = parsed.data;
+        const usage = {
             model,
+            ...(resolvedModel !== undefined && { resolvedModel }),
             provider: DECISION_MODELS[model].provider,
-            inputTokens: usage.input_tokens,
-            outputTokens: usage.output_tokens,
-            costUsd: (usage.input_tokens * DECISION_MODELS[model].inputUsdPerMTok) /
+            inputTokens: tokens.input_tokens,
+            outputTokens: tokens.output_tokens,
+            costUsd: (tokens.input_tokens * DECISION_MODELS[model].inputUsdPerMTok) /
                 1_000_000,
-            latencyMs: Date.now() - startedAt
-        }, context);
+            latencyMs: Date.now() - startedAt,
+            attempts: sent.attempts
+        };
+        reportUsage(usage, context);
         for (const [key, question] of Object.entries(questions)) {
             if (!matchesQuestion(question, answers[key])) {
-                return fail('answer-mismatch', `Answer to "${key}" does not fit it`);
+                return failAfterSend('answer-mismatch', `Answer to "${key}" does not fit it`);
             }
         }
         // Every requested key, kind and option set was validated above.
-        return answers;
+        return {
+            ok: true,
+            answers: answers,
+            usage
+        };
     };
+    const decide = async (input, model) => {
+        const outcome = await result(input, model);
+        return outcome.ok ? outcome.answers : null;
+    };
+    // The call signatures above are checked per call; one implementation serves them all.
+    return Object.assign(decide, { result });
     /**
-     * Only a call that reached the provider is reported - a state over the
+     * Only a call that reached the provider is reported - a request over the
      * wall, a timeout or an unreadable body reports nothing, so a 0 ms row can
-     * never flatter a latency percentile. A failing hook never fails the call.
+     * never flatter a latency percentile (`onError` carries `latencyMs` for
+     * those). A failing hook never fails the call.
      */
     function reportUsage(usage, context) {
         const { registerUsage } = options;
@@ -141,16 +171,34 @@ export function createDecisionApi(options) {
         });
     }
 }
-/** The body of the first OK response, read under the same timeout as the request. */
+/**
+ * The body of the first OK response and how many requests it took. One
+ * deadline (`timeoutMs`) bounds every attempt, the pause between them and the
+ * body read. A retryable status, a transport error or an abandoned attempt
+ * (`attemptTimeoutMs`) is retried once, when the deadline leaves time for it.
+ */
 async function postWithRetry(input) {
     const provider = DECISION_MODELS[input.model].provider;
-    const failure = (kind, message) => new DecisionError(kind, input.model, message);
+    const endsAt = Date.now() + input.timeoutMs;
+    let attempts = 0;
+    const failure = (kind, message, status) => {
+        const error = new DecisionError(kind, input.model, message, status);
+        error.attempts = attempts;
+        return error;
+    };
+    const timedOut = () => failure('timeout', `No answer within ${input.timeoutMs} ms`);
     for (let attempt = 0;; attempt++) {
         if (input.signal?.aborted)
             throw failure('aborted', 'Aborted by the caller');
-        // A fresh timeout per attempt, so a retry gets its own budget.
-        const timeout = AbortSignal.timeout(input.timeoutMs);
+        const remainingMs = endsAt - Date.now();
+        if (remainingMs <= 0)
+            throw timedOut();
+        const canRetry = attempt === 0;
+        const attemptMs = Math.min(input.attemptTimeoutMs ?? remainingMs, remainingMs);
+        const timeout = AbortSignal.timeout(attemptMs);
         const { signal, release } = linkSignals(timeout, input.signal);
+        attempts += 1;
+        let delayMs = RETRY_DELAY_MS;
         try {
             const response = await input.send({
                 url: input.request.url,
@@ -163,17 +211,17 @@ async function postWithRetry(input) {
                 }
             }, { model: input.model, provider, attempt, signal });
             if (response.ok)
-                return await response.text();
-            const delayMs = retryDelayMs(response);
-            if (attempt > 0 ||
+                return { text: await response.text(), attempts };
+            delayMs = retryDelayMs(response);
+            if (!canRetry ||
                 !RETRYABLE_STATUSES.has(response.status) ||
-                delayMs > MAX_RETRY_DELAY_MS) {
+                delayMs > MAX_RETRY_DELAY_MS ||
+                delayMs >= endsAt - Date.now()) {
                 const body = (await response.text()).slice(0, 200);
-                throw new DecisionError(KIND_BY_STATUS[response.status] ?? 'request-failed', input.model, `${input.model} ${response.status}: ${body}`, response.status);
+                throw failure(kindOf(response.status, body), `${input.model} ${response.status}: ${body}`, response.status);
             }
             // An unread body holds its connection until garbage collection.
             await response.body?.cancel().catch(() => { });
-            await new Promise((resolve) => setTimeout(resolve, delayMs));
         }
         catch (error) {
             if (error instanceof DecisionError)
@@ -181,14 +229,46 @@ async function postWithRetry(input) {
             if (input.signal?.aborted)
                 throw failure('aborted', messageOf(error));
             if (timeout.aborted) {
-                throw failure('timeout', `No answer within ${input.timeoutMs} ms`);
+                // The deadline itself, or the last attempt: nothing left to try.
+                if (attemptMs >= remainingMs || !canRetry)
+                    throw timedOut();
+                continue;
+            }
+            if (!canRetry || RETRY_DELAY_MS >= endsAt - Date.now()) {
+                throw failure('request-failed', messageOf(error));
             }
-            throw error;
         }
         finally {
             release();
         }
+        await pause(delayMs, input.signal).catch(() => {
+            throw failure('aborted', 'Aborted by the caller');
+        });
+    }
+}
+/** The kind a refused response maps to; anything unlisted is `request-failed`. */
+function kindOf(status, body) {
+    if (status === 413 || (status === 400 && /max_tokens_exceeded/.test(body))) {
+        return 'request-too-large';
     }
+    return KIND_BY_STATUS[status] ?? 'request-failed';
+}
+/** Waits `ms`, or rejects as soon as `signal` aborts. */
+function pause(ms, signal) {
+    return new Promise((resolve, reject) => {
+        if (signal?.aborted)
+            return reject(signal.reason);
+        const timer = setTimeout(done, ms);
+        signal?.addEventListener('abort', done, { once: true });
+        function done() {
+            clearTimeout(timer);
+            signal?.removeEventListener('abort', done);
+            if (signal?.aborted)
+                reject(signal.reason);
+            else
+                resolve();
+        }
+    });
 }
 /** The pause a response asks for, in seconds or as a date; our own without one. */
 function retryDelayMs(response) {

package/dist/eval.d.ts

@@ -0,0 +1,115 @@
+import type { DecisionResult } from './create-decision-api.ts';
+import type { DecisionModel } from './models.ts';
+import type { DecisionAnswer, DecisionQuestion } from './questions.ts';
+/** One call and the answers a careful human would give to its questions. */
+export type DecisionCase = {
+    name: string;
+    state: unknown;
+    questions: Record<string, DecisionQuestion>;
+    /** By question key: a noul's truth, a choice's option, a score's level. */
+    expected: Record<string, boolean | string | number>;
+};
+export type GradedAnswer = {
+    expected: boolean | string | number;
+    value: string;
+    /** At 0.5 for a noul; `report` regrades nouls at your thresholds. */
+    correct: boolean;
+    /** The probability the model put on the expected answer. */
+    pCorrect: number;
+    /** A noul's probability, kept for the threshold sweep and calibration. */
+    noul?: number;
+};
+export type CaseLog = {
+    model: DecisionModel;
+    caseName: string;
+    run: number;
+    latencyMs: number;
+    inputTokens?: number;
+    costUsd?: number;
+    /** `kind: message` of a failed call. */
+    error?: string;
+    answers?: Record<string, GradedAnswer>;
+};
+/** Right or wrong, plus the probability the model put on the right answer. */
+export declare function grade(answer: DecisionAnswer, expected: boolean | string | number): GradedAnswer;
+/** What `runCases` needs of a client: `decide` from `createDecisionApi` fits. */
+type CaseClient = {
+    result(input: {
+        state: unknown;
+        questions: Record<string, DecisionQuestion>;
+    }, model?: DecisionModel): Promise<DecisionResult<Record<string, DecisionQuestion>>>;
+};
+/**
+ * Asks every case `runs` times on each of `models` (the client's default
+ * model when left out). Models run side by side; each model's calls run one
+ * at a time, so they never queue behind each other. `context` is the usage
+ * context the client requires, if it requires one.
+ */
+export declare function runCases(input: {
+    decide: CaseClient;
+    cases: DecisionCase[];
+    runs?: number;
+    models?: DecisionModel[];
+    context?: object;
+    onCall?: (log: CaseLog) => void;
+}): Promise<CaseLog[]>;
+export declare function percentile(values: number[], p: number): number;
+export declare const mean: (values: number[]) => number;
+/**
+ * Per model and question: accuracy at its threshold (`thresholds[key]`, 0.5
+ * when unset; a noul is positive at or above it), and for nouls the misses
+ * and false positives there and at every threshold from 0.05 to 0.95, the
+ * margin between the lowest positive and the highest negative (below zero
+ * means no threshold separates them), the Brier score and a 10-bin
+ * reliability table. `ok` is false when any call failed or any answer was
+ * wrong: `if (!summary.ok) process.exitCode = 1`.
+ */
+export declare function report(logs: CaseLog[], options?: {
+    thresholds?: Record<string, number>;
+}): {
+    ok: boolean;
+    models: {
+        model: "jev" | "clef" | "clef-flash" | "gpt-6-luna" | "microsoft/microsoft-decision-1" | "typesafe/jev-1.13";
+        calls: number;
+        failed: number;
+        wrong: number;
+        brier: number;
+        latencyP50: number;
+        latencyP90: number;
+        inputTokens: number;
+        costUsd: number;
+        questions: ({
+            key: string;
+            answers: number;
+            threshold: undefined;
+            wrong: number;
+            accuracy: number;
+            noul: undefined;
+        } | {
+            key: string;
+            answers: number;
+            threshold: number;
+            wrong: number;
+            accuracy: number;
+            noul: {
+                misses: number;
+                falsePositives: number;
+                margin: number;
+                brier: number;
+                sweep: {
+                    threshold: number;
+                    misses: number;
+                    falsePositives: number;
+                }[];
+                reliability: {
+                    from: number;
+                    to: number;
+                    answers: number;
+                    meanP: number;
+                    observed: number;
+                }[];
+            };
+        })[];
+    }[];
+};
+export {};

package/dist/eval.js

@@ -0,0 +1,182 @@
+// Labelled cases against a live client: run them, then read accuracy at the
+// thresholds the app ships, a threshold sweep and calibration per question.
+// The app brings the cases; nothing here knows its domain.
+/** Right or wrong, plus the probability the model put on the right answer. */
+export function grade(answer, expected) {
+    if (answer.type === 'noul') {
+        const pCorrect = expected ? answer.noul : 1 - answer.noul;
+        return {
+            expected,
+            value: answer.noul.toFixed(2),
+            correct: pCorrect >= 0.5,
+            pCorrect,
+            noul: answer.noul
+        };
+    }
+    const pCorrect = answer.probabilities[String(expected)] ?? 0;
+    if (answer.type === 'choice') {
+        return {
+            expected,
+            value: answer.choice,
+            correct: answer.choice === expected,
+            pCorrect
+        };
+    }
+    return {
+        expected,
+        value: answer.score.toFixed(2),
+        correct: Math.round(answer.score) === expected,
+        pCorrect
+    };
+}
+/**
+ * Asks every case `runs` times on each of `models` (the client's default
+ * model when left out). Models run side by side; each model's calls run one
+ * at a time, so they never queue behind each other. `context` is the usage
+ * context the client requires, if it requires one.
+ */
+export async function runCases(input) {
+    const runs = input.runs ?? 1;
+    const perModel = await Promise.all((input.models ?? [undefined]).map(async (model) => {
+        const logs = [];
+        for (let run = 1; run <= runs; run++) {
+            for (const decisionCase of input.cases) {
+                const outcome = await input.decide.result({
+                    ...input.context,
+                    state: decisionCase.state,
+                    questions: decisionCase.questions
+                }, model);
+                const log = outcome.ok
+                    ? {
+                        model: outcome.usage?.model ?? model ?? 'jev',
+                        caseName: decisionCase.name,
+                        run,
+                        latencyMs: outcome.usage?.latencyMs ?? 0,
+                        inputTokens: outcome.usage?.inputTokens,
+                        costUsd: outcome.usage?.costUsd,
+                        answers: Object.fromEntries(Object.entries(decisionCase.expected).map(([key, expected]) => [
+                            key,
+                            grade(outcome.answers[key], expected)
+                        ]))
+                    }
+                    : {
+                        model: outcome.error.model,
+                        caseName: decisionCase.name,
+                        run,
+                        latencyMs: outcome.error.latencyMs,
+                        error: `${outcome.error.kind}: ${outcome.error.message}`
+                    };
+                input.onCall?.(log);
+                logs.push(log);
+            }
+        }
+        return logs;
+    }));
+    return perModel.flat();
+}
+export function percentile(values, p) {
+    if (values.length === 0)
+        return Number.NaN;
+    const sorted = [...values].sort((a, b) => a - b);
+    return sorted[Math.max(0, Math.ceil(p * sorted.length) - 1)];
+}
+export const mean = (values) => values.length === 0
+    ? Number.NaN
+    : values.reduce((sum, value) => sum + value, 0) / values.length;
+const SWEEP = Array.from({ length: 19 }, (_, index) => (index + 1) / 20);
+/**
+ * Per model and question: accuracy at its threshold (`thresholds[key]`, 0.5
+ * when unset; a noul is positive at or above it), and for nouls the misses
+ * and false positives there and at every threshold from 0.05 to 0.95, the
+ * margin between the lowest positive and the highest negative (below zero
+ * means no threshold separates them), the Brier score and a 10-bin
+ * reliability table. `ok` is false when any call failed or any answer was
+ * wrong: `if (!summary.ok) process.exitCode = 1`.
+ */
+export function report(logs, options = {}) {
+    const models = [...new Set(logs.map((log) => log.model))];
+    const perModel = models.map((model) => {
+        const own = logs.filter((log) => log.model === model);
+        const answered = own.filter((log) => log.answers);
+        const keys = [
+            ...new Set(answered.flatMap((log) => Object.keys(log.answers ?? {})))
+        ];
+        const questions = keys.map((key) => summarizeQuestion(key, answered.flatMap((log) => log.answers?.[key] ?? []), options.thresholds?.[key] ?? 0.5));
+        const nouls = answered.flatMap((log) => Object.values(log.answers ?? {}).filter((a) => a.noul !== undefined));
+        const latencies = answered.map((log) => log.latencyMs);
+        return {
+            model,
+            calls: own.length,
+            failed: own.length - answered.length,
+            wrong: questions.reduce((sum, question) => sum + question.wrong, 0),
+            brier: brier(nouls),
+            latencyP50: percentile(latencies, 0.5),
+            latencyP90: percentile(latencies, 0.9),
+            inputTokens: sum(answered.map((log) => log.inputTokens ?? 0)),
+            costUsd: sum(answered.map((log) => log.costUsd ?? 0)),
+            questions
+        };
+    });
+    return {
+        ok: perModel.every((row) => row.failed === 0 && row.wrong === 0),
+        models: perModel
+    };
+}
+function summarizeQuestion(key, graded, threshold) {
+    const nouls = graded.filter((answer) => answer.noul !== undefined);
+    if (nouls.length === 0 || nouls.length !== graded.length) {
+        const wrong = graded.filter((answer) => !answer.correct).length;
+        return {
+            key,
+            answers: graded.length,
+            threshold: undefined,
+            wrong,
+            accuracy: 1 - wrong / graded.length,
+            noul: undefined
+        };
+    }
+    const errorsAt = (at) => ({
+        threshold: at,
+        misses: nouls.filter((a) => a.expected && a.noul < at).length,
+        falsePositives: nouls.filter((a) => !a.expected && a.noul >= at).length
+    });
+    const at = errorsAt(threshold);
+    const positives = nouls.filter((a) => a.expected).map((a) => a.noul);
+    const negatives = nouls.filter((a) => !a.expected).map((a) => a.noul);
+    const wrong = at.misses + at.falsePositives;
+    return {
+        key,
+        answers: graded.length,
+        threshold,
+        wrong,
+        accuracy: 1 - wrong / graded.length,
+        noul: {
+            misses: at.misses,
+            falsePositives: at.falsePositives,
+            margin: positives.length && negatives.length
+                ? Math.min(...positives) - Math.max(...negatives)
+                : Number.NaN,
+            brier: brier(nouls),
+            sweep: SWEEP.map(errorsAt),
+            reliability: reliability(nouls)
+        }
+    };
+}
+const sum = (values) => values.reduce((total, v) => total + v, 0);
+/** Mean squared distance from the truth: 0 is perfect, 0.25 is a coin. */
+function brier(nouls) {
+    return mean(nouls.map((a) => ((a.noul ?? 0) - (a.expected ? 1 : 0)) ** 2));
+}
+/** Answers binned by probability: a calibrated model's `observed` tracks `meanP`. */
+function reliability(nouls) {
+    return Array.from({ length: 10 }, (_, bin) => {
+        const inBin = nouls.filter((a) => Math.min(9, Math.floor(a.noul * 10)) === bin);
+        return {
+            from: bin / 10,
+            to: (bin + 1) / 10,
+            answers: inBin.length,
+            meanP: mean(inBin.map((a) => a.noul)),
+            observed: mean(inBin.map((a) => (a.expected ? 1 : 0)))
+        };
+    }).filter((row) => row.answers > 0);
+}

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-export { createDecisionApi, DECISION_MAX_STATE_CHARS, type DecisionApi, DecisionError, type DecisionErrorKind, type DecisionFetch, type DecisionUsage } from './create-decision-api.ts';
+export { createDecisionApi, DECISION_MAX_REQUEST_CHARS, type DecisionApi, DecisionError, type DecisionErrorKind, type DecisionFetch, type DecisionResult, type DecisionUsage } from './create-decision-api.ts';
 export { type DecisionImage, type DecisionImageFormat, type DecisionImageText, type DecisionText, image, withImages } from './images.ts';
 export { DECISION_MODELS, type DecisionImageModel, type DecisionModel, type DecisionProvider, type DecisionProviderKeys, type ModelsFor } from './models.ts';
-export type { DecisionAnswers, DecisionChoice, DecisionNoul, DecisionQuestion, DecisionScore } from './questions.ts';
+export { type DecisionAnswer, type DecisionAnswers, type DecisionChoice, type DecisionChoiceAnswer, type DecisionNoul, type DecisionNoulAnswer, type DecisionQuestion, type DecisionScore, type DecisionScoreAnswer, perItem } from './questions.ts';

package/dist/index.js

@@ -1,3 +1,4 @@
-export { createDecisionApi, DECISION_MAX_STATE_CHARS, DecisionError } from "./create-decision-api.js";
+export { createDecisionApi, DECISION_MAX_REQUEST_CHARS, DecisionError } from "./create-decision-api.js";
 export { image, withImages } from "./images.js";
 export { DECISION_MODELS } from "./models.js";
+export { perItem } from "./questions.js";

package/dist/models.d.ts

@@ -24,17 +24,26 @@ export type DecisionProvider = keyof DecisionProviderKeys;
  * anywhere in the state, instructions and criteria (1.13 and later:
  * https://docs.typesafe.ai/images). Clef and gpt-6-luna read images too, but
  * in wire formats this client does not speak yet.
+ *
+ * `maxQuestions`, `maxChoices` (options in one choice) and `maxLevels` (levels
+ * in one score) are the limits a call is checked against before it is sent,
+ * where a provider documents or was measured to enforce one. Jev's come from
+ * its API schema (https://docs.typesafe.ai/api); Clef's question limit was
+ * measured (it refuses more than 64).
  */
 export declare const DECISION_MODELS: {
     readonly jev: {
         readonly provider: "typesafe";
         readonly inputUsdPerMTok: 0.042;
         readonly images: true;
+        readonly maxChoices: 255;
+        readonly maxLevels: 10;
     };
     readonly clef: {
         readonly provider: "cloudflare";
         readonly inputUsdPerMTok: 0.24;
         readonly images: false;
+        readonly maxQuestions: 64;
     };
     readonly 'clef-flash': {
         readonly provider: "cloudflare";
@@ -55,6 +64,8 @@ export declare const DECISION_MODELS: {
         readonly provider: "openrouter";
         readonly inputUsdPerMTok: 0.042;
         readonly images: true;
+        readonly maxChoices: 255;
+        readonly maxLevels: 10;
     };
 };
 export type DecisionModel = keyof typeof DECISION_MODELS;
@@ -69,6 +80,8 @@ type ModelsOf<Provider> = {
 export type ModelsFor<Keys extends DecisionProviderKeys> = ModelsOf<{
     [Provider in keyof Keys]-?: undefined extends Keys[Provider] ? never : Provider;
 }[keyof Keys]>;
+/** Why a model would refuse these questions, or undefined when it takes them. */
+export declare function overModelLimit(model: DecisionModel, questions: Record<string, DecisionQuestion>): string | undefined;
 type RequestInput = {
     state: unknown;
     questions: Record<string, DecisionQuestion>;
@@ -104,5 +117,6 @@ export declare function readDecisionResponse(model: DecisionModel, json: unknown
         input_tokens: number;
         output_tokens: number;
     };
+    model?: string | undefined;
 }>;
 export {};

package/dist/models.js

@@ -15,10 +15,27 @@ import { DecisionResponseSchema } from "./questions.js";
  * anywhere in the state, instructions and criteria (1.13 and later:
  * https://docs.typesafe.ai/images). Clef and gpt-6-luna read images too, but
  * in wire formats this client does not speak yet.
+ *
+ * `maxQuestions`, `maxChoices` (options in one choice) and `maxLevels` (levels
+ * in one score) are the limits a call is checked against before it is sent,
+ * where a provider documents or was measured to enforce one. Jev's come from
+ * its API schema (https://docs.typesafe.ai/api); Clef's question limit was
+ * measured (it refuses more than 64).
  */
 export const DECISION_MODELS = {
-    jev: { provider: 'typesafe', inputUsdPerMTok: 0.042, images: true },
-    clef: { provider: 'cloudflare', inputUsdPerMTok: 0.24, images: false },
+    jev: {
+        provider: 'typesafe',
+        inputUsdPerMTok: 0.042,
+        images: true,
+        maxChoices: 255,
+        maxLevels: 10
+    },
+    clef: {
+        provider: 'cloudflare',
+        inputUsdPerMTok: 0.24,
+        images: false,
+        maxQuestions: 64
+    },
     'clef-flash': {
         provider: 'cloudflare',
         inputUsdPerMTok: 0.09,
@@ -38,9 +55,30 @@ export const DECISION_MODELS = {
     'typesafe/jev-1.13': {
         provider: 'openrouter',
         inputUsdPerMTok: 0.042,
-        images: true
+        images: true,
+        maxChoices: 255,
+        maxLevels: 10
     }
 };
+/** Why a model would refuse these questions, or undefined when it takes them. */
+export function overModelLimit(model, questions) {
+    const limits = DECISION_MODELS[model];
+    const count = Object.keys(questions).length;
+    if (limits.maxQuestions !== undefined && count > limits.maxQuestions) {
+        return `${count} questions, over ${model}'s ${limits.maxQuestions}`;
+    }
+    for (const [key, question] of Object.entries(questions)) {
+        if (question.type === 'noul')
+            continue;
+        const [size, limit, what] = question.type === 'choice'
+            ? [Object.keys(question.criteria).length, limits.maxChoices, 'options']
+            : [question.criteria.length, limits.maxLevels, 'levels'];
+        if (limit !== undefined && size > limit) {
+            return `"${key}" has ${size} ${what}, over ${model}'s ${limit}`;
+        }
+    }
+    return undefined;
+}
 /** One POST: where it goes, and the headers and body it carries. */
 export function buildDecisionRequest(model, input, keys) {
     const provider = DECISION_MODELS[model].provider;
@@ -144,6 +182,7 @@ const OpenAiProbabilitiesSchema = z.array(z.object({
     probability: z.number()
 }));
 const OpenAiResponseSchema = z.object({
+    model: z.string().optional(),
     answers: z.array(z.discriminatedUnion('type', [
         z.object({
             type: z.literal('predicate'),
@@ -189,6 +228,7 @@ function fromOpenAiResponse(response) {
         ];
     });
     return {
+        model: response.model,
         answers: Object.fromEntries(answers),
         usage: {
             input_tokens: response.usage.input_tokens,

package/dist/questions.d.ts

@@ -25,24 +25,25 @@ export type DecisionScore = {
     criteria: DecisionText[];
 };
 export type DecisionQuestion = DecisionNoul | DecisionChoice | DecisionScore;
-type DecisionNoulAnswer = {
+export type DecisionNoulAnswer = {
     type: 'noul';
     noul: number;
 };
-type DecisionChoiceAnswer<Option extends string = string> = {
+export type DecisionChoiceAnswer<Option extends string = string> = {
     type: 'choice';
     choice: Option;
     confidence: number;
     probabilities: Record<Option, number>;
 };
-type DecisionScoreAnswer = {
+export type DecisionScoreAnswer = {
     type: 'score';
     score: number;
     confidence: number;
     probabilities: Record<string, number>;
 };
+export type DecisionAnswer = DecisionNoulAnswer | DecisionChoiceAnswer | DecisionScoreAnswer;
 /** The answer a given question shape produces, so call sites need no casts. */
-type AnswerFor<Q> = Q extends {
+export type AnswerFor<Q> = Q extends {
     type: 'noul';
 } ? DecisionNoulAnswer : Q extends {
     type: 'choice';
@@ -53,22 +54,9 @@ type AnswerFor<Q> = Q extends {
 export type DecisionAnswers<Q extends Record<string, DecisionQuestion>> = {
     [K in keyof Q]: AnswerFor<Q[K]>;
 };
-declare const AnswerSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
-    type: z.ZodLiteral<"noul">;
-    noul: z.ZodNumber;
-}, z.core.$strip>, z.ZodObject<{
-    type: z.ZodLiteral<"choice">;
-    choice: z.ZodString;
-    confidence: z.ZodNumber;
-    probabilities: z.ZodRecord<z.ZodString, z.ZodNumber>;
-}, z.core.$strip>, z.ZodObject<{
-    type: z.ZodLiteral<"score">;
-    score: z.ZodNumber;
-    confidence: z.ZodNumber;
-    probabilities: z.ZodRecord<z.ZodString, z.ZodNumber>;
-}, z.core.$strip>], "type">;
 /** What every vendor's response is read into. */
 export declare const DecisionResponseSchema: z.ZodObject<{
+    model: z.ZodOptional<z.ZodString>;
     answers: z.ZodRecord<z.ZodString, z.ZodDiscriminatedUnion<[z.ZodObject<{
         type: z.ZodLiteral<"noul">;
         noul: z.ZodNumber;
@@ -90,5 +78,18 @@ export declare const DecisionResponseSchema: z.ZodObject<{
 }, z.core.$strip>;
 export type DecisionResponse = z.infer<typeof DecisionResponseSchema>;
 /** The contract depends on the question, not just the answer's own tag. */
-export declare function matchesQuestion(question: DecisionQuestion, answer: z.infer<typeof AnswerSchema> | undefined): boolean;
-export {};
+export declare function matchesQuestion(question: DecisionQuestion, answer: DecisionAnswer | undefined): boolean;
+/**
+ * The same question about each item of a list, under keys `${prefix}_${index}`.
+ * Spread `questions` into a call (next to other questions if you like) and
+ * pass its answers to `read`, which pairs each item with its typed answer.
+ */
+export declare function perItem<Item, const Q extends DecisionQuestion, const Prefix extends string>(items: readonly Item[], prefix: Prefix, ask: (item: Item, index: number) => Q): {
+    questions: Record<`${Prefix}_${number}`, Q>;
+    /** Every key is there: a call answers all its questions or `null`. */
+    read: (answers: { readonly [K in `${Prefix}_${number}`]: AnswerFor<Q>; }) => {
+        item: Item;
+        index: number;
+        answer: AnswerFor<Q>;
+    }[];
+};

package/dist/questions.js

@@ -21,6 +21,8 @@ const AnswerSchema = z.discriminatedUnion('type', [
 ]);
 /** What every vendor's response is read into. */
 export const DecisionResponseSchema = z.object({
+    /** The model that answered, when the provider says (it may resolve an alias). */
+    model: z.string().optional(),
     answers: z.record(z.string(), AnswerSchema),
     usage: z.object({
         input_tokens: z.number().int().nonnegative(),
@@ -50,3 +52,20 @@ function matchesOptions(options, probabilities) {
     return (Object.keys(probabilities).length === options.length &&
         options.every((option) => Object.hasOwn(probabilities, option)));
 }
+/**
+ * The same question about each item of a list, under keys `${prefix}_${index}`.
+ * Spread `questions` into a call (next to other questions if you like) and
+ * pass its answers to `read`, which pairs each item with its typed answer.
+ */
+export function perItem(items, prefix, ask) {
+    const questions = Object.fromEntries(items.map((item, index) => [`${prefix}_${index}`, ask(item, index)]));
+    return {
+        questions,
+        /** Every key is there: a call answers all its questions or `null`. */
+        read: (answers) => items.map((item, index) => ({
+            item,
+            index,
+            answer: answers[`${prefix}_${index}`]
+        }))
+    };
+}

package/dist/testing.d.ts

@@ -0,0 +1,27 @@
+import { DecisionError } from './create-decision-api.ts';
+import { type DecisionModel } from './models.ts';
+import { type DecisionAnswer, type DecisionQuestion } from './questions.ts';
+/** One call the code under test made: its input (context included) and model. */
+export type FakeDecisionCall = {
+    state: unknown;
+    questions: Record<string, DecisionQuestion>;
+    model: DecisionModel | undefined;
+    [field: string]: unknown;
+};
+/**
+ * Answers by question key: a number is a noul's probability or a score, a
+ * string is the choice, or give the full answer.
+ */
+export type FakeAnswers = Record<string, number | string | DecisionAnswer>;
+type Respond = (call: FakeDecisionCall) => FakeAnswers | DecisionError | null | Promise<FakeAnswers | DecisionError | null>;
+/**
+ * A fake `decide`, typed as the real one: `fakeDecide<typeof decide>(...)`.
+ * `respond` answers each call; `null` fails it as `request-failed`, and a
+ * `DecisionError` fails it with that error. An answer that does not fit its
+ * question, or a missing one, throws in the test, since the real client would
+ * never return it. `calls` records every call.
+ */
+export declare function fakeDecide<Api>(respond: Respond): Api & {
+    calls: FakeDecisionCall[];
+};
+export {};

package/dist/testing.js

@@ -0,0 +1,74 @@
+// A stand-in for `createDecisionApi`'s client in tests: no network, typed as
+// the real one, and as strict about answers as the real one.
+import { DecisionError } from "./create-decision-api.js";
+import { DECISION_MODELS } from "./models.js";
+import { matchesQuestion } from "./questions.js";
+/**
+ * A fake `decide`, typed as the real one: `fakeDecide<typeof decide>(...)`.
+ * `respond` answers each call; `null` fails it as `request-failed`, and a
+ * `DecisionError` fails it with that error. An answer that does not fit its
+ * question, or a missing one, throws in the test, since the real client would
+ * never return it. `calls` records every call.
+ */
+export function fakeDecide(respond) {
+    const calls = [];
+    const result = async (input, model) => {
+        const call = { ...input, model };
+        calls.push(call);
+        const used = model ?? 'jev';
+        const response = await respond(call);
+        if (response === null || response instanceof DecisionError) {
+            const error = response ?? new DecisionError('request-failed', used, 'fakeDecide');
+            return { ok: false, error };
+        }
+        const usage = {
+            model: used,
+            provider: DECISION_MODELS[used].provider,
+            inputTokens: 0,
+            outputTokens: 0,
+            costUsd: 0,
+            latencyMs: 0,
+            attempts: 1
+        };
+        return {
+            ok: true,
+            answers: expand(call.questions, response),
+            usage
+        };
+    };
+    const decide = async (input, model) => {
+        const outcome = await result(input, model);
+        return outcome.ok ? outcome.answers : null;
+    };
+    return Object.assign(decide, { result, calls });
+}
+function expand(questions, given) {
+    return Object.fromEntries(Object.entries(questions).map(([key, question]) => {
+        const answer = toAnswer(question, given[key]);
+        if (!matchesQuestion(question, answer)) {
+            throw new Error(`fakeDecide: the answer to "${key}" (${JSON.stringify(given[key])}) does not fit its ${question.type} question`);
+        }
+        return [key, answer];
+    }));
+}
+/** Shorthand to a full answer: all the probability on the given value. */
+function toAnswer(question, value) {
+    if (value === undefined || typeof value === 'object')
+        return value;
+    if (question.type === 'noul') {
+        return typeof value === 'number' ? { type: 'noul', noul: value } : undefined;
+    }
+    const options = question.type === 'choice'
+        ? Object.keys(question.criteria)
+        : question.criteria.map((_, index) => String(index));
+    const picked = String(question.type === 'score' ? Math.round(Number(value)) : value);
+    const probabilities = Object.fromEntries(options.map((option) => [option, option === picked ? 1 : 0]));
+    if (question.type === 'choice') {
+        return typeof value === 'string'
+            ? { type: 'choice', choice: value, confidence: 1, probabilities }
+            : undefined;
+    }
+    return typeof value === 'number'
+        ? { type: 'score', score: value, confidence: 1, probabilities }
+        : undefined;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "decision-api",
-	"version": "1.3.0",
+	"version": "2.0.0",
 	"description": "One typed client for decision models (TypeSafe Jev, Cloudflare Clef, OpenAI gpt-6-luna, OpenRouter models such as Microsoft-Decision-1) that answer questions with calibrated probabilities",
 	"license": "MIT",
 	"author": "Kristoffer Hansen",
@@ -22,6 +22,14 @@
 		".": {
 			"types": "./dist/index.d.ts",
 			"default": "./dist/index.js"
+		},
+		"./eval": {
+			"types": "./dist/eval.d.ts",
+			"default": "./dist/eval.js"
+		},
+		"./testing": {
+			"types": "./dist/testing.d.ts",
+			"default": "./dist/testing.js"
 		}
 	},
 	"files": [