nexus-ai-pro 2.20.2 → 2.22.0

package/dist/decisions/core.d.ts

@@ -1,13 +1,18 @@
 /**
- * Typed decisions: questions a model answers with a yes or no, a choice, or a score, each with its
- * probabilities and a confidence read from them, so software acts on an answer without parsing text.
+ * Typed decisions: questions a model answers with a yes or no, a choice, a score, or a box around
+ * something in a picture, each with a confidence read alike from any provider, so software acts on an
+ * answer without parsing text.
  */
 import type { CacheAdapter } from '../cache/adapters.js';
+import type { ImageContent } from '../types/messages.js';
 import type { OperationLifecycleLike, ProviderCallContext } from '../types/lifecycle.js';
 import type { ResponseCost } from '../types/response.js';
 import type { RunKind } from '../types/tracing.js';
-/** The kinds of question there are: a yes or no, a choice among options, or a score on levels. */
-export type DecisionQuestionType = 'yes-no' | 'choice' | 'score';
+/**
+ * The kinds of question there are: a yes or no, a choice among options, a score on levels, or a box
+ * around something in a picture.
+ */
+export type DecisionQuestionType = 'yes-no' | 'choice' | 'score' | 'box';
 /** A question answered yes or no, with the probability that it is yes. */
 export interface YesNoQuestion {
     /** Discriminates the question. */
@@ -38,12 +43,40 @@ export interface ScoreQuestion {
     /** What each level means, lowest first. Two to ten. */
     levels: readonly string[];
 }
+/** A question answered with where something is in a picture of the state: a box around it. */
+export interface BoxQuestion {
+    /** Discriminates the question. */
+    type: 'box';
+    /**
+     * What to find, and in which picture, by its path in backticks, such as "the dent in `photo`". A box
+     * is given even for something the picture does not show: ask a yes or no beside it whether it does.
+     */
+    question: string;
+}
 /** Any question. */
-export type DecisionQuestion = YesNoQuestion | ChoiceQuestion | ScoreQuestion;
+export type DecisionQuestion = YesNoQuestion | ChoiceQuestion | ScoreQuestion | BoxQuestion;
 /** Questions by the names their answers come back under. */
 export type DecisionQuestions = Record<string, DecisionQuestion>;
-/** What every question of a decision is about: text, or any JSON value. */
+/**
+ * What every question of a decision is about: text, or any JSON value. An object or a list may hold
+ * pictures, as image parts (`{ type: 'image', source }`) or image assets, anywhere in it; a question
+ * names each by its path in backticks, such as `photo` or `frames[2].frame`.
+ */
 export type DecisionState = string | Record<string, unknown> | readonly unknown[];
+/**
+ * Where something is in a picture: each edge a fraction, from 0 to 1, of the picture's height (`top`,
+ * `bottom`) or width (`left`, `right`), measured from its top-left corner.
+ */
+export interface DecisionBox {
+    /** The top edge, from the picture's top. */
+    top: number;
+    /** The bottom edge, from the picture's top: at least `top`. */
+    bottom: number;
+    /** The left edge, from the picture's left. */
+    left: number;
+    /** The right edge, from the picture's left: at least `left`. */
+    right: number;
+}
 /** A yes-or-no question's answer. */
 export interface YesNoAnswer {
     /** Discriminates the answer. */
@@ -86,15 +119,27 @@ export interface ScoreAnswer {
      */
     confidence: number;
 }
+/** A box question's answer. */
+export interface BoxAnswer {
+    /** Discriminates the answer. */
+    type: 'box';
+    /** Where it is. */
+    box: DecisionBox;
+    /**
+     * How sure the answer is, from 0 to 1: the provider's own, such as TypeSafe AI's Jev gives, or how
+     * far several readings of the box agree, for a model asked more than once.
+     */
+    confidence: number;
+}
 /** Any answer. */
-export type DecisionAnswer = YesNoAnswer | ChoiceAnswer | ScoreAnswer;
+export type DecisionAnswer = YesNoAnswer | ChoiceAnswer | ScoreAnswer | BoxAnswer;
 /** The options of a choice question, as a union of their names. */
 type OptionsOf<O> = O extends readonly (infer K)[] ? K : keyof O;
 /** The answer a question gets: a choice's answer names only its options. */
 export type AnswerOf<Q> = Q extends YesNoQuestion ? YesNoAnswer : Q extends {
     type: 'choice';
     options: infer O;
-} ? ChoiceAnswer<Extract<OptionsOf<O>, string>> : Q extends ScoreQuestion ? ScoreAnswer : DecisionAnswer;
+} ? ChoiceAnswer<Extract<OptionsOf<O>, string>> : Q extends ScoreQuestion ? ScoreAnswer : Q extends BoxQuestion ? BoxAnswer : DecisionAnswer;
 /** Answers by their questions' names, each typed by its question. */
 export type DecisionAnswers<Q extends DecisionQuestions> = {
     [K in keyof Q]: AnswerOf<Q[K]>;
@@ -152,12 +197,12 @@ export type DecisionProviderQuestion = YesNoQuestion | {
         name: string;
         description?: string;
     }>;
-} | ScoreQuestion;
+} | ScoreQuestion | BoxQuestion;
 /** What a provider is asked. */
 export interface DecisionProviderRequest {
     /** The model to ask. */
     model: string;
-    /** What every question is about. */
+    /** What every question is about, with any pictures it holds, which `statePictures()` lists. */
     state: DecisionState;
     /** The questions, by name. */
     questions: Record<string, DecisionProviderQuestion>;
@@ -172,6 +217,10 @@ export type DecisionDistribution = {
 } | {
     type: 'score';
     probabilities: readonly number[];
+} | {
+    type: 'box';
+    box: DecisionBox;
+    confidence: number;
 };
 /** What a provider answers. */
 export interface DecisionProviderResult {
@@ -194,6 +243,8 @@ export interface DecisionProviderCapabilities {
     maxLevels: number;
     /** Most questions one decision may ask, when it has a limit. */
     maxQuestions?: number;
+    /** Whether it reads the pictures a state holds. A provider without it is not asked about a picture. */
+    images?: boolean;
     /** The model asked when a request names none. */
     defaultModel: string;
     /** The models it serves, when it lists them. */
@@ -232,30 +283,7 @@ export interface DecisionProvider {
     /** Answers every question of a decision. */
     decide(request: DecisionProviderRequest, context: DecisionCallContext): Promise<DecisionProviderResult>;
 }
-/** Why a decision failed. */
-export type DecisionErrorCode = 'DECISION_NO_PROVIDER' | 'DECISION_UNSUPPORTED' | 'DECISION_INVALID' | 'DECISION_FAILED';
-/** Raised for a decision no provider can serve, a question that is malformed, or a provider's failure. */
-export declare class DecisionError extends Error {
-    /**
-     * A stable code: `DECISION_NO_PROVIDER`, `DECISION_UNSUPPORTED` (a provider cannot take the
-     * question), `DECISION_INVALID` (the request, or an answer, is malformed), or `DECISION_FAILED`.
-     */
-    readonly code: DecisionErrorCode;
-    /** Whether asking again, or another provider, may help: a timeout or an overloaded service may. */
-    readonly retryable: boolean;
-    /** The HTTP status a provider answered with, when it did. */
-    readonly status?: number | undefined;
-    constructor(message: string, 
-    /**
-     * A stable code: `DECISION_NO_PROVIDER`, `DECISION_UNSUPPORTED` (a provider cannot take the
-     * question), `DECISION_INVALID` (the request, or an answer, is malformed), or `DECISION_FAILED`.
-     */
-    code: DecisionErrorCode, 
-    /** Whether asking again, or another provider, may help: a timeout or an overloaded service may. */
-    retryable?: boolean, 
-    /** The HTTP status a provider answered with, when it did. */
-    status?: number | undefined);
-}
+export { DecisionError, type DecisionErrorCode } from './errors.js';
 /** A yes-or-no answer's confidence: how far its probability is from an even chance, `|2p - 1|`. */
 export declare function yesNoConfidence(probability: number): number;
 /**
@@ -275,6 +303,11 @@ export declare function scoreConfidence(probabilities: readonly number[]): numbe
  * `DECISION_INVALID` for probabilities that do not fit the question.
  */
 export declare function decisionAnswer(question: DecisionProviderQuestion, distribution: DecisionDistribution): DecisionAnswer;
+/**
+ * How far two boxes overlap, from 0 to 1: the area they share over the area they cover together, the
+ * intersection over union by which a box is judged against where the thing is.
+ */
+export declare function boxOverlap(a: DecisionBox, b: DecisionBox): number;
 /** What to do with an answer: act on it, have a person review it, or neither. */
 export type GateOutcome = 'act' | 'review' | 'abstain';
 /** The confidences a gate acts at, and has reviewed at. */
@@ -345,6 +378,7 @@ export interface CalibrationReport {
 /**
  * The probability an answer gives its own answer: a yes or no's `max(p, 1 - p)`, a choice's for its
  * option, and a score's for its level. Its confidence measures how far that is above an even spread.
+ * A box has no probabilities, so its confidence stands for one.
  */
 export declare function answerProbability(answer: DecisionAnswer): number;
 /**
@@ -355,11 +389,13 @@ export declare function answerProbability(answer: DecisionAnswer): number;
  */
 export declare function calibrate(samples: readonly CalibrationSample[], options?: CalibrateOptions): CalibrationReport;
 /**
- * Whether an answer is right: a yes-or-no or a choice by its answer, and a score by its most probable
- * level, or by its weighted score within `tolerance` levels when one is given.
+ * Whether an answer is right: a yes-or-no or a choice by its answer, a score by its most probable
+ * level, or by its weighted score within `tolerance` levels when one is given, and a box by how far it
+ * overlaps the expected one, `boxOverlap()`, at least `overlap` (one half by default).
  */
-export declare function answeredRight(answer: DecisionAnswer, expected: boolean | string | number, options?: {
+export declare function answeredRight(answer: DecisionAnswer, expected: boolean | string | number | DecisionBox, options?: {
     tolerance?: number;
+    overlap?: number;
 }): boolean;
 /** A decision model's price, in US dollars per million tokens. */
 export interface DecisionPrice {
@@ -493,8 +529,11 @@ export interface DecisionConformanceCase {
     state: DecisionState;
     /** The question. */
     question: DecisionQuestion;
-    /** The right answer: a yes or no, a choice's option, or a score's level. */
-    expected: boolean | string | number;
+    /**
+     * The right answer: a yes or no, a choice's option, a score's level, or where a box should be, which
+     * an answer overlapping it by half or more matches.
+     */
+    expected: boolean | string | number | DecisionBox;
 }
 /** Options for `runDecisionConformance()`. */
 export interface DecisionConformanceOptions {
@@ -505,6 +544,11 @@ export interface DecisionConformanceOptions {
      * as it is told passes without them; a real model is held to them.
      */
     cases?: readonly DecisionConformanceCase[];
+    /**
+     * A picture for the checks of a provider that reads pictures or answers boxes: it is asked a yes or
+     * no about the picture, and a box around what it shows. Without one, those checks are skipped.
+     */
+    picture?: ImageContent;
 }
 /** How a provider did against the decision contract. */
 export interface DecisionConformanceReport {
@@ -514,12 +558,13 @@ export interface DecisionConformanceReport {
     passed: string[];
     /** Checks that failed, each with why. */
     failures: string[];
+    /** Checks that were not run, each with why, such as a box with no picture given to ask about. */
+    skipped: string[];
 }
 /**
  * Checks a provider against the decision contract: its capabilities are coherent; each question type
- * it declares comes back as probabilities that fit the question; several questions in one decision
- * each get their own answer; a stopped call is not answered; and every case is answered right. Never
- * throws; failures are listed.
+ * it declares comes back as probabilities, or a box, that fit the question; a provider that reads
+ * pictures answers about one; several questions in one decision each get their own answer; a stopped
+ * call is not answered; and every case is answered right. Never throws; failures are listed.
  */
 export declare function runDecisionConformance(provider: DecisionProvider, options?: DecisionConformanceOptions): Promise<DecisionConformanceReport>;
-export {};

package/dist/decisions/core.js

@@ -2,18 +2,9 @@ import { createCacheKey } from '../cache/memory-cache.js';
 import { withLifecycle } from '../lifecycle/run.js';
 import { generateRequestId } from '../utils/ids.js';
 import { sha256Hex } from '../utils/sha256.js';
-export class DecisionError extends Error {
-    code;
-    retryable;
-    status;
-    constructor(message, code, retryable = false, status) {
-        super(message);
-        this.code = code;
-        this.retryable = retryable;
-        this.status = status;
-        this.name = 'DecisionError';
-    }
-}
+import { DecisionError } from './errors.js';
+import { pictureDigest, replacePictures, statePictures } from './pictures.js';
+export { DecisionError } from './errors.js';
 export function yesNoConfidence(probability) {
     return clamp(Math.abs(2 * probability - 1));
 }
@@ -73,8 +64,36 @@ export function decisionAnswer(question, distribution) {
             confidence: scoreConfidence(weights),
         };
     }
+    if (question.type === 'box' && distribution.type === 'box') {
+        const edge = (name) => {
+            const value = distribution.box?.[name];
+            if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || value > 1) {
+                throw new DecisionError(`The box's ${name} edge is ${String(value)}, not a number from 0 to 1`, 'DECISION_INVALID');
+            }
+            return value;
+        };
+        const box = { top: edge('top'), bottom: edge('bottom'), left: edge('left'), right: edge('right') };
+        if (box.top > box.bottom || box.left > box.right) {
+            throw new DecisionError(`The box's edges cross: top ${box.top} and bottom ${box.bottom}, left ${box.left} and right ${box.right}`, 'DECISION_INVALID');
+        }
+        const confidence = distribution.confidence;
+        if (typeof confidence !== 'number' || !Number.isFinite(confidence) || confidence < 0 || confidence > 1) {
+            throw new DecisionError(`The box's confidence is ${String(confidence)}, not a number from 0 to 1`, 'DECISION_INVALID');
+        }
+        return { type: 'box', box, confidence };
+    }
     throw new DecisionError(`Unknown question type "${question.type}"`, 'DECISION_INVALID');
 }
+export function boxOverlap(a, b) {
+    const width = Math.max(0, Math.min(a.right, b.right) - Math.max(a.left, b.left));
+    const height = Math.max(0, Math.min(a.bottom, b.bottom) - Math.max(a.top, b.top));
+    const shared = width * height;
+    const area = (box) => Math.max(0, box.right - box.left) * Math.max(0, box.bottom - box.top);
+    const union = area(a) + area(b) - shared;
+    if (union <= 0)
+        return a.top === b.top && a.left === b.left && a.bottom === b.bottom && a.right === b.right ? 1 : 0;
+    return shared / union;
+}
 export function gate(answer, thresholds) {
     if (answer.confidence >= thresholds.act)
         return 'act';
@@ -85,6 +104,8 @@ export function answerProbability(answer) {
         return Math.max(answer.probability, 1 - answer.probability);
     if (answer.type === 'choice')
         return answer.probabilities[answer.choice] ?? 0;
+    if (answer.type === 'box')
+        return answer.confidence;
     return answer.probabilities[answer.level] ?? 0;
 }
 export function calibrate(samples, options = {}) {
@@ -144,6 +165,11 @@ export function answeredRight(answer, expected, options = {}) {
         return answer.yes === expected;
     if (answer.type === 'choice')
         return answer.choice === expected;
+    if (answer.type === 'box') {
+        if (expected === null || typeof expected !== 'object')
+            return false;
+        return boxOverlap(answer.box, expected) >= (options.overlap ?? 0.5);
+    }
     if (typeof expected !== 'number')
         return false;
     return options.tolerance === undefined
@@ -194,6 +220,7 @@ export class DecisionManager {
     async decide(request) {
         request.signal?.throwIfAborted();
         const questions = normalizeQuestions(request.questions);
+        const pictures = picturesOf(request.state, questions);
         const chosen = request.provider ?? this.config.defaultProvider ?? this.providers.keys().next().value;
         if (chosen === undefined)
             throw new DecisionError('No decision provider is registered', 'DECISION_NO_PROVIDER');
@@ -203,14 +230,14 @@ export class DecisionManager {
             const provider = this.providers.get(name);
             if (!provider)
                 throw new DecisionError(`No decision provider is registered as "${name}"`, 'DECISION_NO_PROVIDER');
-            const unfit = unsupported(provider, questions);
+            const unfit = unsupported(provider, questions, pictures);
             if (unfit) {
                 if (position > 0)
                     continue;
                 throw new DecisionError(`Decision provider "${name}" ${unfit}`, 'DECISION_UNSUPPORTED');
             }
             try {
-                return (await this.ask(name, provider, request, questions));
+                return (await this.ask(name, provider, request, questions, pictures));
             }
             catch (error) {
                 last = error;
@@ -220,12 +247,15 @@ export class DecisionManager {
         }
         throw last;
     }
-    async ask(name, provider, request, questions) {
+    async ask(name, provider, request, questions, pictures) {
         const model = request.model ?? provider.info.capabilities.defaultModel;
         const started = Date.now();
         const cache = request.cache === false ? undefined : this.config.cache;
+        const described = pictures.length
+            ? replacePictures(request.state, (picture) => ({ picture: pictureDigest(picture.part) }))
+            : request.state;
         const key = cache
-            ? `nexus:decision:${sha256Hex(createCacheKey({ name, model, tenant: request.tenantId, state: request.state, questions }))}`
+            ? `nexus:decision:${sha256Hex(createCacheKey({ name, model, tenant: request.tenantId, state: described, questions }))}`
             : undefined;
         if (cache && key) {
             const hit = await cache.store.get(key);
@@ -245,7 +275,7 @@ export class DecisionManager {
         const run = this.runtime.tracer?.startRun({
             name: `decide ${Object.keys(questions).join(', ')}`,
             kind: 'decision',
-            inputs: { state: request.state, questions },
+            inputs: { state: described, questions },
             model,
             provider: name,
             ...(request.metadata ? { metadata: request.metadata } : {}),
@@ -257,11 +287,18 @@ export class DecisionManager {
             provider: name,
             model,
             ...(request.tenantId !== undefined ? { tenantId: request.tenantId } : {}),
-            metadata: { questions: Object.keys(questions).length, ...request.metadata },
+            metadata: {
+                questions: Object.keys(questions).length,
+                ...(pictures.length ? { pictures: pictures.length } : {}),
+                ...request.metadata,
+            },
         }, request.signal ? { signal: request.signal } : {}, async (ticket) => {
             const estimate = charged
                 ? undefined
-                : priceDecision(model, { inputTokens: Math.ceil(JSON.stringify({ state: request.state, questions }).length / 4) }, this.prices());
+                : priceDecision(model, {
+                    inputTokens: Math.ceil(JSON.stringify({ state: described, questions }).length / 4) +
+                        pictures.length * PICTURE_TOKENS,
+                }, this.prices());
             if (ticket && estimate)
                 await ticket.reserve(estimate.amount);
             const answered = await provider.decide({ model, state: request.state, questions }, {
@@ -321,7 +358,7 @@ export async function decide(provider, request) {
     return new DecisionManager({ providers: { [provider.info.name]: provider } }).decide(request);
 }
 export async function runDecisionConformance(provider, options = {}) {
-    const report = { provider: provider.info.name, passed: [], failures: [] };
+    const report = { provider: provider.info.name, passed: [], failures: [], skipped: [] };
     const check = async (name, run) => {
         try {
             await run();
@@ -342,6 +379,8 @@ export async function runDecisionConformance(provider, options = {}) {
             throw new Error('a choice needs two options');
         if (capabilities.types.includes('score') && !(capabilities.maxLevels >= 2))
             throw new Error('a score needs two levels');
+        if (capabilities.types.includes('box') && capabilities.images !== true)
+            throw new Error('it answers boxes, and reads no pictures to find them in');
         if (!capabilities.defaultModel.trim())
             throw new Error('it names no default model');
     });
@@ -358,11 +397,19 @@ export async function runDecisionConformance(provider, options = {}) {
             options: { early: 'Before it was due', late: 'After it was due' },
         },
         score: { type: 'score', question: 'How much of it was paid?', levels: ['None', 'Some', 'All'] },
+        box: { type: 'box', question: 'What `photo` shows most, wherever it is' },
     };
+    const picture = options.picture;
+    const stateFor = (type) => type === 'box' && picture ? { invoice: state, photo: picture } : state;
+    const askable = capabilities.types.filter((type) => type !== 'box' || picture);
     for (const type of capabilities.types) {
+        if (!askable.includes(type)) {
+            report.skipped.push(`a ${type} question: no picture was given to ask about`);
+            continue;
+        }
         await check(`a ${type} question`, async () => {
             const question = samples[type];
-            const { answers } = await ask(state, { asked: question });
+            const { answers } = await ask(stateFor(type), { asked: question });
             const answer = answers.asked;
             if (answer.type !== type)
                 throw new Error(`answered as a ${answer.type}`);
@@ -370,9 +417,21 @@ export async function runDecisionConformance(provider, options = {}) {
                 throw new Error(`confidence ${answer.confidence}`);
         });
     }
+    if (capabilities.images) {
+        if (picture) {
+            await check('a question about a picture', async () => {
+                const { answers } = await ask({ invoice: state, photo: picture }, { shown: { type: 'yes-no', question: 'Does `photo` show anything at all?' } });
+                if (answers.shown.type !== 'yes-no')
+                    throw new Error(`answered as a ${answers.shown.type}`);
+            });
+        }
+        else {
+            report.skipped.push('a question about a picture: no picture was given to ask about');
+        }
+    }
     await check('several questions in one decision', async () => {
-        const questions = Object.fromEntries(capabilities.types.map((type) => [`q-${type}`, samples[type]]));
-        const { answers } = await ask(state, questions);
+        const questions = Object.fromEntries(askable.map((type) => [`q-${type}`, samples[type]]));
+        const { answers } = await ask(askable.includes('box') ? stateFor('box') : state, questions);
         for (const name of Object.keys(questions))
             if (!answers[name])
                 throw new Error(`"${name}" got no answer`);
@@ -380,11 +439,12 @@ export async function runDecisionConformance(provider, options = {}) {
     await check('a stopped decision is not answered', async () => {
         const controller = new AbortController();
         controller.abort(new Error('stopped'));
-        const question = samples[capabilities.types[0] ?? 'yes-no'];
+        const type = askable[0] ?? 'yes-no';
+        const question = samples[type];
         const answered = await provider
             .decide({
             model: options.model ?? capabilities.defaultModel,
-            state,
+            state: stateFor(type),
             questions: normalizeQuestions({ asked: question }),
         }, { signal: controller.signal, requestId: generateRequestId() })
             .then(() => true, () => false);
@@ -396,7 +456,13 @@ export async function runDecisionConformance(provider, options = {}) {
             const { answers } = await ask(item.state, { asked: item.question });
             const answer = answers.asked;
             if (!answeredRight(answer, item.expected)) {
-                const given = answer.type === 'yes-no' ? answer.yes : answer.type === 'choice' ? answer.choice : answer.level;
+                const given = answer.type === 'yes-no'
+                    ? answer.yes
+                    : answer.type === 'choice'
+                        ? answer.choice
+                        : answer.type === 'box'
+                            ? answer.box
+                            : answer.level;
                 throw new Error(`expected ${JSON.stringify(item.expected)}, answered ${JSON.stringify(given)}`);
             }
         });
@@ -438,18 +504,39 @@ function normalizeQuestions(questions) {
             }
             normalized[name] = question;
         }
+        else if (question?.type === 'box') {
+            if (typeof question.question !== 'string' || !question.question.trim()) {
+                throw new DecisionError(`Box "${name}" needs a question: what to find, and in which picture`, 'DECISION_INVALID');
+            }
+            normalized[name] = { type: 'box', question: question.question };
+        }
         else {
-            throw new DecisionError(`Question "${name}" is a yes-no, a choice, or a score, not ${JSON.stringify(question?.type)}`, 'DECISION_INVALID');
+            throw new DecisionError(`Question "${name}" is a yes-no, a choice, a score, or a box, not ${JSON.stringify(question?.type)}`, 'DECISION_INVALID');
         }
     }
     return normalized;
 }
-function unsupported(provider, questions) {
+const PICTURE_TOKENS = 1_600;
+function picturesOf(state, questions) {
+    const pictures = typeof state === 'string' ? [] : statePictures(state);
+    if (pictures.some((picture) => picture.path === '')) {
+        throw new DecisionError('A picture that is the whole state has no path for a question to name it by: hold it under a key, such as { photo: picture }', 'DECISION_INVALID');
+    }
+    const box = Object.entries(questions).find(([, question]) => question.type === 'box');
+    if (box && pictures.length === 0) {
+        throw new DecisionError(`Box "${box[0]}" asks where something is in a picture, and the state holds none`, 'DECISION_INVALID');
+    }
+    return pictures;
+}
+function unsupported(provider, questions, pictures = []) {
     const capabilities = provider.info.capabilities;
     const count = Object.keys(questions).length;
     if (capabilities.maxQuestions !== undefined && count > capabilities.maxQuestions) {
         return `takes at most ${capabilities.maxQuestions} questions, not ${count}`;
     }
+    if (pictures.length > 0 && capabilities.images !== true) {
+        return `does not read pictures, and the state holds ${pictures.length}, such as \`${pictures[0]?.path}\``;
+    }
     for (const [name, question] of Object.entries(questions)) {
         if (!capabilities.types.includes(question.type))
             return `does not answer ${question.type} questions, such as "${name}"`;

package/dist/decisions/errors.d.ts

@@ -0,0 +1,25 @@
+/** Why a decision failed, apart from the rest of the contract, so every part of it can raise one. */
+/** Why a decision failed. */
+export type DecisionErrorCode = 'DECISION_NO_PROVIDER' | 'DECISION_UNSUPPORTED' | 'DECISION_INVALID' | 'DECISION_FAILED';
+/** Raised for a decision no provider can serve, a question that is malformed, or a provider's failure. */
+export declare class DecisionError extends Error {
+    /**
+     * A stable code: `DECISION_NO_PROVIDER`, `DECISION_UNSUPPORTED` (a provider cannot take the
+     * question), `DECISION_INVALID` (the request, or an answer, is malformed), or `DECISION_FAILED`.
+     */
+    readonly code: DecisionErrorCode;
+    /** Whether asking again, or another provider, may help: a timeout or an overloaded service may. */
+    readonly retryable: boolean;
+    /** The HTTP status a provider answered with, when it did. */
+    readonly status?: number | undefined;
+    constructor(message: string, 
+    /**
+     * A stable code: `DECISION_NO_PROVIDER`, `DECISION_UNSUPPORTED` (a provider cannot take the
+     * question), `DECISION_INVALID` (the request, or an answer, is malformed), or `DECISION_FAILED`.
+     */
+    code: DecisionErrorCode, 
+    /** Whether asking again, or another provider, may help: a timeout or an overloaded service may. */
+    retryable?: boolean, 
+    /** The HTTP status a provider answered with, when it did. */
+    status?: number | undefined);
+}

package/dist/decisions/errors.js

@@ -0,0 +1,12 @@
+export class DecisionError extends Error {
+    code;
+    retryable;
+    status;
+    constructor(message, code, retryable = false, status) {
+        super(message);
+        this.code = code;
+        this.retryable = retryable;
+        this.status = status;
+        this.name = 'DecisionError';
+    }
+}

package/dist/decisions/index.d.ts

@@ -1,9 +1,10 @@
 /**
- * Typed decisions: questions a model answers with a yes or no, a choice, or a score, each with its
- * probabilities and a confidence read from them, so software acts on an answer without parsing text.
- * The contract, its manager, gates, and calibration; decisions from any completion model; and
- * decisions in graphs, agents, and evaluations.
+ * Typed decisions: questions a model answers with a yes or no, a choice, a score, or a box around
+ * something in a picture, each with a confidence read alike from any provider, so software acts on an
+ * answer without parsing text. The contract, its manager, gates, and calibration; the pictures a state
+ * holds; decisions from any completion model; and decisions in graphs, agents, and evaluations.
  */
 export { type CompletionDecisionMethod, completionDecisions, type CompletionDecisionsOptions, type DecisionCompletionClient, } from './completion.js';
 export { decisionEvaluator, type DecisionEvaluatorOptions, decisionGuard, type DecisionGuardOptions, type DecisionGuardReview, type DecisionManagerLike, decisionModel, type DecisionModelOptions, decisionPermissions, type DecisionPermissionsOptions, type DecisionRequestOptions, decisionRoute, type DecisionRouteOptions, reviewUnsure, } from './integrations.js';
-export { answeredRight, type AnswerOf, answerProbability, type CachedDecision, calibrate, type CalibrateOptions, type CalibrationBin, type CalibrationReport, type CalibrationSample, type ChoiceAnswer, choiceConfidence, type ChoiceQuestion, decide, DECISION_PRICES, decisionAnswer, type DecisionAnswer, type DecisionAnswers, type DecisionCallContext, type DecisionConfig, type DecisionConformanceCase, type DecisionConformanceOptions, type DecisionConformanceReport, type DecisionDistribution, DecisionError, type DecisionErrorCode, DecisionManager, type DecisionPrice, type DecisionProvider, type DecisionProviderCapabilities, type DecisionProviderInfo, type DecisionProviderQuestion, type DecisionProviderRequest, type DecisionProviderResult, type DecisionQuestion, type DecisionQuestions, type DecisionQuestionType, type DecisionRequest, type DecisionResult, type DecisionRunLike, type DecisionRuntime, type DecisionState, type DecisionTracerLike, type DecisionUsage, gate, type GateOutcome, type GateThresholds, priceDecision, runDecisionConformance, type ScoreAnswer, scoreConfidence, type ScoreQuestion, type YesNoAnswer, yesNoConfidence, type YesNoQuestion, } from './core.js';
+export { answeredRight, type AnswerOf, answerProbability, type BoxAnswer, boxOverlap, type BoxQuestion, type CachedDecision, calibrate, type CalibrateOptions, type CalibrationBin, type CalibrationReport, type CalibrationSample, type ChoiceAnswer, choiceConfidence, type ChoiceQuestion, decide, DECISION_PRICES, decisionAnswer, type DecisionAnswer, type DecisionAnswers, type DecisionBox, type DecisionCallContext, type DecisionConfig, type DecisionConformanceCase, type DecisionConformanceOptions, type DecisionConformanceReport, type DecisionDistribution, DecisionError, type DecisionErrorCode, DecisionManager, type DecisionPrice, type DecisionProvider, type DecisionProviderCapabilities, type DecisionProviderInfo, type DecisionProviderQuestion, type DecisionProviderRequest, type DecisionProviderResult, type DecisionQuestion, type DecisionQuestions, type DecisionQuestionType, type DecisionRequest, type DecisionResult, type DecisionRunLike, type DecisionRuntime, type DecisionState, type DecisionTracerLike, type DecisionUsage, gate, type GateOutcome, type GateThresholds, priceDecision, runDecisionConformance, type ScoreAnswer, scoreConfidence, type ScoreQuestion, type YesNoAnswer, yesNoConfidence, type YesNoQuestion, } from './core.js';
+export { type DecisionPicture, type ReadDecisionPicture, readPicture, statePictures } from './pictures.js';

package/dist/decisions/index.js

@@ -1,3 +1,4 @@
 export { completionDecisions, } from './completion.js';
 export { decisionEvaluator, decisionGuard, decisionModel, decisionPermissions, decisionRoute, reviewUnsure, } from './integrations.js';
-export { answeredRight, answerProbability, calibrate, choiceConfidence, decide, DECISION_PRICES, decisionAnswer, DecisionError, DecisionManager, gate, priceDecision, runDecisionConformance, scoreConfidence, yesNoConfidence, } from './core.js';
+export { answeredRight, answerProbability, boxOverlap, calibrate, choiceConfidence, decide, DECISION_PRICES, decisionAnswer, DecisionError, DecisionManager, gate, priceDecision, runDecisionConformance, scoreConfidence, yesNoConfidence, } from './core.js';
+export { readPicture, statePictures } from './pictures.js';

package/dist/decisions/integrations.d.ts

@@ -6,7 +6,7 @@
 import type { AgentMiddleware } from '../agent/create-agent.js';
 import type { PermissionPolicyLike, PermissionRequest } from '../agent/permissions.js';
 import type { EvaluationContext, EvaluationScore, Evaluator } from '../types/evaluate.js';
-import type { ChoiceAnswer, ChoiceQuestion, DecisionQuestions, DecisionRequest, DecisionResult, DecisionState, ScoreQuestion, YesNoAnswer, YesNoQuestion } from './core.js';
+import { type BoxQuestion, type ChoiceAnswer, type ChoiceQuestion, type DecisionBox, type DecisionQuestions, type DecisionRequest, type DecisionResult, type DecisionState, type ScoreQuestion, type YesNoAnswer, type YesNoQuestion } from './core.js';
 /** Anything that answers decisions, such as a `DecisionManager`. */
 export interface DecisionManagerLike {
     /** Answers every question of a decision. */
@@ -117,8 +117,8 @@ export declare function decisionModel<const K extends string>(decisions: Decisio
 export interface DecisionEvaluatorOptions<I = unknown, O = unknown> {
     /** The score's key. */
     key: string;
-    /** The question: a yes or no, a score, or a choice. */
-    question: YesNoQuestion | ScoreQuestion | ChoiceQuestion;
+    /** The question: a yes or no, a score, a choice, or a box about a picture the state holds. */
+    question: YesNoQuestion | ScoreQuestion | ChoiceQuestion | BoxQuestion;
     /** What the question is about. Defaults to the example's inputs, the output, and what was expected. */
     state?: (context: EvaluationContext<I, O>) => DecisionState;
     /**
@@ -126,13 +126,22 @@ export interface DecisionEvaluatorOptions<I = unknown, O = unknown> {
      * passes (the top one by default); for a choice, the option that does.
      */
     passing?: boolean | number | string;
+    /**
+     * For a box, where the thing is, such as from the example's expected output: the score is how far
+     * the box overlaps it, `boxOverlap()`, and it passes at `overlap` or more. Without it, the score is
+     * the box's confidence.
+     */
+    where?: (context: EvaluationContext<I, O>) => DecisionBox | undefined;
+    /** The overlap a box passes at, from 0 to 1. Defaults to one half. */
+    overlap?: number;
     /** Whom and what to ask. */
     request?: DecisionRequestOptions;
 }
 /**
  * An evaluator that asks a decision: the score is the probability of yes, the score's level as a
- * share of the top one, or the passing option's probability, and its metadata carries the answer's
- * confidence and probabilities, so an uncertain score can go to people with `reviewUnsure()`.
+ * share of the top one, the passing option's probability, or how far a box overlaps where the thing
+ * is, and its metadata carries the answer's confidence and probabilities, or its box, so an uncertain
+ * score can go to people with `reviewUnsure()`.
  */
 export declare function decisionEvaluator<I = unknown, O = unknown>(decisions: DecisionManagerLike, options: DecisionEvaluatorOptions<I, O>): Evaluator<I, O>;
 /**