nexus-ai-pro 2.20.2 → 2.21.0

package/dist/core/defaults.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Which release's defaults apply where an option is left out: 2.x's, or a preview of 3.0's. 3.0 turns on
+ * three settings that 2.x keeps opt-in, because turning them on changes what an upgrade does; the
+ * preview turns on all three at once, so an application can run as 3.0 will before it upgrades. An
+ * option set explicitly always wins, under either.
+ */
+/** A release whose defaults apply: `2.x`, the default, or `3.0`, a preview of the next major release's. */
+export type NexusDefaults = '2.x' | '3.0';
+/** One default that 3.0 changes: what it is, what 2.x does, what 3.0 does, and what to do first. */
+export interface DefaultChange {
+    /** The setting, as its option is named. */
+    setting: string;
+    /** Where it is set. */
+    module: string;
+    /** What 2.x does when the option is left out. */
+    today: string;
+    /** What 3.0, and the preview, does when it is left out. */
+    next: string;
+    /** What to do before turning it on, if anything. */
+    before: string;
+}
+/** The defaults 3.0 changes, each turned on by the preview. */
+export declare const DEFAULT_CHANGES: readonly DefaultChange[];
+/**
+ * Chooses which release's defaults apply to everything created after it: `3.0` previews the next
+ * major release's, `2.x` returns to today's. Call it before creating a tracer, an operation store, or
+ * a deployment: each reads the defaults when it is created. Overrides `NEXUS_DEFAULTS`.
+ */
+export declare function useDefaults(release: NexusDefaults): void;
+/**
+ * The defaults in effect: as `useDefaults()` chose, else the `NEXUS_DEFAULTS` environment variable,
+ * `2.x` or `3.0`, else `2.x`. Throws for any other value of the variable, rather than ignore a typo.
+ */
+export declare function nexusDefaults(): NexusDefaults;

package/dist/core/defaults.js

@@ -0,0 +1,39 @@
+export const DEFAULT_CHANGES = [
+    {
+        setting: 'Tracer incremental',
+        module: 'nexus-ai-pro/tracing',
+        today: 'false: a trace is written when its root run finishes',
+        next: 'true on a durable trace store: each run is written as it starts and again as it ends, so a run in a process that dies is not lost',
+        before: 'Nothing; close what a process that died left open with closeAbandonedRuns().',
+    },
+    {
+        setting: 'RedisOperationStore index',
+        module: 'nexus-ai-pro/operations',
+        today: 'false: finding queued work reads every record',
+        next: 'true where the client has zadd, zrem, and zrangebyscore: queued and leased work is kept in sorted sets',
+        before: 'Run 2.2 or later on every worker sharing the prefix, and call reindex() once.',
+    },
+    {
+        setting: 'Deployments bucketStrategy',
+        module: 'nexus-ai-pro/server',
+        today: 'fnv1a-v1 for a deployment first recorded',
+        next: 'hash-v2 for a deployment first recorded, which spreads sequential thread ids evenly; a recorded deployment keeps its own',
+        before: 'Nothing; a deployment already recorded keeps the strategy it records, and bucketing() changes one.',
+    },
+];
+let chosen;
+export function useDefaults(release) {
+    chosen = checked(release, 'useDefaults()');
+}
+export function nexusDefaults() {
+    if (chosen)
+        return chosen;
+    const variable = globalThis.process?.env
+        ?.NEXUS_DEFAULTS;
+    return variable === undefined || variable === '' ? '2.x' : checked(variable, 'NEXUS_DEFAULTS');
+}
+function checked(release, where) {
+    if (release === '2.x' || release === '3.0')
+        return release;
+    throw new Error(`${where} takes 2.x or 3.0, not ${JSON.stringify(release)}`);
+}

package/dist/decisions/completion.d.ts

@@ -1,8 +1,3 @@
-/**
- * Decisions from any completion model: each question asked on its own, answered with a one-token code,
- * and its probabilities read from the code token's log probabilities, or counted over samples where
- * the model reports none.
- */
 import type { CompletionRequest } from '../types/messages.js';
 import type { NexusResponse } from '../types/response.js';
 import { type DecisionProvider } from './core.js';
@@ -25,8 +20,14 @@ export interface CompletionDecisionsOptions {
      * a model that reports none or refuses to, which it then asks by sampling from the start.
      */
     method?: CompletionDecisionMethod;
-    /** Calls per question, when sampling. Defaults to 8. */
+    /** Calls per question, when sampling, and per box. Defaults to 8. */
     samples?: number;
+    /**
+     * Whether the model reads pictures, so a decision about the pictures a state holds, or a box, may
+     * be asked of it. Defaults to `true`; set `false` for a text-only model, which such a decision then
+     * passes over to a fallback.
+     */
+    images?: boolean;
     /** The temperature samples are drawn at. Defaults to 1. */
     temperature?: number;
     /** Questions asked at once. Defaults to 8. */
@@ -41,7 +42,9 @@ export interface CompletionDecisionsOptions {
  * Its probabilities are read from the log probabilities of that token's alternatives where the model
  * reports them, the way TypeSafe AI's Jev reads its own; elsewhere the question is asked `samples`
  * times, and each answer counted. Sampled probabilities come in steps of one sample, and cost as many
- * calls.
+ * calls. The pictures a state holds are shown to the model after the state, each labelled by its
+ * path; a box is asked `samples` times, and is the median of each edge, its confidence how far the
+ * samples overlap it.
  */
 export declare function completionDecisions(client: DecisionCompletionClient, options: CompletionDecisionsOptions): DecisionProvider & {
     readonly sampledModels: ReadonlySet<string>;

package/dist/decisions/completion.js

@@ -1,14 +1,17 @@
-import { DecisionError, } from './core.js';
+import { boxOverlap, DecisionError, } from './core.js';
+import { replacePictures, statePictures } from './pictures.js';
 const LOGPROB_OPTIONS = 20;
 export function completionDecisions(client, options) {
     const method = options.method ?? 'auto';
+    const images = options.images !== false;
     const sampled = new Set();
     const info = {
         name: options.name ?? options.model,
         capabilities: {
-            types: ['yes-no', 'choice', 'score'],
+            types: images ? ['yes-no', 'choice', 'score', 'box'] : ['yes-no', 'choice', 'score'],
             maxOptions: 255,
             maxLevels: 10,
+            images,
             defaultModel: options.model,
         },
         chargedByItsCalls: true,
@@ -21,7 +24,9 @@ export function completionDecisions(client, options) {
             messages: [
                 {
                     role: 'system',
-                    content: 'You answer one question about a state with exactly one of the codes given, and nothing else.',
+                    content: question.type === 'box'
+                        ? 'You answer where something is in a picture with exactly four numbers, and nothing else.'
+                        : 'You answer one question about a state with exactly one of the codes given, and nothing else.',
                 },
                 { role: 'user', content: prompt },
             ],
@@ -29,6 +34,8 @@ export function completionDecisions(client, options) {
             ...(context.tenantId !== undefined ? { tenantId: context.tenantId } : {}),
             ...extra,
         });
+        if (question.type === 'box')
+            return askBox(request, model, usage);
         const readable = codes.length <= LOGPROB_OPTIONS && codes.every((code) => code.length === 1);
         if (method !== 'sampling' && readable && !sampled.has(model)) {
             let response;
@@ -64,6 +71,28 @@ export function completionDecisions(client, options) {
         }
         return distributionOf(question, codes, weights);
     };
+    const askBox = async (request, model, usage) => {
+        if (method === 'logprobs') {
+            throw new DecisionError('A box is read from several samples of the answer, not from log probabilities', 'DECISION_UNSUPPORTED');
+        }
+        const count = Math.max(1, Math.floor(options.samples ?? 8));
+        const readings = await Promise.all(Array.from({ length: count }, () => client.complete(request({ temperature: options.temperature ?? 1, maxTokens: 32 })).then((response) => {
+            usage.add(response);
+            return boxIn(response.content);
+        })));
+        const boxes = readings.filter((box) => box !== undefined);
+        if (boxes.length === 0) {
+            throw new DecisionError(`"${model}" answered none of ${count} samples with a box`, 'DECISION_FAILED');
+        }
+        const box = {
+            top: median(boxes.map((item) => item.top)),
+            bottom: median(boxes.map((item) => item.bottom)),
+            left: median(boxes.map((item) => item.left)),
+            right: median(boxes.map((item) => item.right)),
+        };
+        const agreement = boxes.reduce((sum, item) => sum + boxOverlap(item, box), 0) / count;
+        return { type: 'box', box, confidence: agreement };
+    };
     return {
         info,
         sampledModels: sampled,
@@ -112,6 +141,8 @@ class Totals {
     }
 }
 function codesOf(question) {
+    if (question.type === 'box')
+        return [];
     if (question.type === 'yes-no')
         return ['Y', 'N'];
     if (question.type === 'score')
@@ -121,10 +152,37 @@ function codesOf(question) {
         : question.options.map((_, index) => String(index + 1));
 }
 function promptOf(state, question, codes, instructions) {
-    const text = typeof state === 'string' ? state : JSON.stringify(state, null, 2);
-    const lines = [`State:\n${text}`, ''];
+    const pictures = typeof state === 'string' ? [] : statePictures(state);
+    const shown = pictures.length ? replacePictures(state, (picture) => `[picture \`${picture.path}\`]`) : state;
+    const text = typeof shown === 'string' ? shown : JSON.stringify(shown, null, 2);
+    const asked = questionLines(question, codes, instructions).join('\n');
+    if (pictures.length === 0)
+        return `State:\n${text}\n\n${asked}`;
+    return [
+        { type: 'text', text: `State:\n${text}\n\nThe pictures it holds, by path:` },
+        ...pictures.flatMap((picture) => [
+            { type: 'text', text: `\`${picture.path}\`:` },
+            imagePart(picture.part),
+        ]),
+        { type: 'text', text: asked },
+    ];
+}
+function imagePart(part) {
+    if (part.type === 'image')
+        return part;
+    if (part.type === 'asset')
+        return { type: 'image', source: { asset: part.asset } };
+    return part;
+}
+function questionLines(question, codes, instructions) {
+    const lines = [];
     if (instructions)
         lines.push(instructions, '');
+    if (question.type === 'box') {
+        lines.push(`Find: ${question.question}`);
+        lines.push('', "Answer with four numbers from 0 to 1, separated by spaces: the top, bottom, left, and right edges of a box around it, each a fraction of the picture's height or width from its top-left corner.");
+        return lines;
+    }
     if (question.type === 'yes-no') {
         lines.push(`Question: ${question.question ?? 'Is the statement true of the state?'}`);
         lines.push(`Y: yes${question.criteria?.yes ? `, ${question.criteria.yes}` : ''}`);
@@ -143,7 +201,21 @@ function promptOf(state, question, codes, instructions) {
         });
     }
     lines.push('', `Answer with one code: ${codes.join(', ')}.`);
-    return lines.join('\n');
+    return lines;
+}
+function boxIn(content) {
+    const numbers = (content.match(/-?\d*\.?\d+/g) ?? []).slice(0, 4).map(Number);
+    if (numbers.length < 4 || numbers.some((value) => !(value >= 0 && value <= 1)))
+        return undefined;
+    const [top, bottom, left, right] = numbers;
+    return top <= bottom && left <= right ? { top, bottom, left, right } : undefined;
+}
+function median(values) {
+    const sorted = [...values].sort((a, b) => a - b);
+    const middle = Math.floor(sorted.length / 2);
+    return sorted.length % 2
+        ? sorted[middle]
+        : (sorted[middle - 1] + sorted[middle]) / 2;
 }
 function codeOf(token) {
     return token.replace(/^[\s"'`*(]+|[\s"'`*).:,]+$/g, '').toUpperCase();
@@ -184,6 +256,8 @@ function distributionOf(question, codes, weights) {
         return { type: 'yes-no', yes: share(0) };
     if (question.type === 'score')
         return { type: 'score', probabilities: codes.map((_, index) => share(index)) };
+    if (question.type === 'box')
+        throw new DecisionError('A box has no codes to weigh', 'DECISION_INVALID');
     return {
         type: 'choice',
         probabilities: Object.fromEntries(question.options.map((option, index) => [option.name, share(index)])),

package/dist/decisions/core.d.ts

@@ -1,13 +1,18 @@
 /**
- * Typed decisions: questions a model answers with a yes or no, a choice, or a score, each with its
- * probabilities and a confidence read from them, so software acts on an answer without parsing text.
+ * Typed decisions: questions a model answers with a yes or no, a choice, a score, or a box around
+ * something in a picture, each with a confidence read alike from any provider, so software acts on an
+ * answer without parsing text.
  */
 import type { CacheAdapter } from '../cache/adapters.js';
+import type { ImageContent } from '../types/messages.js';
 import type { OperationLifecycleLike, ProviderCallContext } from '../types/lifecycle.js';
 import type { ResponseCost } from '../types/response.js';
 import type { RunKind } from '../types/tracing.js';
-/** The kinds of question there are: a yes or no, a choice among options, or a score on levels. */
-export type DecisionQuestionType = 'yes-no' | 'choice' | 'score';
+/**
+ * The kinds of question there are: a yes or no, a choice among options, a score on levels, or a box
+ * around something in a picture.
+ */
+export type DecisionQuestionType = 'yes-no' | 'choice' | 'score' | 'box';
 /** A question answered yes or no, with the probability that it is yes. */
 export interface YesNoQuestion {
     /** Discriminates the question. */
@@ -38,12 +43,40 @@ export interface ScoreQuestion {
     /** What each level means, lowest first. Two to ten. */
     levels: readonly string[];
 }
+/** A question answered with where something is in a picture of the state: a box around it. */
+export interface BoxQuestion {
+    /** Discriminates the question. */
+    type: 'box';
+    /**
+     * What to find, and in which picture, by its path in backticks, such as "the dent in `photo`". A box
+     * is given even for something the picture does not show: ask a yes or no beside it whether it does.
+     */
+    question: string;
+}
 /** Any question. */
-export type DecisionQuestion = YesNoQuestion | ChoiceQuestion | ScoreQuestion;
+export type DecisionQuestion = YesNoQuestion | ChoiceQuestion | ScoreQuestion | BoxQuestion;
 /** Questions by the names their answers come back under. */
 export type DecisionQuestions = Record<string, DecisionQuestion>;
-/** What every question of a decision is about: text, or any JSON value. */
+/**
+ * What every question of a decision is about: text, or any JSON value. An object or a list may hold
+ * pictures, as image parts (`{ type: 'image', source }`) or image assets, anywhere in it; a question
+ * names each by its path in backticks, such as `photo` or `frames[2].frame`.
+ */
 export type DecisionState = string | Record<string, unknown> | readonly unknown[];
+/**
+ * Where something is in a picture: each edge a fraction, from 0 to 1, of the picture's height (`top`,
+ * `bottom`) or width (`left`, `right`), measured from its top-left corner.
+ */
+export interface DecisionBox {
+    /** The top edge, from the picture's top. */
+    top: number;
+    /** The bottom edge, from the picture's top: at least `top`. */
+    bottom: number;
+    /** The left edge, from the picture's left. */
+    left: number;
+    /** The right edge, from the picture's left: at least `left`. */
+    right: number;
+}
 /** A yes-or-no question's answer. */
 export interface YesNoAnswer {
     /** Discriminates the answer. */
@@ -86,15 +119,27 @@ export interface ScoreAnswer {
      */
     confidence: number;
 }
+/** A box question's answer. */
+export interface BoxAnswer {
+    /** Discriminates the answer. */
+    type: 'box';
+    /** Where it is. */
+    box: DecisionBox;
+    /**
+     * How sure the answer is, from 0 to 1: the provider's own, such as TypeSafe AI's Jev gives, or how
+     * far several readings of the box agree, for a model asked more than once.
+     */
+    confidence: number;
+}
 /** Any answer. */
-export type DecisionAnswer = YesNoAnswer | ChoiceAnswer | ScoreAnswer;
+export type DecisionAnswer = YesNoAnswer | ChoiceAnswer | ScoreAnswer | BoxAnswer;
 /** The options of a choice question, as a union of their names. */
 type OptionsOf<O> = O extends readonly (infer K)[] ? K : keyof O;
 /** The answer a question gets: a choice's answer names only its options. */
 export type AnswerOf<Q> = Q extends YesNoQuestion ? YesNoAnswer : Q extends {
     type: 'choice';
     options: infer O;
-} ? ChoiceAnswer<Extract<OptionsOf<O>, string>> : Q extends ScoreQuestion ? ScoreAnswer : DecisionAnswer;
+} ? ChoiceAnswer<Extract<OptionsOf<O>, string>> : Q extends ScoreQuestion ? ScoreAnswer : Q extends BoxQuestion ? BoxAnswer : DecisionAnswer;
 /** Answers by their questions' names, each typed by its question. */
 export type DecisionAnswers<Q extends DecisionQuestions> = {
     [K in keyof Q]: AnswerOf<Q[K]>;
@@ -152,12 +197,12 @@ export type DecisionProviderQuestion = YesNoQuestion | {
         name: string;
         description?: string;
     }>;
-} | ScoreQuestion;
+} | ScoreQuestion | BoxQuestion;
 /** What a provider is asked. */
 export interface DecisionProviderRequest {
     /** The model to ask. */
     model: string;
-    /** What every question is about. */
+    /** What every question is about, with any pictures it holds, which `statePictures()` lists. */
     state: DecisionState;
     /** The questions, by name. */
     questions: Record<string, DecisionProviderQuestion>;
@@ -172,6 +217,10 @@ export type DecisionDistribution = {
 } | {
     type: 'score';
     probabilities: readonly number[];
+} | {
+    type: 'box';
+    box: DecisionBox;
+    confidence: number;
 };
 /** What a provider answers. */
 export interface DecisionProviderResult {
@@ -194,6 +243,8 @@ export interface DecisionProviderCapabilities {
     maxLevels: number;
     /** Most questions one decision may ask, when it has a limit. */
     maxQuestions?: number;
+    /** Whether it reads the pictures a state holds. A provider without it is not asked about a picture. */
+    images?: boolean;
     /** The model asked when a request names none. */
     defaultModel: string;
     /** The models it serves, when it lists them. */
@@ -232,30 +283,7 @@ export interface DecisionProvider {
     /** Answers every question of a decision. */
     decide(request: DecisionProviderRequest, context: DecisionCallContext): Promise<DecisionProviderResult>;
 }
-/** Why a decision failed. */
-export type DecisionErrorCode = 'DECISION_NO_PROVIDER' | 'DECISION_UNSUPPORTED' | 'DECISION_INVALID' | 'DECISION_FAILED';
-/** Raised for a decision no provider can serve, a question that is malformed, or a provider's failure. */
-export declare class DecisionError extends Error {
-    /**
-     * A stable code: `DECISION_NO_PROVIDER`, `DECISION_UNSUPPORTED` (a provider cannot take the
-     * question), `DECISION_INVALID` (the request, or an answer, is malformed), or `DECISION_FAILED`.
-     */
-    readonly code: DecisionErrorCode;
-    /** Whether asking again, or another provider, may help: a timeout or an overloaded service may. */
-    readonly retryable: boolean;
-    /** The HTTP status a provider answered with, when it did. */
-    readonly status?: number | undefined;
-    constructor(message: string, 
-    /**
-     * A stable code: `DECISION_NO_PROVIDER`, `DECISION_UNSUPPORTED` (a provider cannot take the
-     * question), `DECISION_INVALID` (the request, or an answer, is malformed), or `DECISION_FAILED`.
-     */
-    code: DecisionErrorCode, 
-    /** Whether asking again, or another provider, may help: a timeout or an overloaded service may. */
-    retryable?: boolean, 
-    /** The HTTP status a provider answered with, when it did. */
-    status?: number | undefined);
-}
+export { DecisionError, type DecisionErrorCode } from './errors.js';
 /** A yes-or-no answer's confidence: how far its probability is from an even chance, `|2p - 1|`. */
 export declare function yesNoConfidence(probability: number): number;
 /**
@@ -275,6 +303,11 @@ export declare function scoreConfidence(probabilities: readonly number[]): numbe
  * `DECISION_INVALID` for probabilities that do not fit the question.
  */
 export declare function decisionAnswer(question: DecisionProviderQuestion, distribution: DecisionDistribution): DecisionAnswer;
+/**
+ * How far two boxes overlap, from 0 to 1: the area they share over the area they cover together, the
+ * intersection over union by which a box is judged against where the thing is.
+ */
+export declare function boxOverlap(a: DecisionBox, b: DecisionBox): number;
 /** What to do with an answer: act on it, have a person review it, or neither. */
 export type GateOutcome = 'act' | 'review' | 'abstain';
 /** The confidences a gate acts at, and has reviewed at. */
@@ -345,6 +378,7 @@ export interface CalibrationReport {
 /**
  * The probability an answer gives its own answer: a yes or no's `max(p, 1 - p)`, a choice's for its
  * option, and a score's for its level. Its confidence measures how far that is above an even spread.
+ * A box has no probabilities, so its confidence stands for one.
  */
 export declare function answerProbability(answer: DecisionAnswer): number;
 /**
@@ -355,11 +389,13 @@ export declare function answerProbability(answer: DecisionAnswer): number;
  */
 export declare function calibrate(samples: readonly CalibrationSample[], options?: CalibrateOptions): CalibrationReport;
 /**
- * Whether an answer is right: a yes-or-no or a choice by its answer, and a score by its most probable
- * level, or by its weighted score within `tolerance` levels when one is given.
+ * Whether an answer is right: a yes-or-no or a choice by its answer, a score by its most probable
+ * level, or by its weighted score within `tolerance` levels when one is given, and a box by how far it
+ * overlaps the expected one, `boxOverlap()`, at least `overlap` (one half by default).
  */
-export declare function answeredRight(answer: DecisionAnswer, expected: boolean | string | number, options?: {
+export declare function answeredRight(answer: DecisionAnswer, expected: boolean | string | number | DecisionBox, options?: {
     tolerance?: number;
+    overlap?: number;
 }): boolean;
 /** A decision model's price, in US dollars per million tokens. */
 export interface DecisionPrice {
@@ -493,8 +529,11 @@ export interface DecisionConformanceCase {
     state: DecisionState;
     /** The question. */
     question: DecisionQuestion;
-    /** The right answer: a yes or no, a choice's option, or a score's level. */
-    expected: boolean | string | number;
+    /**
+     * The right answer: a yes or no, a choice's option, a score's level, or where a box should be, which
+     * an answer overlapping it by half or more matches.
+     */
+    expected: boolean | string | number | DecisionBox;
 }
 /** Options for `runDecisionConformance()`. */
 export interface DecisionConformanceOptions {
@@ -505,6 +544,11 @@ export interface DecisionConformanceOptions {
      * as it is told passes without them; a real model is held to them.
      */
     cases?: readonly DecisionConformanceCase[];
+    /**
+     * A picture for the checks of a provider that reads pictures or answers boxes: it is asked a yes or
+     * no about the picture, and a box around what it shows. Without one, those checks are skipped.
+     */
+    picture?: ImageContent;
 }
 /** How a provider did against the decision contract. */
 export interface DecisionConformanceReport {
@@ -514,12 +558,13 @@ export interface DecisionConformanceReport {
     passed: string[];
     /** Checks that failed, each with why. */
     failures: string[];
+    /** Checks that were not run, each with why, such as a box with no picture given to ask about. */
+    skipped: string[];
 }
 /**
  * Checks a provider against the decision contract: its capabilities are coherent; each question type
- * it declares comes back as probabilities that fit the question; several questions in one decision
- * each get their own answer; a stopped call is not answered; and every case is answered right. Never
- * throws; failures are listed.
+ * it declares comes back as probabilities, or a box, that fit the question; a provider that reads
+ * pictures answers about one; several questions in one decision each get their own answer; a stopped
+ * call is not answered; and every case is answered right. Never throws; failures are listed.
  */
 export declare function runDecisionConformance(provider: DecisionProvider, options?: DecisionConformanceOptions): Promise<DecisionConformanceReport>;
-export {};