@theoremjs/react 0.3.0 → 0.3.1

package/client/composer-hints.js

@@ -6,7 +6,7 @@
 export const STASH_SHORTCUT = 'mod+shift+s';
 /** Whether a keydown is the stash shortcut (⌘⇧S / Ctrl+Shift+S). */
 export function isStashShortcut(event) {
-    return event.code === 'KeyS' && event.shiftKey && !event.altKey && (event.metaKey || event.ctrlKey);
+    return (event.code === 'KeyS' && event.shiftKey && !event.altKey && (event.metaKey || event.ctrlKey));
 }
 /** The stash hint — shown while the whole draft is selected in the focused input and it can be stashed. */
 export function resolveComposerHint(args) {

package/client/decision-images.d.ts

@@ -0,0 +1,76 @@
+import type { DecisionAsset, DecisionJson } from '@theoremjs/agents';
+/**
+ * An image the page holds for a decision, by its reference: the text the state and the questions
+ * write where the image goes, such as `<pantry_shelf>`.
+ */
+export type DecisionImages = Record<string, DecisionAsset>;
+/** The image types a decision reads. */
+export declare const IMAGE_ACCEPT = "image/png,image/jpeg";
+/**
+ * A reference for a file that no image holds yet: its base name in angle brackets, then `_2`, `_3`,
+ * and so on. The model reads any text as a reference, so the brackets keep it from matching words.
+ */
+export declare function imageName(file: string, taken: readonly string[]): string;
+/** A file as an asset, or null when it is not a PNG or JPEG. */
+export declare function readImage(file: File): Promise<DecisionAsset | null>;
+/** The image as a `src` the page can show. */
+export declare function imageSrc(image: DecisionAsset): string;
+/** The held images the state still names, the only ones a decision sends. */
+export declare function namedImages(images: DecisionImages, state: DecisionJson): DecisionImages;
+/**
+ * The field a dropped file becomes: its reference, unused by any image or field, and the field
+ * named after it, such as `plant` holding `<plant>`.
+ */
+export declare function imageField(file: string, images: DecisionImages, fields: readonly string[]): {
+    key: string;
+    reference: string;
+};
+/** The state as named fields, or undefined when it is a list, a value, or nothing. */
+export declare function stateFields(state: DecisionJson | undefined): Record<string, DecisionJson> | undefined;
+/**
+ * The images and state after files are dropped: each file a field named after it, holding a
+ * reference no image the page or the host holds, nor any field, has yet.
+ */
+export declare function droppedImages(dropped: readonly (readonly [File, DecisionAsset])[], held: {
+    images: DecisionImages;
+    host?: DecisionImages;
+    state: Record<string, DecisionJson>;
+}): {
+    images: DecisionImages;
+    state: Record<string, DecisionJson>;
+};
+/**
+ * The image a box answer is drawn on: `field` itself when it is a held reference, else the one at
+ * that state field, else the first the state refers to, else the first held. A box answer names no image, so this is the
+ * page's best reading of where it points.
+ */
+export declare function boxSource(images: DecisionImages, state: DecisionJson, field?: string): DecisionAsset | undefined;
+/** A box answer's edges, each a share of the image from its top or left. */
+export type DecisionBox = {
+    top: number;
+    bottom: number;
+    left: number;
+    right: number;
+};
+/** The box in an image's pixels, and an outline that stays visible at any size. */
+export declare function boxRect(box: DecisionBox, width: number, height: number): {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    line: number;
+};
+/** The colours a marked image is drawn in: the box's outline, and the shade over the rest. */
+export type BoxMarks = {
+    outline: string;
+    shade: string;
+};
+/** The whole image with the box outlined and the rest shaded, as a PNG `src`. */
+export declare function markedBox(src: string, box: DecisionBox, marks: BoxMarks): Promise<string>;
+/** The part of an image a box takes, as the frame's size and the image's place inside it, in percent. */
+export declare function cropOf(box: DecisionBox): {
+    width: number;
+    height: number;
+    left: number;
+    top: number;
+};

package/client/decision-images.js

@@ -0,0 +1,139 @@
+const MIME_TYPES = new Set(['image/png', 'image/jpeg']);
+/** The image types a decision reads. */
+export const IMAGE_ACCEPT = 'image/png,image/jpeg';
+/**
+ * A reference for a file that no image holds yet: its base name in angle brackets, then `_2`, `_3`,
+ * and so on. The model reads any text as a reference, so the brackets keep it from matching words.
+ */
+export function imageName(file, taken) {
+    const base = file
+        .replace(/\.[^.]*$/, '')
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '_')
+        .replace(/^_+|_+$/g, '') || 'image';
+    let name = `<${base}>`;
+    for (let n = 2; taken.includes(name); n += 1)
+        name = `<${base}_${String(n)}>`;
+    return name;
+}
+/** A file as an asset, or null when it is not a PNG or JPEG. */
+export async function readImage(file) {
+    if (!MIME_TYPES.has(file.type))
+        return null;
+    const bytes = new Uint8Array(await file.arrayBuffer());
+    let binary = '';
+    for (let i = 0; i < bytes.length; i += 0x8000)
+        binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
+    return { mimeType: file.type, data: btoa(binary) };
+}
+/** The image as a `src` the page can show. */
+export function imageSrc(image) {
+    return `data:${image.mimeType};base64,${image.data}`;
+}
+function mentions(value, name) {
+    if (typeof value === 'string')
+        return value === name;
+    if (Array.isArray(value))
+        return value.some((each) => mentions(each, name));
+    if (value !== null && typeof value === 'object')
+        return Object.values(value).some((each) => mentions(each, name));
+    return false;
+}
+/** The held images the state still names, the only ones a decision sends. */
+export function namedImages(images, state) {
+    return Object.fromEntries(Object.entries(images).filter(([name]) => mentions(state, name)));
+}
+/**
+ * The field a dropped file becomes: its reference, unused by any image or field, and the field
+ * named after it, such as `plant` holding `<plant>`.
+ */
+export function imageField(file, images, fields) {
+    const reference = imageName(file, [...Object.keys(images), ...fields.map((key) => `<${key}>`)]);
+    return { key: reference.slice(1, -1), reference };
+}
+/** The state as named fields, or undefined when it is a list, a value, or nothing. */
+export function stateFields(state) {
+    return state !== null && typeof state === 'object' && !Array.isArray(state) ? state : undefined;
+}
+/**
+ * The images and state after files are dropped: each file a field named after it, holding a
+ * reference no image the page or the host holds, nor any field, has yet.
+ */
+export function droppedImages(dropped, held) {
+    let { images, state } = held;
+    for (const [file, image] of dropped) {
+        const { key, reference } = imageField(file.name, { ...held.host, ...images }, Object.keys(state));
+        images = { ...images, [reference]: image };
+        state = { ...state, [key]: reference };
+    }
+    return { images, state };
+}
+function at(state, path) {
+    let value = state;
+    for (const step of path.split('.')) {
+        if (value === null || typeof value !== 'object')
+            return undefined;
+        value = Array.isArray(value) ? value[Number(step)] : value[step];
+    }
+    return value;
+}
+/**
+ * The image a box answer is drawn on: `field` itself when it is a held reference, else the one at
+ * that state field, else the first the state refers to, else the first held. A box answer names no image, so this is the
+ * page's best reading of where it points.
+ */
+export function boxSource(images, state, field) {
+    const names = Object.keys(images);
+    const pointed = field === undefined || field in images ? field : at(state, field);
+    const named = (typeof pointed === 'string' && pointed in images ? pointed : undefined) ??
+        names.find((name) => mentions(state, name));
+    return images[named ?? names[0] ?? ''];
+}
+/** The box in an image's pixels, and an outline that stays visible at any size. */
+export function boxRect(box, width, height) {
+    return {
+        x: box.left * width,
+        y: box.top * height,
+        width: (box.right - box.left) * width,
+        height: (box.bottom - box.top) * height,
+        line: Math.max(2, Math.round(Math.min(width, height) / 200)),
+    };
+}
+/** How much of the shade's colour lies over the rest of the image, so the image still shows through it. */
+const SHADE_ALPHA = 0.45;
+/** The whole image with the box outlined and the rest shaded, as a PNG `src`. */
+export async function markedBox(src, box, marks) {
+    const image = new Image();
+    image.src = src;
+    await image.decode();
+    const canvas = document.createElement('canvas');
+    canvas.width = image.naturalWidth;
+    canvas.height = image.naturalHeight;
+    const context = canvas.getContext('2d');
+    if (!context)
+        return src;
+    const rect = boxRect(box, canvas.width, canvas.height);
+    context.drawImage(image, 0, 0);
+    context.fillStyle = marks.shade;
+    context.globalAlpha = SHADE_ALPHA;
+    context.beginPath();
+    context.rect(0, 0, canvas.width, canvas.height);
+    context.rect(rect.x, rect.y, rect.width, rect.height);
+    context.fill('evenodd');
+    context.globalAlpha = 1;
+    context.strokeStyle = marks.outline;
+    context.lineWidth = rect.line;
+    context.strokeRect(rect.x, rect.y, rect.width, rect.height);
+    return canvas.toDataURL('image/png');
+}
+/** The part of an image a box takes, as the frame's size and the image's place inside it, in percent. */
+export function cropOf(box) {
+    const width = box.right - box.left;
+    const height = box.bottom - box.top;
+    return {
+        width: 100 / width,
+        height: 100 / height,
+        left: (-box.left / width) * 100,
+        top: (-box.top / height) * 100,
+    };
+}

package/client/decision-transport.d.ts

@@ -4,7 +4,7 @@
  *
  * @module
  */
-import { type DecisionJson, type DecisionProfile, type DecisionQuestion, type DecisionResult, type TraceRecord } from '@theoremjs/agents';
+import { type DecisionAsset, type DecisionJson, type DecisionProfile, type DecisionQuestion, type DecisionResult, type TraceRecord } from '@theoremjs/agents';
 import type { TraceFeed } from './trace-feed.ts';
 import { type HttpOptions } from './transport.ts';
 /** One question as the page sees it: its answer type and the names of its options or levels. Instructions stay on the host. */
@@ -13,6 +13,11 @@ export type DecisionQuestionView = {
     type: DecisionQuestion['type'];
     /** A choice's labels, a score's levels from 0 up; empty for a number. */
     labels: string[];
+    /**
+     * A box's image: the one its question gives, a reference or a state field, else the field its
+     * instructions first name in backticks.
+     */
+    field?: string;
 };
 /** What a decision profile shows the page. */
 export type DecisionInterface = {
@@ -23,12 +28,19 @@ export type DecisionInterface = {
     contract: string;
     maxStateBytes?: number;
     questions: DecisionQuestionView[];
+    /** The host's own images, by the reference its questions write, sent with every decision. */
+    images?: Record<string, DecisionAsset>;
     observability?: {
         record: boolean;
     };
 };
-/** The page's view of a decision profile and the questions its host asks. */
-export declare function decisionInterface(profile: Pick<DecisionProfile, 'identity' | 'decision' | 'inputs' | 'observability'>, questions: Record<string, DecisionQuestion>): DecisionInterface;
+/** The page's view of a decision profile, the questions its host asks, and the host's own images. */
+export declare function decisionInterface(profile: Pick<DecisionProfile, 'identity' | 'decision' | 'inputs' | 'observability'>, questions: Record<string, DecisionQuestion>, images?: Record<string, DecisionAsset>): DecisionInterface;
+/**
+ * The images a decision sends: the host's own and the page's. A page image may not take a name
+ * the host's use, so the questions always read the host's picture.
+ */
+export declare function decisionAssets(host: Record<string, DecisionAsset> | undefined, page: Record<string, DecisionAsset> | undefined): Record<string, DecisionAsset> | undefined;
 /** A decision's reply: the answers, and the trace records a host that delivers them sent back. */
 export type DecisionReply = {
     result: DecisionResult;
@@ -36,10 +48,20 @@ export type DecisionReply = {
 };
 export interface DecisionTransport {
     describe(signal?: AbortSignal): Promise<DecisionInterface>;
-    decide(state: Exclude<DecisionJson, null>, signal?: AbortSignal): Promise<DecisionReply>;
+    /** `assets` are the images the state names, by reference. */
+    decide(state: Exclude<DecisionJson, null>, signal?: AbortSignal, assets?: Record<string, DecisionAsset>): Promise<DecisionReply>;
     /** Where the host's trace records land, for the inspector. */
     traces?: TraceFeed;
 }
+/** Marks a decision's failure with the trace records it left, for the page's inspector. */
+export declare function withFailedTraces(err: unknown, traces: readonly TraceRecord[]): unknown;
+/** The trace records a failed decision left; none when its host sent none. */
+export declare function failedTraces(err: unknown): readonly TraceRecord[];
+export declare const failedDecisionTraces: typeof failedTraces;
+/** POSTs `body` and reads its reply with `read`; a failure keeps the trace records its body carried. */
+export declare function postWithTraces<Reply>(url: string, body: unknown, signal: AbortSignal | undefined, options: HttpOptions, read: (raw: unknown) => Reply): Promise<Reply>;
+/** POSTs a decision and reads its reply; a failure keeps the trace records its body carried. */
+export declare function postDecision(url: string, body: unknown, signal: AbortSignal | undefined, options?: HttpOptions): Promise<DecisionReply>;
 /** A host's decide reply, checked: the answers, and any trace records it delivers. */
 export declare function readDecisionReply(raw: unknown): DecisionReply;
 /** Transport for a host mounted with `createTheoremDecisionHandler`. Default endpoint `/api/decision`. */

package/client/decision-transport.js

@@ -4,7 +4,7 @@
  *
  * @module
  */
-import { resolveObservabilityPolicy, traceRecordSchema, z, } from '@theoremjs/agents';
+import { resolveObservabilityPolicy, TheoremError, traceRecordSchema, z, } from '@theoremjs/agents';
 import { fetchJson } from "./transport.js";
 import { checkWire } from "./wire-line.js";
 /** A score level's name: its text, or the first text inside a structured entry. */
@@ -14,8 +14,23 @@ function entryLabel(entry, fallback) {
     const entries = Array.isArray(entry) ? entry : Object.keys(entry);
     return entries.length ? entryLabel(entries[0], fallback) : fallback;
 }
-/** The page's view of a decision profile and the questions its host asks. */
-export function decisionInterface(profile, questions) {
+function withField(field) {
+    return field === undefined ? {} : { field };
+}
+/** The first field an entry names in backticks, as in "the mug in `photo`". */
+function namedField(entry) {
+    if (typeof entry === 'string')
+        return /\x60([^\x60\s]+)\x60/.exec(entry)?.[1];
+    const entries = Array.isArray(entry) ? entry : Object.values(entry);
+    for (const each of entries) {
+        const field = namedField(each);
+        if (field !== undefined)
+            return field;
+    }
+    return undefined;
+}
+/** The page's view of a decision profile, the questions its host asks, and the host's own images. */
+export function decisionInterface(profile, questions, images) {
     return {
         type: 'decision',
         identity: { handle: profile.identity.handle },
@@ -31,12 +46,28 @@ export function decisionInterface(profile, questions) {
                 : question.type === 'score'
                     ? question.criteria.map((entry, level) => entryLabel(entry, String(level)))
                     : [],
+            ...(question.type === 'box'
+                ? withField(question.image ?? namedField(question.instructions))
+                : {}),
         })),
+        ...(images && Object.keys(images).length > 0 ? { images } : {}),
         ...(profile.observability
             ? { observability: { record: resolveObservabilityPolicy(profile.observability).record } }
             : {}),
     };
 }
+/**
+ * The images a decision sends: the host's own and the page's. A page image may not take a name
+ * the host's use, so the questions always read the host's picture.
+ */
+export function decisionAssets(host, page) {
+    const clash = Object.keys(page ?? {}).find((name) => host && Object.hasOwn(host, name));
+    if (clash !== undefined)
+        // labels-exempt: internal diagnostic; the user reads error.request
+        throw new TheoremError('request', `the image ${clash} is the host's, so the page's needs another name`);
+    const all = { ...host, ...page };
+    return Object.keys(all).length > 0 ? all : undefined;
+}
 const probability = z.number().min(0).max(1);
 const probabilities = z
     .record(z.string(), probability)
@@ -49,6 +80,11 @@ const answerSchema = z.discriminatedUnion('type', [
         probabilities,
     }),
     z.object({ type: z.literal('noul'), noul: probability }),
+    z.object({
+        type: z.literal('box'),
+        box: z.object({ top: probability, bottom: probability, left: probability, right: probability }),
+        confidence: probability,
+    }),
     z.object({
         type: z.literal('score'),
         score: z.number().nonnegative(),
@@ -71,9 +107,42 @@ const replySchema = z.object({
     }),
     traces: z.array(traceRecordSchema).optional(),
 });
+/** The trace records a failed decision left, by the failure thrown for it. */
+const traceFailures = new WeakMap();
+/** Marks a decision's failure with the trace records it left, for the page's inspector. */
+export function withFailedTraces(err, traces) {
+    if (traces.length > 0 && typeof err === 'object' && err !== null)
+        traceFailures.set(err, traces);
+    return err;
+}
+/** The trace records a failed decision left; none when its host sent none. */
+export function failedTraces(err) {
+    return (typeof err === 'object' && err !== null && traceFailures.get(err)) || [];
+}
+export const failedDecisionTraces = failedTraces;
+const failedBody = z.object({ traces: z.array(traceRecordSchema) });
+/** POSTs `body` and reads its reply with `read`; a failure keeps the trace records its body carried. */
+export async function postWithTraces(url, body, signal, options, read) {
+    let traces = [];
+    const failed = (raw) => {
+        const parsed = failedBody.safeParse(raw);
+        if (parsed.success)
+            traces = parsed.data.traces;
+    };
+    try {
+        return read(await fetchJson(url, { body, signal, failed }, options));
+    }
+    catch (err) {
+        throw withFailedTraces(err, traces);
+    }
+}
+/** POSTs a decision and reads its reply; a failure keeps the trace records its body carried. */
+export function postDecision(url, body, signal, options = {}) {
+    return postWithTraces(url, body, signal, options, readDecisionReply);
+}
 /** A host's decide reply, checked: the answers, and any trace records it delivers. */
 export function readDecisionReply(raw) {
-    // lexicon-exempt: internal diagnostic; the user reads error.bad_response
+    // labels-exempt: internal diagnostic; the user reads error.bad_response
     return checkWire(replySchema, raw, 'the decision');
 }
 const describedSchema = z.object({
@@ -84,9 +153,13 @@ const describedSchema = z.object({
         maxStateBytes: z.number().optional(),
         questions: z.array(z.object({
             id: z.string(),
-            type: z.enum(['choice', 'noul', 'score']),
+            type: z.enum(['choice', 'noul', 'score', 'box']),
             labels: z.array(z.string()),
+            field: z.string().optional(),
         })),
+        images: z
+            .record(z.string(), z.object({ mimeType: z.enum(['image/png', 'image/jpeg']), data: z.string() }))
+            .optional(),
         observability: z.object({ record: z.boolean() }).optional(),
     }),
 });
@@ -97,6 +170,6 @@ export function createDecisionTransport(options = {}) {
         async describe(signal) {
             return checkWire(describedSchema, await fetchJson(base, { signal }, options), 'the decision description').interface;
         },
-        decide: async (state, signal) => readDecisionReply(await fetchJson(`${base}/decide`, { body: { state }, signal }, options)),
+        decide: (state, signal, assets) => postDecision(`${base}/decide`, { state, ...(assets ? { assets } : {}) }, signal, options),
     };
 }

package/client/embedding-input.d.ts

@@ -0,0 +1,84 @@
+/**
+ * The rules behind an embedding console: the inputs as the JSON a request
+ * sends, text split into documents, documents packed into the requests the
+ * profile allows, all checked the way the kernel checks them, and documents
+ * ranked by how alike they point to a query.
+ *
+ * @module
+ */
+import { type EmbeddingInput, type EmbeddingPart, type EmbeddingUsage } from '@theoremjs/agents';
+import type { AttachmentValidationIssue } from '@theoremjs/agents/interface';
+import type { EmbeddingInterface } from './embedding-transport.ts';
+/** A part that is not text: a file sent inline, or one the provider already holds. */
+export type EmbeddingMediaPart = Exclude<EmbeddingPart, {
+    type: 'text';
+}>;
+export type EmbeddingInputCheck = {
+    ok: true;
+    inputs: EmbeddingInput[];
+} | {
+    ok: false;
+    reason: 'invalid_json' | 'no_inputs';
+} | {
+    ok: false;
+    reason: 'too_many_inputs';
+    max: number;
+    inputs: EmbeddingInput[];
+} | {
+    ok: false;
+    reason: 'empty_input' | 'text_off';
+    index: number;
+    inputs: EmbeddingInput[];
+} | {
+    ok: false;
+    reason: 'attachments';
+    issues: AttachmentValidationIssue[];
+    inputs: EmbeddingInput[];
+};
+/** The request's inputs, read from its JSON; `undefined` when it is not a list of inputs. */
+export declare function readEmbeddingInputs(text: string): EmbeddingInput[] | undefined;
+/** One input, read from its JSON; `undefined` when it is not one. */
+export declare function readEmbeddingInput(text: string): EmbeddingInput | undefined;
+/** One input as fields edit it: its words, and the files beside them. */
+export type EmbeddingFields = {
+    text: string;
+    media: EmbeddingMediaPart[];
+};
+/** The input as fields, or `null` when it holds more than one text part and fields would lose it. */
+export declare function inputFields(input: EmbeddingInput): EmbeddingFields | null;
+/** Fields as the input they send: plain text alone, else its parts with the text first. */
+export declare function fieldsInput(fields: EmbeddingFields): EmbeddingInput;
+/** The inputs, held to the profile's rules and its model's limits as the kernel holds them in one request. */
+export declare function checkInputs(inputs: readonly EmbeddingInput[], rules: EmbeddingInterface['inputs']): EmbeddingInputCheck;
+/** The request's JSON as the inputs it sends, held to the profile's rules. */
+export declare function checkEmbeddingInputs(text: string, rules: EmbeddingInterface['inputs']): EmbeddingInputCheck;
+/** Where text is cut into documents: not at all, at each line, at blank lines, at commas, or at a separator of your own. */
+export type EmbeddingSplit = 'whole' | 'lines' | 'paragraphs' | 'commas' | {
+    separator: string;
+};
+/** The text as documents, each trimmed, none empty. */
+export declare function splitText(text: string, split: EmbeddingSplit): string[];
+/** A file read as words and split, not sent as media: plain text, Markdown, CSV, and JSON. */
+export declare function readsAsText(file: {
+    name: string;
+    type: string;
+}): boolean;
+/** The most inputs the console sends in one request when the model sets no cap. */
+export declare const MAX_BATCH_INPUTS = 100;
+/**
+ * The inputs, in order, as the fewest requests the profile and its model
+ * allow: each holds at most `most` inputs and passes {@linkcode checkInputs}.
+ * An input refused on its own goes alone, for the kernel to refuse.
+ */
+export declare function embeddingBatches(inputs: readonly EmbeddingInput[], rules: EmbeddingInterface['inputs'], most?: number): EmbeddingInput[][];
+/** Two calls' usage as one: token counts and costs added, estimated when either was. */
+export declare function addUsage(a: EmbeddingUsage | undefined, b: EmbeddingUsage | undefined): EmbeddingUsage | undefined;
+/** How alike two vectors point, from -1 to 1. */
+export declare function cosineSimilarity(a: readonly number[], b: readonly number[]): number;
+/** A document and how alike it points to the query, from -1 to 1. */
+export type EmbeddingMatch = {
+    index: number;
+    score: number;
+};
+/** The documents, most alike to the query first; a vector of another width is left out. */
+export declare function rankMatches(query: readonly number[], documents: readonly (readonly number[])[]): EmbeddingMatch[];