@superdurable/dex-agents 0.4.0 → 0.5.0

package/dist/src/agent-media.d.ts

@@ -0,0 +1,279 @@
+import type { AgentTurnContext } from "./agent-framework-runner.js";
+import type { AgentImageReference } from "./agent-session-values.js";
+/**
+ * Largest image a media store accepts, in bytes. Its base64 text is 5,000,000 bytes, which fits the per-image limits
+ * of Amazon Bedrock (5 MB of base64) and the Claude API (10 MB of base64).
+ * @since 0.5.0
+ */
+export declare const maxImageBytes = 3750000;
+/**
+ * Most images one user message or one tool result may carry.
+ * @since 0.5.0
+ */
+export declare const maxImagesPerMessage = 20;
+/**
+ * Image types a media store accepts, detected from the bytes.
+ * @since 0.5.0
+ */
+export declare const supportedImageMimeTypes: readonly string[];
+/**
+ * Longest {@link AgentImageReference.uri} in bytes as UTF-8, the same unit in every core; S3 object keys are at most
+ * 1024 bytes.
+ * @since 0.5.0
+ */
+export declare const maxImageUriLength = 1024;
+/**
+ * Image bytes or an image reference that Dex agents refuse.
+ *
+ * `putImage` throws it for bytes that are empty, larger than {@link maxImageBytes}, or not PNG, JPEG, GIF, or WebP.
+ * {@link validateImageReference} and {@link validateImageReferences} throw it for a reference whose fields are
+ * invalid. The message says what is wrong.
+ * @since 0.5.0
+ */
+export declare class ImageValidationError extends Error {
+    /**
+     * @param message - What is wrong with the image or reference.
+     */
+    constructor(message: string);
+}
+/**
+ * A media store cannot return the bytes of a valid reference.
+ *
+ * `getImage` throws it when the reference belongs to another store, when the store has no object for it, or when the
+ * stored bytes do not match the reference's SHA-256, size, and type.
+ * @since 0.5.0
+ */
+export declare class MediaStoreError extends Error {
+    /**
+     * @param message - Which image could not be read, and why.
+     * @param options - Standard error options, such as the `cause` from the storage service.
+     */
+    constructor(message: string, options?: ErrorOptions);
+}
+/**
+ * Keeps image bytes outside session state, addressed by their content.
+ *
+ * Implementations key every object by the SHA-256 of its bytes, under a location of their own, so `putImage` is
+ * idempotent: the same bytes give the same reference and one object, however often a retried tool Step or client
+ * uploads them. Use `S3MediaStore` from `@superdurable/dex-agents/s3` for an S3-compatible bucket,
+ * `LocalDirectoryMediaStore` when every client and Worker runs on one machine, or {@link InMemoryMediaStore} in tests.
+ * Upload with `putImage` before a reference enters a message or a tool result, and read with `getImage` or
+ * {@link loadImages} when building model input.
+ *
+ * @example
+ * ```ts
+ * const reference = await store.putImage(screenshotPng);
+ * return { output: "Took a screenshot.", images: [reference] };
+ * ```
+ * @since 0.5.0
+ */
+export interface MediaStore {
+    /**
+     * Stores one image and returns its reference.
+     *
+     * @param data - PNG, JPEG, GIF, or WebP bytes, at most {@link maxImageBytes}.
+     * @returns The reference, whose `uri` ends in `/sha256/<sha256>`.
+     * @throws {ImageValidationError} When the bytes are empty, too large, or of another type. Nothing is stored.
+     * @throws The store's own error when the upload fails; uploading the same bytes again is safe.
+     */
+    putImage(data: Uint8Array): Promise<AgentImageReference>;
+    /**
+     * Returns the bytes of one image this store holds.
+     *
+     * @param reference - A reference that this store's `putImage` returned.
+     * @returns The exact bytes, checked with {@link verifyImageBytes} against `sha256`, `sizeBytes`, and `mimeType`.
+     * @throws {ImageValidationError} When the reference's fields are invalid.
+     * @throws {MediaStoreError} When the reference belongs to another store, the object is missing, or its bytes do
+     * not match the reference.
+     * @throws The store's own error when the read fails.
+     */
+    getImage(reference: AgentImageReference): Promise<Uint8Array>;
+}
+/**
+ * Keeps images in this process, for tests and single-process tools.
+ *
+ * References use the `memory://<name>` location. Two stores with the same name accept each other's references, but
+ * each holds only what was put into it, so a Worker in another process cannot read them.
+ *
+ * @example
+ * ```ts
+ * const store = new InMemoryMediaStore();
+ * const reference = await store.putImage(pngBytes);
+ * assert.deepEqual(await store.getImage(reference), pngBytes);
+ * ```
+ * @since 0.5.0
+ */
+export declare class InMemoryMediaStore implements MediaStore {
+    /** The location that every reference of this store starts with: `"memory://<name>"`. */
+    readonly storeUri: string;
+    private readonly images;
+    /**
+     * Creates an empty store.
+     *
+     * @param name - The location name in `memory://<name>`, `"default"` when omitted. It must be non-empty and contain
+     * no `/` or whitespace.
+     * @throws {TypeError} When `name` is empty or contains `/` or whitespace.
+     * @throws {RangeError} When `name` makes image URIs longer than {@link maxImageUriLength}.
+     */
+    constructor(name?: string);
+    /** How many distinct images the store holds; uploading the same bytes twice counts once. */
+    get imageCount(): number;
+    /**
+     * Stores one image and returns its reference. See {@link MediaStore.putImage}.
+     *
+     * @param data - PNG, JPEG, GIF, or WebP bytes, at most {@link maxImageBytes}.
+     * @returns The reference.
+     * @throws {ImageValidationError} When the bytes are empty, too large, or of another type.
+     */
+    putImage(data: Uint8Array): Promise<AgentImageReference>;
+    /**
+     * Returns the bytes of one image. See {@link MediaStore.getImage}.
+     *
+     * @param reference - A reference that this store's `putImage` returned.
+     * @returns A copy of the exact bytes; changing it leaves the stored image as it was.
+     * @throws {ImageValidationError} When the reference's fields are invalid.
+     * @throws {MediaStoreError} When the reference belongs to another store, the store does not hold the image, or its
+     * bytes do not match.
+     */
+    getImage(reference: AgentImageReference): Promise<Uint8Array>;
+}
+/**
+ * Returns the MIME type of image bytes, read from their structure.
+ *
+ * The signature picks the type, then a cheap structural check must hold; it does not decode the image. PNG needs its
+ * signature followed by the `IHDR` chunk. JPEG needs SOI, a marker right after it, and EOI as the last two bytes. GIF
+ * needs a `GIF87a` or `GIF89a` header and the logical screen descriptor. WebP needs a RIFF size equal to the byte count
+ * minus 8 and a first chunk of `VP8 `, `VP8L`, or `VP8X`.
+ *
+ * @param data - Image bytes.
+ * @returns `"image/png"`, `"image/jpeg"`, `"image/gif"`, or `"image/webp"`.
+ * @throws {ImageValidationError} When the bytes are empty, start with no supported signature, or fail the structural
+ * check of their type.
+ * @since 0.5.0
+ */
+export declare function detectImageMimeType(data: Uint8Array): string;
+/**
+ * Checks image bytes and returns the reference a store keeps them under.
+ *
+ * A {@link MediaStore} implementation calls it before it uploads: the reference names the object,
+ * `imageUri(storeUri, sha256)`.
+ *
+ * @param data - Image bytes.
+ * @param storeUri - The store's location, such as `"s3://media/agents"`.
+ * @returns The reference, with the detected MIME type, SHA-256, and size.
+ * @throws {ImageValidationError} When the bytes are empty, larger than {@link maxImageBytes}, or not a well-formed
+ * PNG, JPEG, GIF, or WebP, or when {@link validateImageReference} refuses the reference, such as for a URI longer than
+ * {@link maxImageUriLength}.
+ * @since 0.5.0
+ */
+export declare function buildImageReference(data: Uint8Array, storeUri: string): AgentImageReference;
+/**
+ * Returns the URI of the image with this SHA-256 in a store.
+ *
+ * @param storeUri - The store's location; trailing `/` characters are dropped.
+ * @param sha256 - SHA-256 of the bytes as 64 lowercase hex digits.
+ * @returns `"<storeUri>/sha256/<sha256>"`.
+ * @since 0.5.0
+ */
+export declare function imageUri(storeUri: string, sha256: string): string;
+/**
+ * Checks that every image URI of a store fits {@link maxImageUriLength}.
+ *
+ * A {@link MediaStore} implementation calls it in its constructor, so a long location fails at once instead of on the
+ * first upload.
+ *
+ * @param storeUri - The store's location.
+ * @throws {RangeError} When `imageUri(storeUri, sha256)` is longer than {@link maxImageUriLength} bytes as UTF-8.
+ * @since 0.5.0
+ */
+export declare function requireShortImageUris(storeUri: string): void;
+/**
+ * Checks the fields of one image reference.
+ *
+ * The checks run in order, and the first failure is thrown: `type` is `"image"`; `mimeType` is supported; `sha256` is
+ * 64 lowercase hex digits; `sizeBytes` is from 1 to {@link maxImageBytes}; `uri` is at most
+ * {@link maxImageUriLength} bytes as UTF-8, looks like `<scheme>://<location>/sha256/<sha256>`, and ends with the
+ * reference's own `sha256`. It does not read the store.
+ *
+ * @param reference - The reference, such as one decoded from a request.
+ * @throws {ImageValidationError} When a field is invalid.
+ * @since 0.5.0
+ */
+export declare function validateImageReference(reference: AgentImageReference): void;
+/**
+ * Checks the images of one user message or one tool result.
+ *
+ * @param references - The images, at most {@link maxImagesPerMessage}.
+ * @throws {ImageValidationError} When there are too many images, or with `"images[<index>]: "` before the reason
+ * when one is invalid.
+ * @since 0.5.0
+ */
+export declare function validateImageReferences(references: readonly AgentImageReference[]): void;
+/**
+ * Checks that a valid reference names an image of the store at `storeUri`.
+ *
+ * A {@link MediaStore} implementation calls it first in `getImage`, so a request cannot make the Worker read outside
+ * the store's location.
+ *
+ * @param reference - The reference to read.
+ * @param storeUri - The store's location.
+ * @throws {ImageValidationError} When the reference's fields are invalid.
+ * @throws {MediaStoreError} When its `uri` is not `imageUri(storeUri, sha256)`.
+ * @since 0.5.0
+ */
+export declare function requireStoreImageReference(reference: AgentImageReference, storeUri: string): void;
+/**
+ * Checks bytes read from a store against their reference.
+ *
+ * The SHA-256 binds the bytes, and {@link detectImageMimeType} must find the reference's `mimeType` in them, so a
+ * reference whose type was changed after upload fails here.
+ *
+ * @param reference - The reference that was read.
+ * @param data - The bytes the store returned.
+ * @returns `data`, unchanged.
+ * @throws {MediaStoreError} When the size or the SHA-256 differs, or when the bytes are not a well-formed image of
+ * `mimeType`.
+ * @since 0.5.0
+ */
+export declare function verifyImageBytes(reference: AgentImageReference, data: Uint8Array): Uint8Array;
+/**
+ * Reads the bytes of several images, each distinct image once.
+ *
+ * A runner calls it while building model input, with the references it chose from {@link collectTurnImages} or from
+ * its own replayed items. The reads run concurrently.
+ *
+ * @example
+ * ```ts
+ * const images = await loadImages(store, collectTurnImages(turn));
+ * const parts = (message.images ?? []).map((reference) => imagePart(reference.mimeType, images.get(reference.sha256)!));
+ * ```
+ * @param store - The store that holds the images.
+ * @param references - The images to read, in any order, with repeats allowed.
+ * @returns The bytes of each image, keyed by {@link AgentImageReference.sha256}.
+ * @throws {ImageValidationError} When a reference's fields are invalid.
+ * @throws {MediaStoreError} When the store cannot return an image's exact bytes.
+ * @throws The store's own error when a read fails.
+ * @since 0.5.0
+ */
+export declare function loadImages(store: MediaStore, references: Iterable<AgentImageReference>): Promise<ReadonlyMap<string, Uint8Array>>;
+/**
+ * Returns every image a model call of this turn may show, each once.
+ *
+ * It collects `AgentMessage.images` of `turn.messages` in sequence order, then `AgentToolCall.images` of the turn's
+ * records in deferral order, keeping the first reference to each SHA-256. Whether a model call shows earlier turns'
+ * images is the runner's choice; pass only the references it needs to {@link loadImages}.
+ *
+ * @param turn - The context of one `runModelStep` call.
+ * @returns The references, without repeats.
+ * @since 0.5.0
+ */
+export declare function collectTurnImages(turn: AgentTurnContext): AgentImageReference[];
+/** Reports whether {@link validateImageReferences} accepts the images; the RPCs answer `false` instead of throwing. */
+export declare function areValidImageReferences(references: readonly AgentImageReference[]): boolean;
+/**
+ * Returns the `images` field of a succeeded record from a tool's result. An invalid reference throws
+ * {@link ImageValidationError}, which fails the tool Step attempt.
+ */
+export declare function toolResultImages(references: readonly AgentImageReference[] | undefined): {
+    readonly images?: readonly AgentImageReference[];
+};