@xandreed/core 0.13.0 → 0.14.0

package/dist/domain/media.entity.functions.js

@@ -0,0 +1,110 @@
+import { Effect, Option, Schema } from "effect";
+import { AttachmentRejected, ImageMediaType, MEDIA_LIMITS, MediaFingerprint } from "./media.entity.js";
+export const IMAGE_TOKEN_ESTIMATE = 1600, base64Length = (bytes) => Math.ceil(bytes / 3) * 4;
+const byteAt = (bytes, index) => bytes[index] ?? 0, u16be = (bytes, index) => byteAt(bytes, index) << 8 | byteAt(bytes, index + 1), u16le = (bytes, index) => byteAt(bytes, index) | byteAt(bytes, index + 1) << 8, u24le = (bytes, index) => byteAt(bytes, index) | byteAt(bytes, index + 1) << 8 | byteAt(bytes, index + 2) << 16, u32be = (bytes, index) => byteAt(bytes, index) * 16777216 + (byteAt(bytes, index + 1) << 16 | byteAt(bytes, index + 2) << 8 | byteAt(bytes, index + 3)), ascii = (text) => Array.from(text, (char) => char.charCodeAt(0)), startsWith = (bytes, offset, signature) => bytes.length >= offset + signature.length && signature.every((byte, index) => bytes[offset + index] === byte), PNG = [137, 80, 78, 71, 13, 10, 26, 10], JPEG = [255, 216, 255];
+export const sniffImageType = (bytes) => startsWith(bytes, 0, PNG) ? Option.some("image/png") : startsWith(bytes, 0, JPEG) ? Option.some("image/jpeg") : startsWith(bytes, 0, ascii("GIF87a")) || startsWith(bytes, 0, ascii("GIF89a")) ? Option.some("image/gif") : startsWith(bytes, 0, ascii("RIFF")) && startsWith(bytes, 8, ascii("WEBP")) ? Option.some("image/webp") : Option.none();
+const sized = (width, height) => width > 0 && height > 0 ? Option.some({ width, height }) : Option.none(), JPEG_FRAMES = new Set([192, 193, 194, 195, 197, 198, 199, 201, 202, 203, 205, 206, 207]), JPEG_MAX_SEGMENTS = 4096, jpegSize = (bytes, offset, segments) => {
+  if (segments > JPEG_MAX_SEGMENTS || offset + 4 > bytes.length || byteAt(bytes, offset) !== 255)
+    return Option.none();
+  const marker = byteAt(bytes, offset + 1);
+  if (marker === 255)
+    return jpegSize(bytes, offset + 1, segments + 1);
+  if (marker === 216 || marker === 1 || marker >= 208 && marker <= 215)
+    return jpegSize(bytes, offset + 2, segments + 1);
+  if (JPEG_FRAMES.has(marker))
+    return offset + 9 <= bytes.length ? sized(u16be(bytes, offset + 7), u16be(bytes, offset + 5)) : Option.none();
+  const length = u16be(bytes, offset + 2);
+  return marker === 217 || marker === 218 || length < 2 ? Option.none() : jpegSize(bytes, offset + 2 + length, segments + 1);
+}, webpSize = (bytes) => {
+  if (bytes.length < 30)
+    return Option.none();
+  if (startsWith(bytes, 12, ascii("VP8 ")))
+    return startsWith(bytes, 23, [157, 1, 42]) ? sized(u16le(bytes, 26) & 16383, u16le(bytes, 28) & 16383) : Option.none();
+  if (startsWith(bytes, 12, ascii("VP8L"))) {
+    const [b0, b1, b2, b3] = [byteAt(bytes, 21), byteAt(bytes, 22), byteAt(bytes, 23), byteAt(bytes, 24)];
+    return byteAt(bytes, 20) === 47 ? sized(1 + ((b1 & 63) << 8 | b0), 1 + ((b3 & 15) << 10 | b2 << 2 | (b1 & 192) >> 6)) : Option.none();
+  }
+  return startsWith(bytes, 12, ascii("VP8X")) ? sized(1 + u24le(bytes, 24), 1 + u24le(bytes, 27)) : Option.none();
+};
+export const imageSizeOf = (bytes, mediaType) => {
+  if (mediaType === "image/png")
+    return bytes.length >= 24 && startsWith(bytes, 12, ascii("IHDR")) ? sized(u32be(bytes, 16), u32be(bytes, 20)) : Option.none();
+  if (mediaType === "image/gif")
+    return bytes.length >= 10 ? sized(u16le(bytes, 6), u16le(bytes, 8)) : Option.none();
+  if (mediaType === "image/webp")
+    return webpSize(bytes);
+  return jpegSize(bytes, 2, 0);
+};
+const hex8 = (value) => value.toString(16).padStart(8, "0");
+export const mediaFingerprintOf = (bytes) => {
+  const fnv = bytes.reduce((hash, byte) => Math.imul(hash ^ byte, 16777619) >>> 0, 2166136261), mixed = bytes.reduce((hash, byte) => {
+    const product = Math.imul(hash ^ byte, 1540483477);
+    return (product ^ product >>> 15) >>> 0;
+  }, 2538058380);
+  return MediaFingerprint.make(`${hex8(fnv)}${hex8(mixed)}${bytes.length.toString(16)}`);
+};
+const rejected = (reason, message) => new AttachmentRejected({ reason, message }), described = (description) => ({
+  ...description.name === void 0 ? {} : { name: description.name },
+  ...description.alt === void 0 ? {} : { alt: description.alt }
+}), encodeBase64 = Schema.encodeSync(Schema.Uint8ArrayFromBase64), decodeBase64 = Schema.decodeUnknownOption(Schema.Uint8ArrayFromBase64);
+export const mediaFromBytes = (bytes, description = {}) => {
+  if (bytes.length === 0)
+    return Effect.fail(rejected("Empty", "the image has no bytes"));
+  if (bytes.length > MEDIA_LIMITS.imageBytes)
+    return Effect.fail(rejected("TooLarge", `the image has ${bytes.length} bytes, more than ${MEDIA_LIMITS.imageBytes}`));
+  return Option.match(sniffImageType(bytes), {
+    onNone: () => Effect.fail(rejected("UnsupportedType", "the bytes are not a PNG, JPEG, GIF or WebP image")),
+    onSome: (mediaType) => Option.match(imageSizeOf(bytes, mediaType), {
+      onNone: () => Effect.fail(rejected("Unreadable", `the ${mediaType} header does not give the image's size`)),
+      onSome: (size) => size.width > MEDIA_LIMITS.maxPixels || size.height > MEDIA_LIMITS.maxPixels ? Effect.fail(rejected("TooManyPixels", `the image is ${size.width}x${size.height} pixels, more than ${MEDIA_LIMITS.maxPixels} a side`)) : Effect.succeed({
+        type: "media",
+        mediaType,
+        data: encodeBase64(bytes),
+        fingerprint: mediaFingerprintOf(bytes),
+        bytes: bytes.length,
+        ...described(description)
+      })
+    })
+  });
+}, mediaFromBase64 = (data, description = {}) => {
+  const stripped = data.replace(/^data:[^,]*;base64,/i, "").replace(/\s+/g, "");
+  if (stripped.length === 0)
+    return Effect.fail(rejected("Empty", "the image has no data"));
+  if (stripped.length > base64Length(MEDIA_LIMITS.imageBytes))
+    return Effect.fail(rejected("TooLarge", `the image's base64 has ${stripped.length} characters, more than ${base64Length(MEDIA_LIMITS.imageBytes)}`));
+  return Option.match(decodeBase64(stripped), {
+    onNone: () => Effect.fail(rejected("InvalidBase64", "the image's data is not base64")),
+    onSome: (bytes) => mediaFromBytes(bytes, description)
+  });
+}, normalizeMedia = (media) => mediaFromBase64(media.data, { ...media.name === void 0 ? {} : { name: media.name }, ...media.alt === void 0 ? {} : { alt: media.alt } });
+const sizeText = (bytes) => bytes < 1024 ? `${bytes} B` : bytes < 1048576 ? `${Math.round(bytes / 1024)} KB` : `${(bytes / 1048576).toFixed(1)} MB`;
+export const mediaLabel = (media) => `[image omitted: ${[media.mediaType, sizeText(media.bytes), ...media.name === void 0 ? [] : [media.name]].join(", ")}${media.alt === void 0 ? "" : `: ${media.alt}`}]`;
+const IMAGE_TYPES = ImageMediaType.literals, isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+export const isMediaValue = (value) => isRecord(value) && value.type === "media" && typeof value.mediaType === "string" && IMAGE_TYPES.includes(value.mediaType) && typeof value.data === "string" && typeof value.fingerprint === "string" && typeof value.bytes === "number";
+const slotsAt = (value, path) => isMediaValue(value) ? [{ path, fingerprint: value.fingerprint, mediaType: value.mediaType, bytes: value.bytes }] : Array.isArray(value) ? value.flatMap((item, index) => slotsAt(item, [...path, index])) : isRecord(value) ? Object.keys(value).sort().flatMap((key) => slotsAt(value[key], [...path, key])) : [];
+export const mediaSlotsOf = (value) => slotsAt(value, []), mediaAt = (value, slot) => slot.path.reduce((at, key) => Option.flatMap(at, (current) => typeof key === "number" ? Array.isArray(current) ? Option.fromNullishOr(current[key]) : Option.none() : isRecord(current) && Object.hasOwn(current, key) ? Option.fromNullishOr(current[key]) : Option.none()), Option.some(value)).pipe(Option.filter(isMediaValue), Option.filter((media) => media.fingerprint === slot.fingerprint && media.mediaType === slot.mediaType && media.bytes === slot.bytes));
+const unavailable = (reason) => `[image unavailable: ${reason}]`, noTally = { slots: [], labels: [], fingerprints: new Set, chars: 0 };
+export const toolResultMedia = (encoded, slots) => Effect.reduce(slots, () => noTally, (tally, slot) => Option.match(mediaAt(encoded, slot), {
+  onNone: () => Effect.succeed({ ...tally, labels: [...tally.labels, unavailable("the result holds no such image")] }),
+  onSome: (media) => tally.fingerprints.has(media.fingerprint) ? Effect.succeed(tally) : normalizeMedia(media).pipe(Effect.match({
+    onFailure: (error) => ({ ...tally, labels: [...tally.labels, unavailable(error.message)] }),
+    onSuccess: (checked) => {
+      if (checked.data !== media.data || checked.fingerprint !== media.fingerprint || checked.mediaType !== media.mediaType || checked.bytes !== media.bytes)
+        return { ...tally, labels: [...tally.labels, unavailable("its type, size or fingerprint is not what its data gives; build it with mediaFromBytes")] };
+      const fingerprints = new Set([...tally.fingerprints, media.fingerprint]);
+      return tally.slots.length >= MEDIA_LIMITS.toolResultImages || tally.chars + media.data.length > MEDIA_LIMITS.requestChars ? { ...tally, fingerprints, labels: [...tally.labels, mediaLabel(media)] } : { ...tally, fingerprints, slots: [...tally.slots, slot], chars: tally.chars + media.data.length };
+    }
+  }))
+})).pipe(Effect.map(({ slots: sent, labels }) => ({ slots: sent, labels }))), elideMediaData = (value) => {
+  if (isMediaValue(value))
+    return { ...value, data: value.fingerprint };
+  if (Array.isArray(value)) {
+    const mapped = value.map(elideMediaData);
+    return mapped.every((item, index) => item === value[index]) ? value : mapped;
+  }
+  if (isRecord(value)) {
+    const entries = Object.keys(value).map((key) => [key, value[key], elideMediaData(value[key])]);
+    return entries.every(([, before, after]) => before === after) ? value : Object.fromEntries(entries.map(([key, , after]) => [key, after]));
+  }
+  return value;
+};

package/dist/domain/media.entity.js

@@ -0,0 +1,38 @@
+import { Schema } from "effect";
+export const MEDIA_LIMITS = {
+  imageBytes: 3932160,
+  maxPixels: 8000,
+  attachments: 8,
+  requestImages: 8,
+  requestChars: 20000000,
+  toolResultImages: 4
+}, ImageMediaType = Schema.Literals(["image/png", "image/jpeg", "image/gif", "image/webp"]), MediaFingerprint = Schema.String.check(Schema.isPattern(/^[0-9a-f]{16,}$/)).pipe(Schema.brand("MediaFingerprint")), Media = Schema.Struct({
+  type: Schema.Literal("media"),
+  mediaType: ImageMediaType,
+  data: Schema.String.check(Schema.isNonEmpty(), Schema.isMaxLength(Math.ceil(MEDIA_LIMITS.imageBytes / 3) * 4), Schema.isBase64()),
+  fingerprint: MediaFingerprint,
+  bytes: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(MEDIA_LIMITS.imageBytes)),
+  name: Schema.optionalKey(Schema.String),
+  alt: Schema.optionalKey(Schema.String)
+}), MediaSlot = Schema.Struct({
+  path: Schema.Array(Schema.Union([Schema.String, Schema.Int])),
+  fingerprint: MediaFingerprint,
+  mediaType: ImageMediaType,
+  bytes: Schema.Int
+}), AttachmentRejectedReason = Schema.Literals([
+  "Empty",
+  "InvalidBase64",
+  "UnsupportedType",
+  "TooLarge",
+  "TooManyPixels",
+  "Unreadable",
+  "TooMany",
+  "TooLargeTogether"
+]);
+
+export class AttachmentRejected extends Schema.TaggedError()("AttachmentRejected", {
+  reason: AttachmentRejectedReason,
+  index: Schema.optionalKey(Schema.Int),
+  message: Schema.String
+}) {
+}

package/dist/domain/media.entity.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/domain/message.entity.d.ts

@@ -48,12 +48,59 @@ export declare const ToolResultPart: Schema.Struct<{
     readonly output: Schema.Unknown;
     readonly providerOptions: Schema.optional<Schema.Unknown>;
     readonly isError: Schema.optional<Schema.Boolean>;
+    /**
+     * Images sent with the result, after `output`'s text: memory's render of
+     * the images a result's slots name (`LogBody.ToolResult.media`). Only a
+     * rendered message has them; the loop's own tail keeps a result's images
+     * inside `output`, as the tool encoded them.
+     */
+    readonly media: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+        readonly type: Schema.Literal<"media">;
+        readonly mediaType: Schema.Literals<readonly ["image/png", "image/jpeg", "image/gif", "image/webp"]>;
+        readonly data: Schema.String;
+        readonly fingerprint: Schema.brand<Schema.String, "MediaFingerprint">;
+        readonly bytes: Schema.Int;
+        readonly name: Schema.optionalKey<Schema.String>;
+        readonly alt: Schema.optionalKey<Schema.String>;
+    }>>>;
 }>;
 export type ToolResultPart = typeof ToolResultPart.Type;
-/** A model-visible message in the user role (the user's own words, turn context, host notes). */
+/** One part of a multi-part user message: text, or an image sent inline. */
+export declare const UserContentPart: Schema.Union<readonly [Schema.Struct<{
+    readonly type: Schema.Literal<"text">;
+    readonly text: Schema.String;
+    readonly providerOptions: Schema.optional<Schema.Unknown>;
+}>, Schema.Struct<{
+    readonly type: Schema.Literal<"media">;
+    readonly mediaType: Schema.Literals<readonly ["image/png", "image/jpeg", "image/gif", "image/webp"]>;
+    readonly data: Schema.String;
+    readonly fingerprint: Schema.brand<Schema.String, "MediaFingerprint">;
+    readonly bytes: Schema.Int;
+    readonly name: Schema.optionalKey<Schema.String>;
+    readonly alt: Schema.optionalKey<Schema.String>;
+}>]>;
+export type UserContentPart = typeof UserContentPart.Type;
+/**
+ * A model-visible message in the user role (the user's own words, turn
+ * context, host notes): text, or parts when it carries images (a user's
+ * attachments). Narrow with `typeof content === "string"` before reading it
+ * as text.
+ */
 export declare const UserRoleMessage: Schema.Struct<{
     readonly role: Schema.Literal<"user">;
-    readonly content: Schema.String;
+    readonly content: Schema.Union<readonly [Schema.String, Schema.$Array<Schema.Union<readonly [Schema.Struct<{
+        readonly type: Schema.Literal<"text">;
+        readonly text: Schema.String;
+        readonly providerOptions: Schema.optional<Schema.Unknown>;
+    }>, Schema.Struct<{
+        readonly type: Schema.Literal<"media">;
+        readonly mediaType: Schema.Literals<readonly ["image/png", "image/jpeg", "image/gif", "image/webp"]>;
+        readonly data: Schema.String;
+        readonly fingerprint: Schema.brand<Schema.String, "MediaFingerprint">;
+        readonly bytes: Schema.Int;
+        readonly name: Schema.optionalKey<Schema.String>;
+        readonly alt: Schema.optionalKey<Schema.String>;
+    }>]>>]>;
     readonly providerOptions: Schema.optional<Schema.Unknown>;
 }>;
 export type UserRoleMessage = typeof UserRoleMessage.Type;
@@ -87,6 +134,21 @@ export declare const ToolMessage: Schema.Struct<{
         readonly output: Schema.Unknown;
         readonly providerOptions: Schema.optional<Schema.Unknown>;
         readonly isError: Schema.optional<Schema.Boolean>;
+        /**
+         * Images sent with the result, after `output`'s text: memory's render of
+         * the images a result's slots name (`LogBody.ToolResult.media`). Only a
+         * rendered message has them; the loop's own tail keeps a result's images
+         * inside `output`, as the tool encoded them.
+         */
+        readonly media: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+            readonly type: Schema.Literal<"media">;
+            readonly mediaType: Schema.Literals<readonly ["image/png", "image/jpeg", "image/gif", "image/webp"]>;
+            readonly data: Schema.String;
+            readonly fingerprint: Schema.brand<Schema.String, "MediaFingerprint">;
+            readonly bytes: Schema.Int;
+            readonly name: Schema.optionalKey<Schema.String>;
+            readonly alt: Schema.optionalKey<Schema.String>;
+        }>>>;
     }>>;
     readonly providerOptions: Schema.optional<Schema.Unknown>;
 }>;
@@ -94,7 +156,19 @@ export type ToolMessage = typeof ToolMessage.Type;
 /** The single conversation unit — persisted as-is, one row per entry. */
 export declare const AgentMessage: Schema.Union<readonly [Schema.Struct<{
     readonly role: Schema.Literal<"user">;
-    readonly content: Schema.String;
+    readonly content: Schema.Union<readonly [Schema.String, Schema.$Array<Schema.Union<readonly [Schema.Struct<{
+        readonly type: Schema.Literal<"text">;
+        readonly text: Schema.String;
+        readonly providerOptions: Schema.optional<Schema.Unknown>;
+    }>, Schema.Struct<{
+        readonly type: Schema.Literal<"media">;
+        readonly mediaType: Schema.Literals<readonly ["image/png", "image/jpeg", "image/gif", "image/webp"]>;
+        readonly data: Schema.String;
+        readonly fingerprint: Schema.brand<Schema.String, "MediaFingerprint">;
+        readonly bytes: Schema.Int;
+        readonly name: Schema.optionalKey<Schema.String>;
+        readonly alt: Schema.optionalKey<Schema.String>;
+    }>]>>]>;
     readonly providerOptions: Schema.optional<Schema.Unknown>;
 }>, Schema.Struct<{
     readonly role: Schema.Literal<"assistant">;
@@ -124,6 +198,21 @@ export declare const AgentMessage: Schema.Union<readonly [Schema.Struct<{
         readonly output: Schema.Unknown;
         readonly providerOptions: Schema.optional<Schema.Unknown>;
         readonly isError: Schema.optional<Schema.Boolean>;
+        /**
+         * Images sent with the result, after `output`'s text: memory's render of
+         * the images a result's slots name (`LogBody.ToolResult.media`). Only a
+         * rendered message has them; the loop's own tail keeps a result's images
+         * inside `output`, as the tool encoded them.
+         */
+        readonly media: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+            readonly type: Schema.Literal<"media">;
+            readonly mediaType: Schema.Literals<readonly ["image/png", "image/jpeg", "image/gif", "image/webp"]>;
+            readonly data: Schema.String;
+            readonly fingerprint: Schema.brand<Schema.String, "MediaFingerprint">;
+            readonly bytes: Schema.Int;
+            readonly name: Schema.optionalKey<Schema.String>;
+            readonly alt: Schema.optionalKey<Schema.String>;
+        }>>>;
     }>>;
     readonly providerOptions: Schema.optional<Schema.Unknown>;
 }>]>;
@@ -137,7 +226,19 @@ export declare const AgentResult: Schema.Struct<{
     readonly finalText: Schema.String;
     readonly messages: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
         readonly role: Schema.Literal<"user">;
-        readonly content: Schema.String;
+        readonly content: Schema.Union<readonly [Schema.String, Schema.$Array<Schema.Union<readonly [Schema.Struct<{
+            readonly type: Schema.Literal<"text">;
+            readonly text: Schema.String;
+            readonly providerOptions: Schema.optional<Schema.Unknown>;
+        }>, Schema.Struct<{
+            readonly type: Schema.Literal<"media">;
+            readonly mediaType: Schema.Literals<readonly ["image/png", "image/jpeg", "image/gif", "image/webp"]>;
+            readonly data: Schema.String;
+            readonly fingerprint: Schema.brand<Schema.String, "MediaFingerprint">;
+            readonly bytes: Schema.Int;
+            readonly name: Schema.optionalKey<Schema.String>;
+            readonly alt: Schema.optionalKey<Schema.String>;
+        }>]>>]>;
         readonly providerOptions: Schema.optional<Schema.Unknown>;
     }>, Schema.Struct<{
         readonly role: Schema.Literal<"assistant">;
@@ -167,12 +268,39 @@ export declare const AgentResult: Schema.Struct<{
             readonly output: Schema.Unknown;
             readonly providerOptions: Schema.optional<Schema.Unknown>;
             readonly isError: Schema.optional<Schema.Boolean>;
+            /**
+             * Images sent with the result, after `output`'s text: memory's render of
+             * the images a result's slots name (`LogBody.ToolResult.media`). Only a
+             * rendered message has them; the loop's own tail keeps a result's images
+             * inside `output`, as the tool encoded them.
+             */
+            readonly media: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+                readonly type: Schema.Literal<"media">;
+                readonly mediaType: Schema.Literals<readonly ["image/png", "image/jpeg", "image/gif", "image/webp"]>;
+                readonly data: Schema.String;
+                readonly fingerprint: Schema.brand<Schema.String, "MediaFingerprint">;
+                readonly bytes: Schema.Int;
+                readonly name: Schema.optionalKey<Schema.String>;
+                readonly alt: Schema.optionalKey<Schema.String>;
+            }>>>;
         }>>;
         readonly providerOptions: Schema.optional<Schema.Unknown>;
     }>]>>;
     readonly newTail: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
         readonly role: Schema.Literal<"user">;
-        readonly content: Schema.String;
+        readonly content: Schema.Union<readonly [Schema.String, Schema.$Array<Schema.Union<readonly [Schema.Struct<{
+            readonly type: Schema.Literal<"text">;
+            readonly text: Schema.String;
+            readonly providerOptions: Schema.optional<Schema.Unknown>;
+        }>, Schema.Struct<{
+            readonly type: Schema.Literal<"media">;
+            readonly mediaType: Schema.Literals<readonly ["image/png", "image/jpeg", "image/gif", "image/webp"]>;
+            readonly data: Schema.String;
+            readonly fingerprint: Schema.brand<Schema.String, "MediaFingerprint">;
+            readonly bytes: Schema.Int;
+            readonly name: Schema.optionalKey<Schema.String>;
+            readonly alt: Schema.optionalKey<Schema.String>;
+        }>]>>]>;
         readonly providerOptions: Schema.optional<Schema.Unknown>;
     }>, Schema.Struct<{
         readonly role: Schema.Literal<"assistant">;
@@ -202,6 +330,21 @@ export declare const AgentResult: Schema.Struct<{
             readonly output: Schema.Unknown;
             readonly providerOptions: Schema.optional<Schema.Unknown>;
             readonly isError: Schema.optional<Schema.Boolean>;
+            /**
+             * Images sent with the result, after `output`'s text: memory's render of
+             * the images a result's slots name (`LogBody.ToolResult.media`). Only a
+             * rendered message has them; the loop's own tail keeps a result's images
+             * inside `output`, as the tool encoded them.
+             */
+            readonly media: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+                readonly type: Schema.Literal<"media">;
+                readonly mediaType: Schema.Literals<readonly ["image/png", "image/jpeg", "image/gif", "image/webp"]>;
+                readonly data: Schema.String;
+                readonly fingerprint: Schema.brand<Schema.String, "MediaFingerprint">;
+                readonly bytes: Schema.Int;
+                readonly name: Schema.optionalKey<Schema.String>;
+                readonly alt: Schema.optionalKey<Schema.String>;
+            }>>>;
         }>>;
         readonly providerOptions: Schema.optional<Schema.Unknown>;
     }>]>>;
@@ -209,6 +352,15 @@ export declare const AgentResult: Schema.Struct<{
      *  degenerate-loop breaker stopped it, so `finalText` is NOT a deliverable. */
     readonly outcome: Schema.Literals<readonly ["ok", "partial"]>;
     readonly reason: Schema.Literals<readonly ["completed", "step-cap", "degenerate-loop"]>;
+    /** The run's provider calls summed (its assistant messages' usage). */
+    readonly usage: Schema.Struct<{
+        readonly inputTokens: Schema.Number;
+        readonly outputTokens: Schema.Number;
+        readonly totalTokens: Schema.Number;
+        readonly cacheReadTokens: Schema.Number;
+        readonly cacheWriteTokens: Schema.withDecodingDefaultKey<Schema.Number, never>;
+        readonly reasoningTokens: Schema.withDecodingDefaultKey<Schema.Number, never>;
+    }>;
 }>;
 export type AgentResult = typeof AgentResult.Type;
 export {};

package/dist/domain/message.entity.js

@@ -1,4 +1,6 @@
 import { Schema } from "effect";
+import { Media } from "./media.entity.js";
+import { TokenUsage } from "./token-usage.entity.js";
 export const ConversationId = Schema.String.check(Schema.isUUID()).pipe(Schema.brand("ConversationId")), ToolCallId = Schema.String.pipe(Schema.brand("ToolCallId"));
 
 export class Checkpoint extends Schema.Class("Checkpoint")({
@@ -29,10 +31,11 @@ export const TextPart = Schema.Struct({
   toolName: Schema.String,
   output: Schema.Unknown,
   providerOptions: Schema.optional(Schema.Unknown),
-  isError: Schema.optional(Schema.Boolean)
-}), UserRoleMessage = Schema.Struct({
+  isError: Schema.optional(Schema.Boolean),
+  media: Schema.optionalKey(Schema.Array(Media))
+}), UserContentPart = Schema.Union([TextPart, Media]), UserRoleMessage = Schema.Struct({
   role: Schema.Literal("user"),
-  content: Schema.String,
+  content: Schema.Union([Schema.String, Schema.Array(UserContentPart)]),
   providerOptions: Schema.optional(Schema.Unknown)
 }), AssistantMessage = Schema.Struct({
   role: Schema.Literal("assistant"),
@@ -47,5 +50,6 @@ export const TextPart = Schema.Struct({
   messages: Schema.Array(AgentMessage),
   newTail: Schema.Array(AgentMessage),
   outcome: Schema.Literals(["ok", "partial"]),
-  reason: Schema.Literals(["completed", "step-cap", "degenerate-loop"])
+  reason: Schema.Literals(["completed", "step-cap", "degenerate-loop"]),
+  usage: TokenUsage
 });

package/dist/domain/model-catalog.entity.d.ts

@@ -6,5 +6,7 @@ export declare const ModelCatalogEntry: Schema.Struct<{
     readonly label: Schema.optional<Schema.String>;
     readonly provider: Schema.String;
     readonly credential: Schema.Literals<readonly ["api_key", "oauth", "local"]>;
+    /** What the model reads: text, and images when its route sends them (a host can grey out an attach button). Absent: not stated. */
+    readonly input: Schema.optionalKey<Schema.$Array<Schema.Literals<readonly ["text", "image"]>>>;
 }>;
 export type ModelCatalogEntry = typeof ModelCatalogEntry.Type;

package/dist/domain/model-catalog.entity.js

@@ -3,5 +3,6 @@ export const ModelCatalogEntry = Schema.Struct({
   selection: Schema.String,
   label: Schema.optional(Schema.String),
   provider: Schema.String,
-  credential: Schema.Literals(["api_key", "oauth", "local"])
+  credential: Schema.Literals(["api_key", "oauth", "local"]),
+  input: Schema.optionalKey(Schema.Array(Schema.Literals(["text", "image"])))
 });

package/dist/domain/token-usage.entity.d.ts

@@ -1,9 +1,53 @@
 import { Schema } from "effect";
-/** Billed tokens for one model call (or a sum of them). */
+/**
+ * Billed tokens for one model call (or a sum of them). The invariants
+ * (`tokenUsage` and `extractUsage` hold them):
+ * - `inputTokens` is the full input: uncached input, cache reads and cache
+ *   writes (`uncachedInputTokens(u)` is the rest);
+ * - `outputTokens` is the full output, reasoning included;
+ * - `totalTokens = inputTokens + outputTokens`.
+ * `cacheWriteTokens` and `reasoningTokens` are missing from records written
+ * before 0.14.0; those decode with 0.
+ */
 export declare const TokenUsage: Schema.Struct<{
     readonly inputTokens: Schema.Number;
     readonly outputTokens: Schema.Number;
     readonly totalTokens: Schema.Number;
     readonly cacheReadTokens: Schema.Number;
+    readonly cacheWriteTokens: Schema.withDecodingDefaultKey<Schema.Number, never>;
+    readonly reasoningTokens: Schema.withDecodingDefaultKey<Schema.Number, never>;
 }>;
 export type TokenUsage = typeof TokenUsage.Type;
+/**
+ * One usage split into the five kinds a call is billed by. Each is at least
+ * 0, and for a usage that holds the invariants they sum to `totalTokens`:
+ * `input` is the uncached input, `output` the output without reasoning.
+ */
+export declare const UsageBreakdown: Schema.Struct<{
+    readonly input: Schema.Number;
+    readonly cacheRead: Schema.Number;
+    readonly cacheWrite: Schema.Number;
+    readonly reasoning: Schema.Number;
+    readonly output: Schema.Number;
+}>;
+export type UsageBreakdown = typeof UsageBreakdown.Type;
+/**
+ * A request's token budget (`budgetTokens`): an integer of at least 1. It is
+ * the target memory compaction keeps each request under (system prompt,
+ * tool schemas and messages, estimated), not a provider limit.
+ */
+export declare const BudgetTokens: Schema.Int;
+export type BudgetTokens = typeof BudgetTokens.Type;
+/**
+ * A model's prices, per million tokens (in the host's currency, usually
+ * USD). `reasoning` is optional: reasoning tokens are priced as output
+ * without it.
+ */
+export declare const ModelPrice: Schema.Struct<{
+    readonly input: Schema.Finite;
+    readonly output: Schema.Finite;
+    readonly cacheRead: Schema.Finite;
+    readonly cacheWrite: Schema.Finite;
+    readonly reasoning: Schema.optionalKey<Schema.Finite>;
+}>;
+export type ModelPrice = typeof ModelPrice.Type;

package/dist/domain/token-usage.entity.functions.d.ts

@@ -1,3 +1,32 @@
-import type { TokenUsage } from "./token-usage.entity.js";
+import { BudgetTokens } from "./token-usage.entity.js";
+import type { ModelPrice, TokenUsage, UsageBreakdown } from "./token-usage.entity.js";
+/** Whether a value is a valid token budget: an integer of at least 1 (`BudgetTokens`). */
+export declare const isBudgetTokens: (value: unknown) => value is BudgetTokens;
 export declare const zeroUsage: TokenUsage;
+/**
+ * A usage from a call's counts, with the invariants held: the input covers
+ * its cache reads and writes, the output covers its reasoning (a provider
+ * reporting more reasoning than output gets output = reasoning), the total
+ * is input plus output, and no count is negative.
+ */
+export declare const tokenUsage: (counts: {
+    readonly inputTokens: number;
+    readonly outputTokens: number;
+    readonly cacheReadTokens?: number;
+    readonly cacheWriteTokens?: number;
+    readonly reasoningTokens?: number;
+}) => TokenUsage;
 export declare const addUsage: (a: TokenUsage, b: TokenUsage) => TokenUsage;
+/** The sum of many usages; `zeroUsage` for none. */
+export declare const sumUsage: (usages: ReadonlyArray<TokenUsage>) => TokenUsage;
+/** The input neither read from nor written to a cache: input minus both, at least 0. */
+export declare const uncachedInputTokens: (usage: TokenUsage) => number;
+/** The five billed kinds of a usage, each at least 0 (see `UsageBreakdown`). */
+export declare const usageBreakdown: (usage: TokenUsage) => UsageBreakdown;
+/**
+ * What a usage costs at `price` (per million tokens): each kind of
+ * `usageBreakdown` at its price, reasoning at `price.reasoning` or else
+ * the output price. Cache writes have one price (a provider's different
+ * write durations are not told apart).
+ */
+export declare const costOf: (usage: TokenUsage, price: ModelPrice) => number;

package/dist/domain/token-usage.entity.functions.js

@@ -1,11 +1,31 @@
-export const zeroUsage = {
+import { Schema } from "effect";
+import { BudgetTokens } from "./token-usage.entity.js";
+export const isBudgetTokens = Schema.is(BudgetTokens), zeroUsage = {
   inputTokens: 0,
   outputTokens: 0,
   totalTokens: 0,
-  cacheReadTokens: 0
+  cacheReadTokens: 0,
+  cacheWriteTokens: 0,
+  reasoningTokens: 0
+};
+const count = (value) => Math.max(0, value ?? 0);
+export const tokenUsage = (counts) => {
+  const cacheReadTokens = count(counts.cacheReadTokens), cacheWriteTokens = count(counts.cacheWriteTokens), reasoningTokens = count(counts.reasoningTokens), inputTokens = Math.max(count(counts.inputTokens), cacheReadTokens + cacheWriteTokens), outputTokens = Math.max(count(counts.outputTokens), reasoningTokens);
+  return { inputTokens, outputTokens, totalTokens: inputTokens + outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens };
 }, addUsage = (a, b) => ({
   inputTokens: a.inputTokens + b.inputTokens,
   outputTokens: a.outputTokens + b.outputTokens,
   totalTokens: a.totalTokens + b.totalTokens,
-  cacheReadTokens: a.cacheReadTokens + b.cacheReadTokens
-});
+  cacheReadTokens: a.cacheReadTokens + b.cacheReadTokens,
+  cacheWriteTokens: a.cacheWriteTokens + b.cacheWriteTokens,
+  reasoningTokens: a.reasoningTokens + b.reasoningTokens
+}), sumUsage = (usages) => usages.reduce(addUsage, zeroUsage), uncachedInputTokens = (usage) => Math.max(0, usage.inputTokens - usage.cacheReadTokens - usage.cacheWriteTokens), usageBreakdown = (usage) => ({
+  input: uncachedInputTokens(usage),
+  cacheRead: Math.max(0, usage.cacheReadTokens),
+  cacheWrite: Math.max(0, usage.cacheWriteTokens),
+  reasoning: Math.max(0, usage.reasoningTokens),
+  output: Math.max(0, usage.outputTokens - usage.reasoningTokens)
+}), costOf = (usage, price) => {
+  const kinds = usageBreakdown(usage);
+  return (kinds.input * price.input + kinds.cacheRead * price.cacheRead + kinds.cacheWrite * price.cacheWrite + kinds.reasoning * (price.reasoning ?? price.output) + kinds.output * price.output) / 1e6;
+};

package/dist/domain/token-usage.entity.js

@@ -1,7 +1,24 @@
-import { Schema } from "effect";
+import { Effect, Schema } from "effect";
+const LaterCount = Schema.Number.pipe(Schema.withDecodingDefaultKey(Effect.succeed(0)));
 export const TokenUsage = Schema.Struct({
   inputTokens: Schema.Number,
   outputTokens: Schema.Number,
   totalTokens: Schema.Number,
-  cacheReadTokens: Schema.Number
+  cacheReadTokens: Schema.Number,
+  cacheWriteTokens: LaterCount,
+  reasoningTokens: LaterCount
+}), UsageBreakdown = Schema.Struct({
+  input: Schema.Number,
+  cacheRead: Schema.Number,
+  cacheWrite: Schema.Number,
+  reasoning: Schema.Number,
+  output: Schema.Number
+}), BudgetTokens = Schema.Int.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(1)));
+const Price = Schema.Finite.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0)));
+export const ModelPrice = Schema.Struct({
+  input: Price,
+  output: Price,
+  cacheRead: Price,
+  cacheWrite: Price,
+  reasoning: Schema.optionalKey(Price)
 });

package/dist/domain/token-usage.entity.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/harness/capability.entity.functions.js

@@ -18,7 +18,8 @@ export const defaultAnnotations = {
     compact: Option.fromNullishOr(view.compact),
     subjects: view.subjects ?? (() => []),
     artifacts: view.artifacts ?? (() => []),
-    digest: Option.fromNullishOr(view.digest)
+    digest: Option.fromNullishOr(view.digest),
+    media: Option.fromNullishOr(view.media)
   })),
   annotations: { ...defaultAnnotations, ...definition.annotations }
 }), defineSkill = (skill) => ({

package/dist/harness/plugin.adapter.d.ts

@@ -5,6 +5,9 @@ import type { TypedPlugin } from "./plugin.entity.js";
  * `build` (the graph's) and `live` (a host's own composition) share one
  * decode: the options merged over the defaults, extra keys refused — so a
  * graph caller, whose options already hold the defaults, decodes the same.
+ * A `build` whose layer fails with a `HarnessError` fails with that error's
+ * code, as `live` does (and names the plugin when the error does not); any
+ * other failure is `plugin.activation`, and a defect `plugin.defect`.
  */
 export declare const definePlugin: <A extends Readonly<Record<string, unknown>>, I, Out, E, In>(definition: {
     readonly id: string;

package/dist/harness/plugin.adapter.js

@@ -1,5 +1,11 @@
 import { Context, Effect, Layer, Schema } from "effect";
 import { HarnessError, PLUGIN_API_VERSION } from "./plugin.entity.js";
+const isHarnessError = Schema.is(Schema.Struct({
+  _tag: Schema.Literal("HarnessError"),
+  code: Schema.String,
+  message: Schema.String,
+  plugin: Schema.optional(Schema.String)
+}));
 export const definePlugin = (definition) => {
   const decodeOptions = (options) => Schema.decodeUnknownEffect(definition.config)(typeof options === "object" && options !== null ? { ...definition.defaults, ...options } : options ?? definition.defaults, { onExcessProperty: "error" });
   return {
@@ -14,7 +20,7 @@ export const definePlugin = (definition) => {
     schema: definition.config,
     config: definition.config,
     defaults: definition.defaults,
-    build: (options, services) => decodeOptions(options).pipe(Effect.flatMap((config) => Layer.build(Layer.fresh(definition.layer(config))).pipe(Effect.provide(Context.makeUnsafe(services.mapUnsafe)), Effect.map((built) => Context.makeUnsafe(Context.omit(Layer.CurrentMemoMap)(built).mapUnsafe)))), Effect.mapError((error) => new HarnessError({ code: "plugin.activation", plugin: definition.id, message: String(error) })), Effect.catchDefect((error) => Effect.fail(new HarnessError({ code: "plugin.defect", plugin: definition.id, message: String(error) })))),
+    build: (options, services) => decodeOptions(options).pipe(Effect.flatMap((config) => Layer.build(Layer.fresh(definition.layer(config))).pipe(Effect.provide(Context.makeUnsafe(services.mapUnsafe)), Effect.map((built) => Context.makeUnsafe(Context.omit(Layer.CurrentMemoMap)(built).mapUnsafe)))), Effect.mapError((error) => isHarnessError(error) ? new HarnessError({ code: error.code, message: error.message, plugin: error.plugin ?? definition.id }) : new HarnessError({ code: "plugin.activation", plugin: definition.id, message: String(error) })), Effect.catchDefect((error) => Effect.fail(new HarnessError({ code: "plugin.defect", plugin: definition.id, message: String(error) })))),
     live: (options) => Layer.unwrap(decodeOptions(options).pipe(Effect.mapError((error) => new HarnessError({ code: "config.options", plugin: definition.id, message: String(error) })), Effect.map(definition.layer)))
   };
 };