effect 4.0.7 → 4.1.0

package/src/ai/Decision.ts

@@ -1,16 +1,20 @@
 /**
  * Defines decisions that a `DecisionModel` answers over a single input.
- * Pair an input schema with named classification, rating, or probability
- * decisions using `make`, then answer them with `DecisionModel.decide`, or
- * with `DecisionModel.decideSettled` to keep valid answers when others fail.
+ * Pair an input schema with named classification, rating, probability, or
+ * locate decisions using `make`, then answer them with `DecisionModel.decide`,
+ * or with `DecisionModel.decideSettled` to keep valid answers when others fail.
  *
  * @see {@link make} for building a definition from an input schema and decisions
  *
  * @stability unstable
  * @since 4.0.0
  */
+import * as Base64 from "../encoding/Base64.ts"
+import * as Predicate from "../Predicate.ts"
 import type * as Result from "../Result.ts"
-import type * as Schema from "../Schema.ts"
+import * as Schema from "../Schema.ts"
+import * as SchemaTransformation from "../SchemaTransformation.ts"
+import type * as DecisionModel from "./DecisionModel.ts"
 
 /**
  * Brand type for decision definitions.
@@ -100,6 +104,162 @@ export interface Probability {
   } | undefined
 }
 
+/**
+ * Decision that finds one target in an image field of the input.
+ * `image` is the field's path in the encoded input, such as `"photo"` or
+ * `"frames[2].frame"`. The answer is a bounding box around the target.
+ *
+ * @see {@link locate} for the constructor
+ * @see {@link LocateAnswer} for the answer produced by this decision
+ *
+ * @stability unstable
+ * @category models
+ * @since 4.1.0
+ */
+export interface Locate<Field extends string = string> {
+  readonly _tag: "Locate"
+  readonly image: Field
+  readonly instructions: string
+}
+
+/**
+ * Key that marks an encoded {@link Image}.
+ *
+ * @stability unstable
+ * @category models
+ * @since 4.1.0
+ */
+export const ImageKey = "~effect/ai/Decision/Image"
+
+/**
+ * JSON encoding of an {@link Image}: base64 data or a URL with its media type,
+ * under {@link ImageKey}.
+ *
+ * @stability unstable
+ * @category models
+ * @since 4.1.0
+ */
+export interface ImageEncoded {
+  readonly [ImageKey]: {
+    readonly mediaType: string
+    readonly base64: string
+  } | {
+    readonly mediaType: string
+    readonly url: string
+  }
+}
+
+const isImage = (u: unknown): u is DecisionModel.Image =>
+  Predicate.isObject(u) && Predicate.isString(u.mediaType) &&
+  (Predicate.isString(u.data) || u.data instanceof globalThis.Uint8Array || u.data instanceof globalThis.URL)
+
+/**
+ * Type-level representation of the {@link Image} schema.
+ *
+ * @stability unstable
+ * @category models
+ * @since 4.1.0
+ */
+export interface Image extends
+  Schema.decodeTo<
+    Schema.declare<DecisionModel.Image>,
+    Schema.Struct<{
+      readonly [ImageKey]: Schema.Union<
+        readonly [
+          Schema.Struct<{ readonly mediaType: Schema.String; readonly base64: Schema.String }>,
+          Schema.Struct<{ readonly mediaType: Schema.String; readonly url: Schema.URLFromString }>
+        ]
+      >
+    }>
+  >
+{
+  readonly "Rebuild": Image
+}
+
+/**
+ * Schema for an image input field. Its type is `DecisionModel.Image`: a
+ * media type with base64 data, bytes, or a URL. Decisions refer to the image
+ * by the field's path, like any other field.
+ *
+ * **Details**
+ *
+ * The field encodes to an {@link ImageEncoded} object. `DecisionModel.decide`
+ * replaces each one in the encoded input with a reference string and sends the
+ * images to the provider separately; an image used in several fields is sent
+ * once. The encoded shape also appears in JSON Schema generated from the input.
+ *
+ * **Example** (Declaring an image field)
+ *
+ * ```ts
+ * import { Schema } from "effect"
+ * import { Decision } from "effect/ai"
+ *
+ * const Listing = Schema.Struct({
+ *   title: Schema.String,
+ *   photo: Decision.Image
+ * })
+ * ```
+ *
+ * @stability unstable
+ * @category schemas
+ * @since 4.1.0
+ */
+export const Image: Image = Schema.Struct({
+  [ImageKey]: Schema.Union([
+    Schema.Struct({ mediaType: Schema.String, base64: Schema.String }),
+    Schema.Struct({ mediaType: Schema.String, url: Schema.URLFromString })
+  ])
+}).pipe(
+  Schema.decodeTo(
+    Schema.declare(isImage, { expected: "DecisionModel.Image" }),
+    SchemaTransformation.transform({
+      decode: ({ [ImageKey]: image }): DecisionModel.Image => ({
+        mediaType: image.mediaType,
+        data: "base64" in image ? image.base64 : image.url
+      }),
+      encode: ({ mediaType, data }) => ({
+        [ImageKey]: data instanceof globalThis.URL
+          ? { mediaType, url: data }
+          : { mediaType, base64: typeof data === "string" ? data : Base64.encode(data) }
+      })
+    })
+  )
+)
+
+type PrevDepth = [never, 0, 1, 2, 3, 4]
+
+type ImagePathSuffixes<A, Depth extends number> = [Depth] extends [never] ? never
+  : A extends ImageEncoded ? ""
+  : A extends ReadonlyArray<unknown> ?
+    number extends A["length"] ? `[${number}]${ImagePathSuffixes<A[number], PrevDepth[Depth]>}`
+    : {
+      [K in keyof A & `${number}`]: `[${K}]${ImagePathSuffixes<A[K], PrevDepth[Depth]>}`
+    }[keyof A & `${number}`]
+  : A extends object ? {
+      [K in keyof A & string]-?: K extends `${string}.${string}` | `${string}[${string}` ? never
+        : {} extends Pick<A, K> ? never
+        : `.${K}${ImagePathSuffixes<A[K], PrevDepth[Depth]>}`
+    }[keyof A & string]
+  : never
+
+/**
+ * Paths to the required {@link Image} fields of an encoded input type, as
+ * accepted by {@link locate}. Object keys are joined with `.`, and array
+ * elements use `[index]`: exact indices for tuples, any number for arrays.
+ *
+ * **Details**
+ *
+ * Paths reach four levels deep. Optional fields and keys containing `.` or
+ * `[` are excluded.
+ *
+ * @stability unstable
+ * @category utility types
+ * @since 4.1.0
+ */
+export type ImagePaths<A> = ImagePathSuffixes<A, 4> extends infer Path extends string ?
+  Path extends "" ? never : Path extends `.${infer Rest}` ? Rest : Path :
+  never
+
 /**
  * Union of every decision kind.
  *
@@ -107,7 +267,7 @@ export interface Probability {
  * @category models
  * @since 4.0.0
  */
-export type Any = Classify<string> | Rate<string> | Probability
+export type Any = Classify<string> | Rate<string> | Probability | Locate
 
 /**
  * Answer to a {@link Classify} decision.
@@ -154,6 +314,38 @@ export interface ProbabilityAnswer {
   readonly probability: number
 }
 
+/**
+ * Smallest rectangle that contains a located target. Each edge is a fraction
+ * of the image's size measured from the top-left corner: `left` and `right`
+ * of its width, `top` and `bottom` of its height. Edges are in `[0, 1]`, with
+ * `left <= right` and `top <= bottom`.
+ *
+ * @stability unstable
+ * @category models
+ * @since 4.1.0
+ */
+export interface BoundingBox {
+  readonly left: number
+  readonly top: number
+  readonly right: number
+  readonly bottom: number
+}
+
+/**
+ * Answer to a {@link Locate} decision.
+ * There is no "not found" answer: a target absent from the image still gets a
+ * box. Pair the decision with a {@link Probability} that the target is present.
+ * `confidence` is an optional, provider-defined measure in `[0, 1]`.
+ *
+ * @stability unstable
+ * @category models
+ * @since 4.1.0
+ */
+export interface LocateAnswer {
+  readonly box: BoundingBox
+  readonly confidence?: number | undefined
+}
+
 /**
  * Answer type for a single decision.
  *
@@ -164,6 +356,7 @@ export interface ProbabilityAnswer {
 export type Answer<D extends Any> = D extends Classify<infer Label> ? ClassifyAnswer<Label>
   : D extends Rate<infer Level> ? RateAnswer<Level>
   : D extends Probability ? ProbabilityAnswer
+  : D extends Locate ? LocateAnswer
   : never
 
 /**
@@ -395,6 +588,58 @@ export const probability = (options: {
   criteria: options.criteria
 })
 
+/**
+ * Creates a decision that finds one target in an image field of the input.
+ * `image` is the field's path in the encoded input: keys joined with `.` and
+ * array indices in brackets, such as `"frames[2].frame"`. The model must be
+ * created with `supportsLocate: true`.
+ *
+ * **Details**
+ *
+ * Paths use encoded key names, so a key renamed with `Schema.encodeKeys` is
+ * targeted by its encoded name. `make` accepts only paths to required
+ * {@link Image} fields, and `DecisionModel.decide` fails with
+ * `AiError.InvalidUserInputError` when the path holds no image, such as an
+ * index past the end of an array.
+ *
+ * The answer is always one box, even when the target is absent. Pair the
+ * decision with a {@link probability} decision that the target is present.
+ *
+ * **Gotchas**
+ *
+ * - For a generic input schema, the compiler error for an invalid path names
+ *   the whole decision type rather than the path.
+ *
+ * **Example** (Finding a mug in a photo)
+ *
+ * ```ts
+ * import { Schema } from "effect"
+ * import { Decision } from "effect/ai"
+ *
+ * const FindMug = Decision.make({
+ *   input: Schema.Struct({ photo: Decision.Image }),
+ *   decisions: {
+ *     present: Decision.probability({ instructions: "`photo` shows a coffee mug" }),
+ *     mug: Decision.locate({ image: "photo", instructions: "the coffee mug" })
+ *   }
+ * })
+ * ```
+ *
+ * @see {@link LocateAnswer} for the answer
+ *
+ * @stability unstable
+ * @category constructors
+ * @since 4.1.0
+ */
+export const locate = <const Field extends string>(options: {
+  readonly image: Field
+  readonly instructions: string
+}): Locate<Field> => ({
+  _tag: "Locate",
+  image: options.image,
+  instructions: options.instructions
+})
+
 /**
  * Creates a decision definition from an input schema and named decisions.
  * `DecisionModel.decide` encodes the input with `Schema.toCodecJson` before
@@ -433,7 +678,13 @@ export const probability = (options: {
  * @category constructors
  * @since 4.0.0
  */
-export const make = <Input extends Schema.Constraint, Decisions extends Record<string, Any>>(options: {
+export const make = <
+  Input extends Schema.Constraint,
+  Decisions extends Record<
+    string,
+    Classify<string> | Rate<string> | Probability | Locate<ImagePaths<Input["Encoded"]>>
+  >
+>(options: {
   readonly input: Input
   readonly decisions: Decisions
 }): Definition<Input, Decisions> => {

package/src/ai/DecisionModel.ts

@@ -9,11 +9,13 @@
  */
 import * as Context from "../Context.ts"
 import * as Effect from "../Effect.ts"
+import * as Uuid from "../internal/uuid.ts"
 import * as Predicate from "../Predicate.ts"
+import * as Random from "../Random.ts"
 import * as Result from "../Result.ts"
 import * as Schema from "../Schema.ts"
 import * as AiError from "./AiError.ts"
-import type * as Decision from "./Decision.ts"
+import * as Decision from "./Decision.ts"
 import type * as Response from "./Response.ts"
 
 /**
@@ -71,14 +73,10 @@ export class DecisionUsage extends Schema.Class<DecisionUsage>(
  */
 export interface DecideOptions<Input extends Schema.Constraint> {
   readonly input: Input["Type"]
-  /**
-   * Images to include with the input.
-   */
-  readonly images?: ReadonlyArray<Image> | undefined
 }
 
 /**
- * An image sent to the provider with the encoded input.
+ * An image in the input, declared with a `Decision.Image` field.
  * `data` is a base64 string, a byte array, or a URL.
  *
  * @stability unstable
@@ -90,6 +88,24 @@ export interface Image {
   readonly data: string | Uint8Array | URL
 }
 
+/**
+ * An image extracted from the encoded input. `state` holds `reference` where
+ * the image was, and `path` is where it was first found. `data` is a base64
+ * string or a URL.
+ *
+ * @stability unstable
+ * @category models
+ * @since 4.1.0
+ */
+export interface ProviderImage {
+  readonly reference: string
+  readonly path: ReadonlyArray<string | number>
+  readonly image: {
+    readonly mediaType: string
+    readonly data: string | URL
+  }
+}
+
 /**
  * Provider-specific response metadata keyed by provider name. Provider
  * packages augment this interface.
@@ -155,16 +171,18 @@ export interface DecideSettledResponse<Decisions extends Record<string, Decision
  * Provider input options for a decision request.
  * `state` is encoded with `Schema.toCodecJson`, not stringified.
  * All `decisions` must be answered in one call.
- * `images` is omitted when no images are supplied.
+ * `images` holds the `Decision.Image` values found in the input and is omitted
+ * when there are none.
+ * `Decisions` is the union of decision kinds the provider receives.
  *
  * @stability unstable
  * @category options
  * @since 4.0.0
  */
-export interface ProviderOptions {
+export interface ProviderOptions<Decisions extends Decision.Any = Decision.Any> {
   readonly state: Schema.Json
-  readonly decisions: Record<string, Decision.Any>
-  readonly images?: ReadonlyArray<Image> | undefined
+  readonly decisions: Record<string, Decisions>
+  readonly images?: ReadonlyArray<ProviderImage> | undefined
 }
 
 /**
@@ -219,6 +237,19 @@ export interface ProviderRefusalAnswer {
   readonly _tag: "Refusal"
 }
 
+/**
+ * Provider answer for a locate decision.
+ *
+ * @stability unstable
+ * @category models
+ * @since 4.1.0
+ */
+export interface ProviderLocateAnswer {
+  readonly _tag: "Locate"
+  readonly box: Decision.BoundingBox
+  readonly confidence?: number | undefined
+}
+
 /**
  * Provider answer for any decision kind.
  *
@@ -231,6 +262,7 @@ export type ProviderAnswer =
   | ProviderRateAnswer
   | ProviderProbabilityAnswer
   | ProviderRefusalAnswer
+  | ProviderLocateAnswer
 
 /**
  * Provider response for a decision request.
@@ -283,6 +315,70 @@ const invalid = (description: string) => unanswered("Invalid", description)
 const aiError = (method: "decide" | "decideSettled", reason: AiError.AiErrorReason): AiError.AiError =>
   AiError.make({ module: "DecisionModel", method, reason })
 
+const invalidUserInput = (method: "decide" | "decideSettled", description: string): AiError.AiError =>
+  aiError(method, new AiError.InvalidUserInputError({ description }))
+
+const toProviderImage = (value: unknown): ProviderImage["image"] | undefined => {
+  if (!Predicate.isObject(value) || Object.keys(value).length !== 1) return undefined
+  const image = value[Decision.ImageKey]
+  if (!Predicate.isObject(image) || Object.keys(image).length !== 2 || !Predicate.isString(image.mediaType)) {
+    return undefined
+  }
+  if (Predicate.isString(image.base64)) return { mediaType: image.mediaType, data: image.base64 }
+  if (Predicate.isString(image.url) && URL.canParse(image.url)) {
+    return { mediaType: image.mediaType, data: new URL(image.url) }
+  }
+  return undefined
+}
+
+const formatPath = (path: ReadonlyArray<string | number>): string =>
+  path.map((segment, index) => typeof segment === "number" ? `[${segment}]` : index === 0 ? segment : `.${segment}`)
+    .join("")
+
+const randomUuid = (random: Random.Random): string => {
+  const bytes = new Uint8Array(16)
+  for (let index = 0; index < bytes.length; index++) {
+    bytes[index] = Math.floor(random.nextDoubleUnsafe() * 256)
+  }
+  return Uuid.v4String(bytes)
+}
+
+/**
+ * Replaces each encoded `Decision.Image` in `state` with a reference. Equal
+ * images share one reference. `paths` holds every path where an image was found.
+ */
+const extractImages = (state: Schema.Json, random: Random.Random) => {
+  const images: Array<ProviderImage> = []
+  const references = new Map<string, string>()
+  const paths = new Set<string>()
+  const walk = (value: Schema.Json, path: ReadonlyArray<string | number>): Schema.Json => {
+    const image = toProviderImage(value)
+    if (image !== undefined) {
+      paths.add(formatPath(path))
+      const key = `${image.mediaType}\n${
+        typeof image.data === "string" ? `base64:${image.data}` : `url:${image.data.href}`
+      }`
+      let reference = references.get(key)
+      if (reference === undefined) {
+        reference = randomUuid(random)
+        references.set(key, reference)
+        images.push({ reference, path, image })
+      }
+      return reference
+    }
+    if (Array.isArray(value)) {
+      return value.map((item, index) => walk(item, [...path, index]))
+    }
+    if (Predicate.isObject(value)) {
+      return Object.fromEntries(
+        Object.entries(value).map(([key, item]) => [key, walk(item as Schema.Json, [...path, key])])
+      )
+    }
+    return value
+  }
+  return { state: walk(state, []), images, paths }
+}
+
 const isFiniteNumber = (value: unknown): value is number => Predicate.isNumber(value) && Number.isFinite(value)
 
 const isUnitInterval = (value: unknown): value is number => isFiniteNumber(value) && value >= 0 && value <= 1
@@ -390,6 +486,23 @@ const validateAnswer = (
       }
       return Result.succeed({ probability: answer.probability })
     }
+    case "Locate": {
+      const box = answer.box
+      if (
+        !Predicate.isObject(box) || !isUnitInterval(box.left) || !isUnitInterval(box.top) ||
+        !isUnitInterval(box.right) || !isUnitInterval(box.bottom)
+      ) {
+        return invalid(`Provider returned a box with edges outside [0, 1] for decision "${key}"`)
+      }
+      if (box.left > box.right || box.top > box.bottom) {
+        return invalid(`Provider returned a box with crossed edges for decision "${key}"`)
+      }
+      const bounds = { left: box.left, top: box.top, right: box.right, bottom: box.bottom }
+      return Result.map(validateConfidence(key, answer), (confidence) => ({
+        box: bounds,
+        ...(confidence === undefined ? undefined : { confidence })
+      }))
+    }
   }
 }
 
@@ -436,16 +549,24 @@ const requireAnswers = <Decisions extends Record<string, Decision.Any>>(
  * Creates a `DecisionModel` that encodes inputs as JSON and validates provider answers.
  * Answers must cover every decision and use its labels. Distributions must
  * sum to 1 within `1e-6`, optional confidence must be in `[0, 1]`, and ratings must be
- * in `[0, criteria.length - 1]`. With `decide`, refused decisions fail with
+ * in `[0, criteria.length - 1]`, and boxes must have edges in `[0, 1]` that do
+ * not cross. With `decide`, refused decisions fail with
  * `AiError.ContentPolicyError` and other unanswered decisions fail with
  * `AiError.InvalidOutputError`; `decideSettled` reports them per decision
  * instead. Encoding failures use `AiError.InvalidUserInputError`.
  *
  * **Details**
  *
- * Set `supportsImages: true` to accept images. Otherwise, nonempty image input
+ * After encoding, each `Decision.Image` in the input is replaced with a
+ * reference string, and the provider receives the images in `images`. Set
+ * `supportsImages: true` to accept images. Otherwise, an input with images
  * fails with `AiError.InvalidUserInputError`.
  *
+ * Set `supportsLocate: true` to accept locate decisions; only then does
+ * `decide` receive them. Otherwise, a definition with a locate decision fails
+ * with `AiError.InvalidUserInputError`, as does a locate decision whose path
+ * holds no image.
+ *
  * Providers that round each probability to `probabilityPrecision` decimal
  * places may return distributions whose sum drifts from 1 by up to half a unit
  * of the last place per label. Setting `probabilityPrecision` accepts that
@@ -459,11 +580,21 @@ const requireAnswers = <Decisions extends Record<string, Decision.Any>>(
  * @category constructors
  * @since 4.0.0
  */
-export const make = (params: {
-  readonly decide: (options: ProviderOptions) => Effect.Effect<ProviderResponse, AiError.AiError>
-  readonly probabilityPrecision?: number | undefined
-  readonly supportsImages?: boolean | undefined
-}): Effect.Effect<DecisionModel> =>
+export const make = (
+  params: {
+    readonly decide: (
+      options: ProviderOptions<Exclude<Decision.Any, Decision.Locate>>
+    ) => Effect.Effect<ProviderResponse, AiError.AiError>
+    readonly probabilityPrecision?: number | undefined
+    readonly supportsImages?: boolean | undefined
+    readonly supportsLocate?: false | undefined
+  } | {
+    readonly decide: (options: ProviderOptions) => Effect.Effect<ProviderResponse, AiError.AiError>
+    readonly probabilityPrecision?: number | undefined
+    readonly supportsImages?: boolean | undefined
+    readonly supportsLocate: true
+  }
+): Effect.Effect<DecisionModel> =>
   Effect.sync(() => {
     const roundingError = params.probabilityPrecision === undefined ? 0 : 0.5 * 10 ** -params.probabilityPrecision
     const settle = <Input extends Schema.Constraint, Decisions extends Record<string, Decision.Any>>(
@@ -472,21 +603,33 @@ export const make = (params: {
       options: DecideOptions<Input>
     ): Effect.Effect<DecideSettledResponse<Decisions>, AiError.AiError, Input["EncodingServices"]> =>
       Schema.encodeEffect(Schema.toCodecJson(definition.input))(options.input).pipe(
-        Effect.mapError((error) => aiError(method, new AiError.InvalidUserInputError({ description: error.message }))),
-        Effect.flatMap((state) => {
-          const images = options.images?.length ? options.images : undefined
-          if (images !== undefined && params.supportsImages !== true) {
-            return Effect.fail(
-              aiError(
-                method,
-                new AiError.InvalidUserInputError({ description: "This decision model does not accept images" })
+        Effect.mapError((error) => invalidUserInput(method, error.message)),
+        Effect.flatMap((encoded) =>
+          Effect.withFiber((fiber) => Effect.succeed(extractImages(encoded, fiber.getRef(Random.Random))))
+        ),
+        Effect.flatMap(({ state, images, paths }) => {
+          if (images.length > 0 && params.supportsImages !== true) {
+            return Effect.fail(invalidUserInput(method, "This decision model does not accept images"))
+          }
+          for (const [key, decision] of Object.entries(definition.decisions)) {
+            if (decision._tag !== "Locate") continue
+            if (params.supportsLocate !== true) {
+              return Effect.fail(invalidUserInput(method, "This decision model does not support locate decisions"))
+            }
+            if (!paths.has(decision.image)) {
+              return Effect.fail(
+                invalidUserInput(method, `Locate decision "${key}" targets "${decision.image}", which holds no image`)
               )
-            )
+            }
           }
-          return params.decide({
+          // Locate decisions were rejected above unless `supportsLocate` is true.
+          const decide = params.decide as (
+            options: ProviderOptions
+          ) => Effect.Effect<ProviderResponse, AiError.AiError>
+          return decide({
             state,
             decisions: definition.decisions,
-            ...(images === undefined ? undefined : { images })
+            ...(images.length === 0 ? undefined : { images })
           })
         }),
         Effect.map((response): DecideSettledResponse<Decisions> => ({