effect 4.0.6 → 4.1.0

package/src/ai/DecisionModel.ts

@@ -9,11 +9,13 @@
  */
 import * as Context from "../Context.ts"
 import * as Effect from "../Effect.ts"
+import * as Uuid from "../internal/uuid.ts"
 import * as Predicate from "../Predicate.ts"
+import * as Random from "../Random.ts"
 import * as Result from "../Result.ts"
 import * as Schema from "../Schema.ts"
 import * as AiError from "./AiError.ts"
-import type * as Decision from "./Decision.ts"
+import * as Decision from "./Decision.ts"
 import type * as Response from "./Response.ts"
 
 /**
@@ -71,14 +73,10 @@ export class DecisionUsage extends Schema.Class<DecisionUsage>(
  */
 export interface DecideOptions<Input extends Schema.Constraint> {
   readonly input: Input["Type"]
-  /**
-   * Images to include with the input.
-   */
-  readonly images?: ReadonlyArray<Image> | undefined
 }
 
 /**
- * An image sent to the provider with the encoded input.
+ * An image in the input, declared with a `Decision.Image` field.
  * `data` is a base64 string, a byte array, or a URL.
  *
  * @stability unstable
@@ -90,6 +88,24 @@ export interface Image {
   readonly data: string | Uint8Array | URL
 }
 
+/**
+ * An image extracted from the encoded input. `state` holds `reference` where
+ * the image was, and `path` is where it was first found. `data` is a base64
+ * string or a URL.
+ *
+ * @stability unstable
+ * @category models
+ * @since 4.1.0
+ */
+export interface ProviderImage {
+  readonly reference: string
+  readonly path: ReadonlyArray<string | number>
+  readonly image: {
+    readonly mediaType: string
+    readonly data: string | URL
+  }
+}
+
 /**
  * Provider-specific response metadata keyed by provider name. Provider
  * packages augment this interface.
@@ -155,16 +171,18 @@ export interface DecideSettledResponse<Decisions extends Record<string, Decision
  * Provider input options for a decision request.
  * `state` is encoded with `Schema.toCodecJson`, not stringified.
  * All `decisions` must be answered in one call.
- * `images` is omitted when no images are supplied.
+ * `images` holds the `Decision.Image` values found in the input and is omitted
+ * when there are none.
+ * `Decisions` is the union of decision kinds the provider receives.
  *
  * @stability unstable
  * @category options
  * @since 4.0.0
  */
-export interface ProviderOptions {
+export interface ProviderOptions<Decisions extends Decision.Any = Decision.Any> {
   readonly state: Schema.Json
-  readonly decisions: Record<string, Decision.Any>
-  readonly images?: ReadonlyArray<Image> | undefined
+  readonly decisions: Record<string, Decisions>
+  readonly images?: ReadonlyArray<ProviderImage> | undefined
 }
 
 /**
@@ -219,6 +237,19 @@ export interface ProviderRefusalAnswer {
   readonly _tag: "Refusal"
 }
 
+/**
+ * Provider answer for a locate decision.
+ *
+ * @stability unstable
+ * @category models
+ * @since 4.1.0
+ */
+export interface ProviderLocateAnswer {
+  readonly _tag: "Locate"
+  readonly box: Decision.BoundingBox
+  readonly confidence?: number | undefined
+}
+
 /**
  * Provider answer for any decision kind.
  *
@@ -231,6 +262,7 @@ export type ProviderAnswer =
   | ProviderRateAnswer
   | ProviderProbabilityAnswer
   | ProviderRefusalAnswer
+  | ProviderLocateAnswer
 
 /**
  * Provider response for a decision request.
@@ -283,6 +315,70 @@ const invalid = (description: string) => unanswered("Invalid", description)
 const aiError = (method: "decide" | "decideSettled", reason: AiError.AiErrorReason): AiError.AiError =>
   AiError.make({ module: "DecisionModel", method, reason })
 
+const invalidUserInput = (method: "decide" | "decideSettled", description: string): AiError.AiError =>
+  aiError(method, new AiError.InvalidUserInputError({ description }))
+
+const toProviderImage = (value: unknown): ProviderImage["image"] | undefined => {
+  if (!Predicate.isObject(value) || Object.keys(value).length !== 1) return undefined
+  const image = value[Decision.ImageKey]
+  if (!Predicate.isObject(image) || Object.keys(image).length !== 2 || !Predicate.isString(image.mediaType)) {
+    return undefined
+  }
+  if (Predicate.isString(image.base64)) return { mediaType: image.mediaType, data: image.base64 }
+  if (Predicate.isString(image.url) && URL.canParse(image.url)) {
+    return { mediaType: image.mediaType, data: new URL(image.url) }
+  }
+  return undefined
+}
+
+const formatPath = (path: ReadonlyArray<string | number>): string =>
+  path.map((segment, index) => typeof segment === "number" ? `[${segment}]` : index === 0 ? segment : `.${segment}`)
+    .join("")
+
+const randomUuid = (random: Random.Random): string => {
+  const bytes = new Uint8Array(16)
+  for (let index = 0; index < bytes.length; index++) {
+    bytes[index] = Math.floor(random.nextDoubleUnsafe() * 256)
+  }
+  return Uuid.v4String(bytes)
+}
+
+/**
+ * Replaces each encoded `Decision.Image` in `state` with a reference. Equal
+ * images share one reference. `paths` holds every path where an image was found.
+ */
+const extractImages = (state: Schema.Json, random: Random.Random) => {
+  const images: Array<ProviderImage> = []
+  const references = new Map<string, string>()
+  const paths = new Set<string>()
+  const walk = (value: Schema.Json, path: ReadonlyArray<string | number>): Schema.Json => {
+    const image = toProviderImage(value)
+    if (image !== undefined) {
+      paths.add(formatPath(path))
+      const key = `${image.mediaType}\n${
+        typeof image.data === "string" ? `base64:${image.data}` : `url:${image.data.href}`
+      }`
+      let reference = references.get(key)
+      if (reference === undefined) {
+        reference = randomUuid(random)
+        references.set(key, reference)
+        images.push({ reference, path, image })
+      }
+      return reference
+    }
+    if (Array.isArray(value)) {
+      return value.map((item, index) => walk(item, [...path, index]))
+    }
+    if (Predicate.isObject(value)) {
+      return Object.fromEntries(
+        Object.entries(value).map(([key, item]) => [key, walk(item as Schema.Json, [...path, key])])
+      )
+    }
+    return value
+  }
+  return { state: walk(state, []), images, paths }
+}
+
 const isFiniteNumber = (value: unknown): value is number => Predicate.isNumber(value) && Number.isFinite(value)
 
 const isUnitInterval = (value: unknown): value is number => isFiniteNumber(value) && value >= 0 && value <= 1
@@ -390,6 +486,23 @@ const validateAnswer = (
       }
       return Result.succeed({ probability: answer.probability })
     }
+    case "Locate": {
+      const box = answer.box
+      if (
+        !Predicate.isObject(box) || !isUnitInterval(box.left) || !isUnitInterval(box.top) ||
+        !isUnitInterval(box.right) || !isUnitInterval(box.bottom)
+      ) {
+        return invalid(`Provider returned a box with edges outside [0, 1] for decision "${key}"`)
+      }
+      if (box.left > box.right || box.top > box.bottom) {
+        return invalid(`Provider returned a box with crossed edges for decision "${key}"`)
+      }
+      const bounds = { left: box.left, top: box.top, right: box.right, bottom: box.bottom }
+      return Result.map(validateConfidence(key, answer), (confidence) => ({
+        box: bounds,
+        ...(confidence === undefined ? undefined : { confidence })
+      }))
+    }
   }
 }
 
@@ -436,16 +549,24 @@ const requireAnswers = <Decisions extends Record<string, Decision.Any>>(
  * Creates a `DecisionModel` that encodes inputs as JSON and validates provider answers.
  * Answers must cover every decision and use its labels. Distributions must
  * sum to 1 within `1e-6`, optional confidence must be in `[0, 1]`, and ratings must be
- * in `[0, criteria.length - 1]`. With `decide`, refused decisions fail with
+ * in `[0, criteria.length - 1]`, and boxes must have edges in `[0, 1]` that do
+ * not cross. With `decide`, refused decisions fail with
  * `AiError.ContentPolicyError` and other unanswered decisions fail with
  * `AiError.InvalidOutputError`; `decideSettled` reports them per decision
  * instead. Encoding failures use `AiError.InvalidUserInputError`.
  *
  * **Details**
  *
- * Set `supportsImages: true` to accept images. Otherwise, nonempty image input
+ * After encoding, each `Decision.Image` in the input is replaced with a
+ * reference string, and the provider receives the images in `images`. Set
+ * `supportsImages: true` to accept images. Otherwise, an input with images
  * fails with `AiError.InvalidUserInputError`.
  *
+ * Set `supportsLocate: true` to accept locate decisions; only then does
+ * `decide` receive them. Otherwise, a definition with a locate decision fails
+ * with `AiError.InvalidUserInputError`, as does a locate decision whose path
+ * holds no image.
+ *
  * Providers that round each probability to `probabilityPrecision` decimal
  * places may return distributions whose sum drifts from 1 by up to half a unit
  * of the last place per label. Setting `probabilityPrecision` accepts that
@@ -459,11 +580,21 @@ const requireAnswers = <Decisions extends Record<string, Decision.Any>>(
  * @category constructors
  * @since 4.0.0
  */
-export const make = (params: {
-  readonly decide: (options: ProviderOptions) => Effect.Effect<ProviderResponse, AiError.AiError>
-  readonly probabilityPrecision?: number | undefined
-  readonly supportsImages?: boolean | undefined
-}): Effect.Effect<DecisionModel> =>
+export const make = (
+  params: {
+    readonly decide: (
+      options: ProviderOptions<Exclude<Decision.Any, Decision.Locate>>
+    ) => Effect.Effect<ProviderResponse, AiError.AiError>
+    readonly probabilityPrecision?: number | undefined
+    readonly supportsImages?: boolean | undefined
+    readonly supportsLocate?: false | undefined
+  } | {
+    readonly decide: (options: ProviderOptions) => Effect.Effect<ProviderResponse, AiError.AiError>
+    readonly probabilityPrecision?: number | undefined
+    readonly supportsImages?: boolean | undefined
+    readonly supportsLocate: true
+  }
+): Effect.Effect<DecisionModel> =>
   Effect.sync(() => {
     const roundingError = params.probabilityPrecision === undefined ? 0 : 0.5 * 10 ** -params.probabilityPrecision
     const settle = <Input extends Schema.Constraint, Decisions extends Record<string, Decision.Any>>(
@@ -472,21 +603,33 @@ export const make = (params: {
       options: DecideOptions<Input>
     ): Effect.Effect<DecideSettledResponse<Decisions>, AiError.AiError, Input["EncodingServices"]> =>
       Schema.encodeEffect(Schema.toCodecJson(definition.input))(options.input).pipe(
-        Effect.mapError((error) => aiError(method, new AiError.InvalidUserInputError({ description: error.message }))),
-        Effect.flatMap((state) => {
-          const images = options.images?.length ? options.images : undefined
-          if (images !== undefined && params.supportsImages !== true) {
-            return Effect.fail(
-              aiError(
-                method,
-                new AiError.InvalidUserInputError({ description: "This decision model does not accept images" })
+        Effect.mapError((error) => invalidUserInput(method, error.message)),
+        Effect.flatMap((encoded) =>
+          Effect.withFiber((fiber) => Effect.succeed(extractImages(encoded, fiber.getRef(Random.Random))))
+        ),
+        Effect.flatMap(({ state, images, paths }) => {
+          if (images.length > 0 && params.supportsImages !== true) {
+            return Effect.fail(invalidUserInput(method, "This decision model does not accept images"))
+          }
+          for (const [key, decision] of Object.entries(definition.decisions)) {
+            if (decision._tag !== "Locate") continue
+            if (params.supportsLocate !== true) {
+              return Effect.fail(invalidUserInput(method, "This decision model does not support locate decisions"))
+            }
+            if (!paths.has(decision.image)) {
+              return Effect.fail(
+                invalidUserInput(method, `Locate decision "${key}" targets "${decision.image}", which holds no image`)
               )
-            )
+            }
           }
-          return params.decide({
+          // Locate decisions were rejected above unless `supportsLocate` is true.
+          const decide = params.decide as (
+            options: ProviderOptions
+          ) => Effect.Effect<ProviderResponse, AiError.AiError>
+          return decide({
             state,
             decisions: definition.decisions,
-            ...(images === undefined ? undefined : { images })
+            ...(images.length === 0 ? undefined : { images })
           })
         }),
         Effect.map((response): DecideSettledResponse<Decisions> => ({

package/src/ai/LanguageModel.ts

@@ -1553,6 +1553,7 @@ export const make: (params: {
             preliminary: false,
             result: r.encodedResult,
             encodedResult: r.encodedResult,
+            ...modelOutputField(r.modelOutput),
             isFailure: r.isFailure
           }) as Response.StreamPart<Tools>
         )
@@ -1680,6 +1681,7 @@ export const make: (params: {
             providerExecuted: false,
             result: result.result,
             encodedResult: result.encodedResult,
+            ...modelOutputField(result.modelOutput),
             isFailure: result.isFailure,
             preliminary: result.preliminary
           }) as Response.StreamPart<Tools>
@@ -2268,6 +2270,7 @@ const executeApprovedToolCalls = <Tools extends Record<string, Tool.Any>>(
       name: toolCall.name,
       result: terminalResult.result,
       encodedResult: terminalResult.encodedResult,
+      ...modelOutputField(terminalResult.modelOutput),
       isFailure: terminalResult.isFailure,
       preliminary: terminalResult.preliminary,
       providerExecuted: false
@@ -2347,6 +2350,10 @@ const makeInterruptedToolResult = (
   })
 }
 
+// Omits the key entirely so unmapped tool result parts keep their shape.
+const modelOutputField = (modelOutput: ReadonlyArray<Prompt.ToolModelOutputPart> | undefined) =>
+  modelOutput === undefined ? undefined : { modelOutput }
+
 type ToolResolutionResult<Tools extends Record<string, Tool.Any>> =
   | Response.ToolResultPart<
     Tool.Name<Tools[keyof Tools]>,
@@ -2417,6 +2424,7 @@ const resolveToolCalls = <Tools extends Record<string, Tool.Any>>(
                 providerExecuted: false,
                 result: result.result,
                 encodedResult: result.encodedResult,
+                ...modelOutputField(result.modelOutput),
                 isFailure: result.isFailure,
                 preliminary: result.preliminary
               }) as ToolResolutionResult<Tools>
@@ -2446,6 +2454,7 @@ const resolveToolCalls = <Tools extends Record<string, Tool.Any>>(
               providerExecuted: false,
               result: result.result,
               encodedResult: result.encodedResult,
+              ...modelOutputField(result.modelOutput),
               isFailure: result.isFailure,
               preliminary: result.preliminary
             }) as ToolResolutionResult<Tools>