@ai-sdk/gateway 4.0.84 → 4.0.85

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ai-sdk/gateway",
   "private": false,
-  "version": "4.0.84",
+  "version": "4.0.85",
   "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,

package/src/gateway-evaluation-model-settings.ts

@@ -0,0 +1 @@
+export type GatewayEvaluationModelId = 'typesafe-ai/jev-latest' | (string & {});

package/src/gateway-evaluation-model.ts

@@ -0,0 +1,173 @@
+import type {
+  Experimental_EvaluationModelV4 as EvaluationModelV4,
+  SharedV4ProviderMetadata,
+} from '@ai-sdk/provider';
+import {
+  combineHeaders,
+  createJsonErrorResponseHandler,
+  createJsonResponseHandler,
+  getErrorMessage,
+  lazySchema,
+  postJsonToApi,
+  resolve,
+  zodSchema,
+  type Resolvable,
+} from '@ai-sdk/provider-utils';
+import { z } from './zod';
+import { asGatewayError } from './errors';
+import { parseAuthMethod } from './errors/parse-auth-method';
+import type { GatewayConfig } from './gateway-config';
+
+export class GatewayEvaluationModel implements EvaluationModelV4 {
+  readonly specificationVersion = 'v4';
+
+  readonly supportedQuestionTypes = ['choice', 'score', 'boolean'] as const;
+
+  constructor(
+    readonly modelId: string,
+    private readonly config: GatewayConfig & {
+      provider: string;
+      o11yHeaders: Resolvable<Record<string, string>>;
+    },
+  ) {}
+
+  get provider(): string {
+    return this.config.provider;
+  }
+
+  async doEvaluate({
+    state,
+    questions,
+    headers,
+    abortSignal,
+    providerOptions,
+  }: Parameters<EvaluationModelV4['doEvaluate']>[0]): Promise<
+    Awaited<ReturnType<EvaluationModelV4['doEvaluate']>>
+  > {
+    const resolvedHeaders = this.config.headers
+      ? await resolve(this.config.headers)
+      : undefined;
+    try {
+      const {
+        responseHeaders,
+        value: responseBody,
+        rawValue,
+      } = await postJsonToApi({
+        url: this.getUrl(),
+        headers: combineHeaders(
+          resolvedHeaders,
+          headers ?? {},
+          this.getModelConfigHeaders(),
+          await resolve(this.config.o11yHeaders),
+        ),
+        body: {
+          state,
+          questions,
+          ...(providerOptions ? { providerOptions } : {}),
+        },
+        successfulResponseHandler: createJsonResponseHandler(
+          gatewayEvaluationResponseSchema,
+        ),
+        failedResponseHandler: createJsonErrorResponseHandler({
+          errorSchema: z.any(),
+          errorToMessage: data => getErrorMessage(data) ?? 'unknown error',
+        }),
+        ...(abortSignal && { abortSignal }),
+        fetch: this.config.fetch,
+      });
+
+      return {
+        answers: responseBody.answers,
+        ...(responseBody.rounding ? { rounding: responseBody.rounding } : {}),
+        ...(responseBody.usage ? { usage: responseBody.usage } : {}),
+        warnings: responseBody.warnings ?? [],
+        providerMetadata:
+          responseBody.providerMetadata as unknown as SharedV4ProviderMetadata,
+        response: {
+          modelId: this.modelId,
+          headers: responseHeaders,
+          body: rawValue,
+        },
+      };
+    } catch (error) {
+      throw await asGatewayError(
+        error,
+        await parseAuthMethod(resolvedHeaders ?? {}),
+      );
+    }
+  }
+
+  private getUrl() {
+    return `${this.config.baseURL}/evaluation-model`;
+  }
+
+  private getModelConfigHeaders() {
+    return {
+      'ai-evaluation-model-specification-version': '4',
+      'ai-model-id': this.modelId,
+    };
+  }
+}
+
+const gatewayEvaluationAnswerSchema = z.discriminatedUnion('type', [
+  z.object({
+    type: z.literal('choice'),
+    choice: z.string(),
+    probabilities: z.record(z.string(), z.number()).optional(),
+  }),
+  z.object({
+    type: z.literal('score'),
+    score: z.number(),
+    probabilities: z.record(z.string(), z.number()).optional(),
+  }),
+  z.object({
+    type: z.literal('boolean'),
+    probability: z.number(),
+  }),
+]);
+
+const gatewayEvaluationWarningSchema = z.discriminatedUnion('type', [
+  z.object({
+    type: z.literal('unsupported'),
+    feature: z.string(),
+    details: z.string().optional(),
+  }),
+  z.object({
+    type: z.literal('compatibility'),
+    feature: z.string(),
+    details: z.string().optional(),
+  }),
+  z.object({
+    type: z.literal('deprecated'),
+    setting: z.string(),
+    message: z.string(),
+  }),
+  z.object({
+    type: z.literal('other'),
+    message: z.string(),
+  }),
+]);
+
+const gatewayEvaluationResponseSchema = lazySchema(() =>
+  zodSchema(
+    z.object({
+      answers: z.record(z.string(), gatewayEvaluationAnswerSchema),
+      rounding: z
+        .object({
+          probabilityDecimals: z.number().optional(),
+          scoreDecimals: z.number().optional(),
+        })
+        .optional(),
+      usage: z
+        .object({
+          inputTokens: z.number().optional(),
+          outputTokens: z.number().optional(),
+        })
+        .optional(),
+      warnings: z.array(gatewayEvaluationWarningSchema).optional(),
+      providerMetadata: z
+        .record(z.string(), z.record(z.string(), z.unknown()))
+        .optional(),
+    }),
+  ),
+);

package/src/gateway-model-entry.ts

@@ -2,6 +2,7 @@ import type { LanguageModelV4 } from '@ai-sdk/provider';
 
 export const KNOWN_MODEL_TYPES = [
   'embedding',
+  'evaluation',
   'image',
   'language',
   'realtime',

package/src/gateway-provider.ts

@@ -36,6 +36,7 @@ import { GatewayLanguageModel } from './gateway-language-model';
 import { GatewayEmbeddingModel } from './gateway-embedding-model';
 import { GatewayImageModel } from './gateway-image-model';
 import { GatewayVideoModel } from './gateway-video-model';
+import { GatewayEvaluationModel } from './gateway-evaluation-model';
 import { GatewayRerankingModel } from './gateway-reranking-model';
 import { GatewaySpeechModel } from './gateway-speech-model';
 import {
@@ -44,6 +45,7 @@ import {
 } from './gateway-transcription-model';
 import { GatewayRealtimeModel } from './gateway-realtime-model';
 import type { GatewayEmbeddingModelId } from './gateway-embedding-model-settings';
+import type { GatewayEvaluationModelId } from './gateway-evaluation-model-settings';
 import type { GatewayImageModelId } from './gateway-image-model-settings';
 import type { GatewayRerankingModelId } from './gateway-reranking-model-settings';
 import type { GatewaySpeechModelId } from './gateway-speech-model-settings';
@@ -61,6 +63,7 @@ import type {
   SpeechModelV4,
   TranscriptionModelV4,
   Experimental_VideoModelV4,
+  Experimental_EvaluationModelV4,
   Experimental_RealtimeFactoryV4 as RealtimeFactoryV4,
   Experimental_RealtimeFactoryV4GetTokenOptions as RealtimeFactoryV4GetTokenOptions,
   ProviderV4,
@@ -155,6 +158,18 @@ export interface GatewayProvider extends ProviderV4 {
    */
   rerankingModel(modelId: GatewayRerankingModelId): RerankingModelV4;
 
+  /**
+   * Creates a model for evaluating state against questions.
+   */
+  evaluation(modelId: GatewayEvaluationModelId): Experimental_EvaluationModelV4;
+
+  /**
+   * Creates a model for evaluating state against questions.
+   */
+  evaluationModel(
+    modelId: GatewayEvaluationModelId,
+  ): Experimental_EvaluationModelV4;
+
   /**
    * Creates a model for text-to-speech generation.
    */
@@ -568,6 +583,17 @@ export function createGateway(
   };
   provider.rerankingModel = createRerankingModel;
   provider.reranking = createRerankingModel;
+  const createEvaluationModel = (modelId: GatewayEvaluationModelId) => {
+    return new GatewayEvaluationModel(modelId, {
+      provider: 'gateway',
+      baseURL,
+      headers: getHeaders,
+      fetch: options.fetch,
+      o11yHeaders: createO11yHeaders(),
+    });
+  };
+  provider.evaluationModel = createEvaluationModel;
+  provider.evaluation = createEvaluationModel;
   const createSpeechModel = (modelId: GatewaySpeechModelId) => {
     return new GatewaySpeechModel(modelId, {
       provider: 'gateway',

package/src/index.ts

@@ -1,4 +1,5 @@
 export type { GatewayEmbeddingModelId } from './gateway-embedding-model-settings';
+export type { GatewayEvaluationModelId } from './gateway-evaluation-model-settings';
 export type { GatewayImageModelId } from './gateway-image-model-settings';
 export type { GatewayModelId } from './gateway-language-model-settings';
 export {