@automate.ax/api-contract 0.157.0 → 0.158.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. package/dist/deployment.d.ts +2 -2
  2. package/dist/index.d.ts +75 -7
  3. package/dist/index.js +1 -1
  4. package/dist/runtime-protocol.d.ts +1 -1
  5. package/dist/runtime-protocol.js +1 -1
  6. package/dist/runtime.d.ts +145 -2
  7. package/dist/runtime.js +92 -0
  8. package/package.json +5 -5
  9. package/src/index.ts +10 -1
  10. package/src/runtime-protocol.ts +1 -1
  11. package/src/runtime.ts +114 -2
package/dist/deployment.d.ts CHANGED
@@ -402,7 +402,7 @@ export declare const deploymentContract: {
402
402
  UNSUPPORTED_RUNTIME_PROTOCOL: {
403
403
  readonly data: z.ZodObject<{
404
404
  receivedVersion: z.ZodNullable<z.ZodNumber>;
405
- supportedVersion: z.ZodLiteral<6>;
405
+ supportedVersion: z.ZodLiteral<7>;
406
406
  }, z.core.$strip>;
407
407
  readonly message: "The automation bundle uses an unsupported runtime protocol.";
408
408
  readonly status: 400;
@@ -1141,7 +1141,7 @@ export declare const deploymentContract: {
1141
1141
  UNSUPPORTED_RUNTIME_PROTOCOL: {
1142
1142
  readonly data: z.ZodObject<{
1143
1143
  receivedVersion: z.ZodNullable<z.ZodNumber>;
1144
- supportedVersion: z.ZodLiteral<6>;
1144
+ supportedVersion: z.ZodLiteral<7>;
1145
1145
  }, z.core.$strip>;
1146
1146
  readonly message: "The automation bundle uses an unsupported runtime protocol.";
1147
1147
  readonly status: 400;
package/dist/index.d.ts CHANGED
@@ -4,8 +4,8 @@ export { AUTOMATION_SDK_VERSION, deploymentOutputSchema, deploymentStatusSchema,
4
4
  export { AUTOMATION_RUNTIME_PROTOCOL_VERSION } from "./runtime-protocol.js";
5
5
  export { AUTOMATION_ERROR_CAUSE_DEPTH, automationErrorSchema, type AutomationErrorValue, } from "./errors.js";
6
6
  export { integrationAccountOutputSchema, integrationAccountRevocationImpactOutputSchema, integrationAccountRevocationOutputSchema, integrationServiceOutputSchema, } from "./account.js";
7
- export { AUTOMATION_INVOCATION_DEFAULT_ENTRYPOINT, AUTOMATION_INVOCATION_RESERVED_ENTRYPOINT_PREFIX, AUTOMATION_IMAGE_GENERATION_ENTRYPOINT_PREFIX, AUTOMATION_LOG_MAX_ENTRIES, AUTOMATION_LOG_MAX_MESSAGE_LENGTH, AUTOMATION_OBSERVABILITY_MAX_ENCODED_BYTES, AUTOMATION_OBSERVABILITY_MAX_FIELDS, DEFAULT_GATEWAY_IMAGE_MODEL_ID, DEFAULT_GATEWAY_MODEL_ID, DEFAULT_ACTION_MAX_ATTEMPTS, MAX_ACTION_ATTEMPTS, actionReplaySafetySchema, automationInvocationInputSchema, automationInvocationOutputSchema, automationInvocationScheduleSchema, generationInputSchema, generationResultSchema, imageGenerationInputSchema, imageGenerationCompletionSchema, imageGenerationResultSchema, platformImageGenerationInputSchema, platformImageGenerationStartSchema, hashSignalCoordinatorConfiguration, runtimeContract, type GenerationInput, type GenerationResult, type ImageGenerationInput, type ImageGenerationCompletion, type ImageGenerationResult, type PlatformImageGenerationInput, type RuntimeGenerationInput, type RuntimeCoordinationOffer, type ActionReplaySafety, type RuntimeActionAttemptCompletion, type RuntimeActionAttemptStart, } from "./runtime.js";
8
- export type { GatewayImageModelId, GatewayModelId } from "@ai-sdk/gateway";
7
+ export { AUTOMATION_INVOCATION_DEFAULT_ENTRYPOINT, AUTOMATION_INVOCATION_RESERVED_ENTRYPOINT_PREFIX, AUTOMATION_IMAGE_GENERATION_ENTRYPOINT_PREFIX, AUTOMATION_LOG_MAX_ENTRIES, AUTOMATION_LOG_MAX_MESSAGE_LENGTH, AUTOMATION_OBSERVABILITY_MAX_ENCODED_BYTES, AUTOMATION_OBSERVABILITY_MAX_FIELDS, DEFAULT_GATEWAY_EVALUATION_MODEL_ID, DEFAULT_GATEWAY_IMAGE_MODEL_ID, DEFAULT_GATEWAY_MODEL_ID, DEFAULT_ACTION_MAX_ATTEMPTS, MAX_ACTION_ATTEMPTS, actionReplaySafetySchema, automationInvocationInputSchema, automationInvocationOutputSchema, automationInvocationScheduleSchema, evaluationInputSchema, evaluationResultSchema, generationInputSchema, generationResultSchema, imageGenerationInputSchema, imageGenerationCompletionSchema, imageGenerationResultSchema, platformImageGenerationInputSchema, platformImageGenerationStartSchema, hashSignalCoordinatorConfiguration, runtimeContract, type GenerationInput, type GenerationResult, type EvaluationInput, type EvaluationResult, type ImageGenerationInput, type ImageGenerationCompletion, type ImageGenerationResult, type PlatformImageGenerationInput, type RuntimeGenerationInput, type RuntimeCoordinationOffer, type ActionReplaySafety, type RuntimeActionAttemptCompletion, type RuntimeActionAttemptStart, } from "./runtime.js";
8
+ export type { GatewayEvaluationModelId, GatewayImageModelId, GatewayModelId, } from "@ai-sdk/gateway";
9
9
  export { createdOrganizationApiKeyOutputSchema, organizationApiKeyOutputSchema, } from "./api-key.js";
10
10
  export { activeSubscriptionOutputSchema, createdInvitationOutputSchema, organizationDetailsOutputSchema, organizationInvitationOutputSchema, organizationMemberOutputSchema, organizationProjectOutputSchema, organizationRoleSchema, organizationRoles, organizationSummaryOutputSchema, proBillingPeriodSchema, publicOrganizationOutputSchema, type ProBillingPeriod, } from "./org.js";
11
11
  export { projectOutputSchema } from "./project.js";
@@ -1331,7 +1331,7 @@ export declare const publicContract: {
1331
1331
  UNSUPPORTED_RUNTIME_PROTOCOL: {
1332
1332
  readonly data: import("zod").ZodObject<{
1333
1333
  receivedVersion: import("zod").ZodNullable<import("zod").ZodNumber>;
1334
- supportedVersion: import("zod").ZodLiteral<6>;
1334
+ supportedVersion: import("zod").ZodLiteral<7>;
1335
1335
  }, import("zod/v4/core").$strip>;
1336
1336
  readonly message: "The automation bundle uses an unsupported runtime protocol.";
1337
1337
  readonly status: 400;
@@ -1758,7 +1758,7 @@ export declare const firstPartyContract: {
1758
1758
  UNSUPPORTED_RUNTIME_PROTOCOL: {
1759
1759
  readonly data: import("zod").ZodObject<{
1760
1760
  receivedVersion: import("zod").ZodNullable<import("zod").ZodNumber>;
1761
- supportedVersion: import("zod").ZodLiteral<6>;
1761
+ supportedVersion: import("zod").ZodLiteral<7>;
1762
1762
  }, import("zod/v4/core").$strip>;
1763
1763
  readonly message: "The automation bundle uses an unsupported runtime protocol.";
1764
1764
  readonly status: 400;
@@ -2497,7 +2497,7 @@ export declare const firstPartyContract: {
2497
2497
  UNSUPPORTED_RUNTIME_PROTOCOL: {
2498
2498
  readonly data: import("zod").ZodObject<{
2499
2499
  receivedVersion: import("zod").ZodNullable<import("zod").ZodNumber>;
2500
- supportedVersion: import("zod").ZodLiteral<6>;
2500
+ supportedVersion: import("zod").ZodLiteral<7>;
2501
2501
  }, import("zod/v4/core").$strip>;
2502
2502
  readonly message: "The automation bundle uses an unsupported runtime protocol.";
2503
2503
  readonly status: 400;
@@ -8382,6 +8382,74 @@ export declare const contract: {
8382
8382
  output: import("zod").ZodCustom<import("@automate.ax/codec/rpc").CodecEnvelope, import("@automate.ax/codec/rpc").CodecEnvelope>;
8383
8383
  status: import("zod").ZodLiteral<"succeeded">;
8384
8384
  }, import("zod/v4/core").$strip>], "status">, import("zod").ZodVoid, Record<never, never>, Record<never, never>>;
8385
+ evaluate: import("@orpc/contract").ContractProcedureBuilderWithInputOutput<import("zod").ZodObject<{
8386
+ headers: import("zod").ZodOptional<import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodString>>;
8387
+ maxRetries: import("zod").ZodOptional<import("zod").ZodNumber>;
8388
+ model: import("zod").ZodDefault<import("zod").ZodCustom<import("@ai-sdk/gateway").GatewayEvaluationModelId, import("@ai-sdk/gateway").GatewayEvaluationModelId>>;
8389
+ providerOptions: import("zod").ZodOptional<import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodJSONSchema>>>;
8390
+ questions: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
8391
+ criteria: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodNullable<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodJSONSchema>, import("zod").ZodArray<import("zod").ZodJSONSchema>]>>>;
8392
+ instructions: import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodJSONSchema>, import("zod").ZodArray<import("zod").ZodJSONSchema>]>;
8393
+ type: import("zod").ZodLiteral<"choice">;
8394
+ }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
8395
+ criteria: import("zod").ZodArray<import("zod").ZodNullable<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodJSONSchema>, import("zod").ZodArray<import("zod").ZodJSONSchema>]>>>;
8396
+ instructions: import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodJSONSchema>, import("zod").ZodArray<import("zod").ZodJSONSchema>]>;
8397
+ type: import("zod").ZodLiteral<"score">;
8398
+ }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
8399
+ criteria: import("zod").ZodOptional<import("zod").ZodObject<{
8400
+ false: import("zod").ZodOptional<import("zod").ZodNullable<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodJSONSchema>, import("zod").ZodArray<import("zod").ZodJSONSchema>]>>>;
8401
+ true: import("zod").ZodOptional<import("zod").ZodNullable<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodJSONSchema>, import("zod").ZodArray<import("zod").ZodJSONSchema>]>>>;
8402
+ }, import("zod/v4/core").$strip>>;
8403
+ instructions: import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodJSONSchema>, import("zod").ZodArray<import("zod").ZodJSONSchema>]>;
8404
+ type: import("zod").ZodLiteral<"boolean">;
8405
+ }, import("zod/v4/core").$strip>], "type">>;
8406
+ state: import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodJSONSchema>, import("zod").ZodArray<import("zod").ZodJSONSchema>]>;
8407
+ }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
8408
+ answers: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
8409
+ choice: import("zod").ZodString;
8410
+ probabilities: import("zod").ZodOptional<import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodNumber>>;
8411
+ type: import("zod").ZodLiteral<"choice">;
8412
+ }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
8413
+ probabilities: import("zod").ZodOptional<import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodNumber>>;
8414
+ score: import("zod").ZodNumber;
8415
+ type: import("zod").ZodLiteral<"score">;
8416
+ }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
8417
+ probability: import("zod").ZodNumber;
8418
+ type: import("zod").ZodLiteral<"boolean">;
8419
+ }, import("zod/v4/core").$strip>], "type">>;
8420
+ providerMetadata: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodJSONSchema>>;
8421
+ response: import("zod").ZodObject<{
8422
+ headers: import("zod").ZodOptional<import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodString>>;
8423
+ id: import("zod").ZodOptional<import("zod").ZodString>;
8424
+ modelId: import("zod").ZodString;
8425
+ timestamp: import("zod").ZodISODateTime;
8426
+ }, import("zod/v4/core").$strip>;
8427
+ rounding: import("zod").ZodOptional<import("zod").ZodObject<{
8428
+ probabilityDecimals: import("zod").ZodOptional<import("zod").ZodNumber>;
8429
+ scoreDecimals: import("zod").ZodOptional<import("zod").ZodNumber>;
8430
+ }, import("zod/v4/core").$strip>>;
8431
+ usage: import("zod").ZodObject<{
8432
+ inputTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
8433
+ outputTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
8434
+ totalTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
8435
+ }, import("zod/v4/core").$strip>;
8436
+ warnings: import("zod").ZodArray<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
8437
+ details: import("zod").ZodOptional<import("zod").ZodString>;
8438
+ feature: import("zod").ZodString;
8439
+ type: import("zod").ZodLiteral<"unsupported">;
8440
+ }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
8441
+ details: import("zod").ZodOptional<import("zod").ZodString>;
8442
+ feature: import("zod").ZodString;
8443
+ type: import("zod").ZodLiteral<"compatibility">;
8444
+ }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
8445
+ message: import("zod").ZodString;
8446
+ setting: import("zod").ZodString;
8447
+ type: import("zod").ZodLiteral<"deprecated">;
8448
+ }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
8449
+ message: import("zod").ZodString;
8450
+ type: import("zod").ZodLiteral<"other">;
8451
+ }, import("zod/v4/core").$strip>], "type">>;
8452
+ }, import("zod/v4/core").$strip>, Record<never, never>, Record<never, never>>;
8385
8453
  generate: import("@orpc/contract").ContractProcedureBuilderWithInputOutput<import("zod").ZodUnion<readonly [import("zod").ZodObject<{
8386
8454
  frequencyPenalty: import("zod").ZodOptional<import("zod").ZodNumber>;
8387
8455
  instructions: import("zod").ZodOptional<import("zod").ZodString>;
@@ -9363,7 +9431,7 @@ export declare const contract: {
9363
9431
  UNSUPPORTED_RUNTIME_PROTOCOL: {
9364
9432
  readonly data: import("zod").ZodObject<{
9365
9433
  receivedVersion: import("zod").ZodNullable<import("zod").ZodNumber>;
9366
- supportedVersion: import("zod").ZodLiteral<6>;
9434
+ supportedVersion: import("zod").ZodLiteral<7>;
9367
9435
  }, import("zod/v4/core").$strip>;
9368
9436
  readonly message: "The automation bundle uses an unsupported runtime protocol.";
9369
9437
  readonly status: 400;
@@ -10102,7 +10170,7 @@ export declare const contract: {
10102
10170
  UNSUPPORTED_RUNTIME_PROTOCOL: {
10103
10171
  readonly data: import("zod").ZodObject<{
10104
10172
  receivedVersion: import("zod").ZodNullable<import("zod").ZodNumber>;
10105
- supportedVersion: import("zod").ZodLiteral<6>;
10173
+ supportedVersion: import("zod").ZodLiteral<7>;
10106
10174
  }, import("zod/v4/core").$strip>;
10107
10175
  readonly message: "The automation bundle uses an unsupported runtime protocol.";
10108
10176
  readonly status: 400;
package/dist/index.js CHANGED
@@ -15,7 +15,7 @@ export { AUTOMATION_SDK_VERSION, deploymentOutputSchema, deploymentStatusSchema,
15
15
  export { AUTOMATION_RUNTIME_PROTOCOL_VERSION } from "./runtime-protocol.js";
16
16
  export { AUTOMATION_ERROR_CAUSE_DEPTH, automationErrorSchema, } from "./errors.js";
17
17
  export { integrationAccountOutputSchema, integrationAccountRevocationImpactOutputSchema, integrationAccountRevocationOutputSchema, integrationServiceOutputSchema, } from "./account.js";
18
- export { AUTOMATION_INVOCATION_DEFAULT_ENTRYPOINT, AUTOMATION_INVOCATION_RESERVED_ENTRYPOINT_PREFIX, AUTOMATION_IMAGE_GENERATION_ENTRYPOINT_PREFIX, AUTOMATION_LOG_MAX_ENTRIES, AUTOMATION_LOG_MAX_MESSAGE_LENGTH, AUTOMATION_OBSERVABILITY_MAX_ENCODED_BYTES, AUTOMATION_OBSERVABILITY_MAX_FIELDS, DEFAULT_GATEWAY_IMAGE_MODEL_ID, DEFAULT_GATEWAY_MODEL_ID, DEFAULT_ACTION_MAX_ATTEMPTS, MAX_ACTION_ATTEMPTS, actionReplaySafetySchema, automationInvocationInputSchema, automationInvocationOutputSchema, automationInvocationScheduleSchema, generationInputSchema, generationResultSchema, imageGenerationInputSchema, imageGenerationCompletionSchema, imageGenerationResultSchema, platformImageGenerationInputSchema, platformImageGenerationStartSchema, hashSignalCoordinatorConfiguration, runtimeContract, } from "./runtime.js";
18
+ export { AUTOMATION_INVOCATION_DEFAULT_ENTRYPOINT, AUTOMATION_INVOCATION_RESERVED_ENTRYPOINT_PREFIX, AUTOMATION_IMAGE_GENERATION_ENTRYPOINT_PREFIX, AUTOMATION_LOG_MAX_ENTRIES, AUTOMATION_LOG_MAX_MESSAGE_LENGTH, AUTOMATION_OBSERVABILITY_MAX_ENCODED_BYTES, AUTOMATION_OBSERVABILITY_MAX_FIELDS, DEFAULT_GATEWAY_EVALUATION_MODEL_ID, DEFAULT_GATEWAY_IMAGE_MODEL_ID, DEFAULT_GATEWAY_MODEL_ID, DEFAULT_ACTION_MAX_ATTEMPTS, MAX_ACTION_ATTEMPTS, actionReplaySafetySchema, automationInvocationInputSchema, automationInvocationOutputSchema, automationInvocationScheduleSchema, evaluationInputSchema, evaluationResultSchema, generationInputSchema, generationResultSchema, imageGenerationInputSchema, imageGenerationCompletionSchema, imageGenerationResultSchema, platformImageGenerationInputSchema, platformImageGenerationStartSchema, hashSignalCoordinatorConfiguration, runtimeContract, } from "./runtime.js";
19
19
  export { createdOrganizationApiKeyOutputSchema, organizationApiKeyOutputSchema, } from "./api-key.js";
20
20
  export { activeSubscriptionOutputSchema, createdInvitationOutputSchema, organizationDetailsOutputSchema, organizationInvitationOutputSchema, organizationMemberOutputSchema, organizationProjectOutputSchema, organizationRoleSchema, organizationRoles, organizationSummaryOutputSchema, proBillingPeriodSchema, publicOrganizationOutputSchema, } from "./org.js";
21
21
  export { projectOutputSchema } from "./project.js";
package/dist/runtime-protocol.d.ts CHANGED
@@ -1,2 +1,2 @@
1
1
  /** Runtime protocol required by newly built automation bundles. */
2
- export declare const AUTOMATION_RUNTIME_PROTOCOL_VERSION = 6;
2
+ export declare const AUTOMATION_RUNTIME_PROTOCOL_VERSION = 7;
package/dist/runtime-protocol.js CHANGED
@@ -1,2 +1,2 @@
1
1
  /** Runtime protocol required by newly built automation bundles. */
2
- export const AUTOMATION_RUNTIME_PROTOCOL_VERSION = 6;
2
+ export const AUTOMATION_RUNTIME_PROTOCOL_VERSION = 7;
package/dist/runtime.d.ts CHANGED
@@ -1,9 +1,11 @@
1
1
  import { type SensitivityMask } from "@automate.ax/codec";
2
- import type { GatewayImageModelId, GatewayModelId } from "@ai-sdk/gateway";
2
+ import type { GatewayEvaluationModelId, GatewayImageModelId, GatewayModelId } from "@ai-sdk/gateway";
3
3
  import * as z from "zod";
4
- export type { GatewayImageModelId, GatewayModelId } from "@ai-sdk/gateway";
4
+ export type { GatewayEvaluationModelId, GatewayImageModelId, GatewayModelId, } from "@ai-sdk/gateway";
5
5
  /** Default model used by the platform Vercel AI Gateway account. */
6
6
  export declare const DEFAULT_GATEWAY_MODEL_ID = "openai/gpt-4.1-nano";
7
+ /** Default evaluation model used by the platform Vercel AI Gateway account. */
8
+ export declare const DEFAULT_GATEWAY_EVALUATION_MODEL_ID = "typesafe-ai/jev";
7
9
  /** Default image model used by the platform Vercel AI Gateway account. */
8
10
  export declare const DEFAULT_GATEWAY_IMAGE_MODEL_ID = "bfl/flux-2-flex";
9
11
  export declare const AUTOMATION_INVOCATION_DEFAULT_ENTRYPOINT = "default";
@@ -1222,6 +1224,79 @@ declare const RUNTIME_GENERATION_INPUT_SCHEMA: z.ZodUnion<readonly [z.ZodObject<
1222
1224
  }, z.core.$strip>>;
1223
1225
  }, z.core.$strip>]>;
1224
1226
  export type RuntimeGenerationInput = z.output<typeof RUNTIME_GENERATION_INPUT_SCHEMA>;
1227
+ /** Shared options accepted by the built-in evaluate action. */
1228
+ export declare const evaluationInputSchema: z.ZodObject<{
1229
+ headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
1230
+ maxRetries: z.ZodOptional<z.ZodNumber>;
1231
+ model: z.ZodDefault<z.ZodCustom<GatewayEvaluationModelId, GatewayEvaluationModelId>>;
1232
+ providerOptions: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>>>;
1233
+ questions: z.ZodRecord<z.ZodString, z.ZodDiscriminatedUnion<[z.ZodObject<{
1234
+ criteria: z.ZodRecord<z.ZodString, z.ZodNullable<z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>, z.ZodArray<z.ZodJSONSchema>]>>>;
1235
+ instructions: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>, z.ZodArray<z.ZodJSONSchema>]>;
1236
+ type: z.ZodLiteral<"choice">;
1237
+ }, z.core.$strip>, z.ZodObject<{
1238
+ criteria: z.ZodArray<z.ZodNullable<z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>, z.ZodArray<z.ZodJSONSchema>]>>>;
1239
+ instructions: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>, z.ZodArray<z.ZodJSONSchema>]>;
1240
+ type: z.ZodLiteral<"score">;
1241
+ }, z.core.$strip>, z.ZodObject<{
1242
+ criteria: z.ZodOptional<z.ZodObject<{
1243
+ false: z.ZodOptional<z.ZodNullable<z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>, z.ZodArray<z.ZodJSONSchema>]>>>;
1244
+ true: z.ZodOptional<z.ZodNullable<z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>, z.ZodArray<z.ZodJSONSchema>]>>>;
1245
+ }, z.core.$strip>>;
1246
+ instructions: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>, z.ZodArray<z.ZodJSONSchema>]>;
1247
+ type: z.ZodLiteral<"boolean">;
1248
+ }, z.core.$strip>], "type">>;
1249
+ state: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>, z.ZodArray<z.ZodJSONSchema>]>;
1250
+ }, z.core.$strip>;
1251
+ /** Serializable subset of the AI SDK evaluation result. */
1252
+ export declare const evaluationResultSchema: z.ZodObject<{
1253
+ answers: z.ZodRecord<z.ZodString, z.ZodDiscriminatedUnion<[z.ZodObject<{
1254
+ choice: z.ZodString;
1255
+ probabilities: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
1256
+ type: z.ZodLiteral<"choice">;
1257
+ }, z.core.$strip>, z.ZodObject<{
1258
+ probabilities: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
1259
+ score: z.ZodNumber;
1260
+ type: z.ZodLiteral<"score">;
1261
+ }, z.core.$strip>, z.ZodObject<{
1262
+ probability: z.ZodNumber;
1263
+ type: z.ZodLiteral<"boolean">;
1264
+ }, z.core.$strip>], "type">>;
1265
+ providerMetadata: z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>>;
1266
+ response: z.ZodObject<{
1267
+ headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
1268
+ id: z.ZodOptional<z.ZodString>;
1269
+ modelId: z.ZodString;
1270
+ timestamp: z.ZodISODateTime;
1271
+ }, z.core.$strip>;
1272
+ rounding: z.ZodOptional<z.ZodObject<{
1273
+ probabilityDecimals: z.ZodOptional<z.ZodNumber>;
1274
+ scoreDecimals: z.ZodOptional<z.ZodNumber>;
1275
+ }, z.core.$strip>>;
1276
+ usage: z.ZodObject<{
1277
+ inputTokens: z.ZodOptional<z.ZodNumber>;
1278
+ outputTokens: z.ZodOptional<z.ZodNumber>;
1279
+ totalTokens: z.ZodOptional<z.ZodNumber>;
1280
+ }, z.core.$strip>;
1281
+ warnings: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
1282
+ details: z.ZodOptional<z.ZodString>;
1283
+ feature: z.ZodString;
1284
+ type: z.ZodLiteral<"unsupported">;
1285
+ }, z.core.$strip>, z.ZodObject<{
1286
+ details: z.ZodOptional<z.ZodString>;
1287
+ feature: z.ZodString;
1288
+ type: z.ZodLiteral<"compatibility">;
1289
+ }, z.core.$strip>, z.ZodObject<{
1290
+ message: z.ZodString;
1291
+ setting: z.ZodString;
1292
+ type: z.ZodLiteral<"deprecated">;
1293
+ }, z.core.$strip>, z.ZodObject<{
1294
+ message: z.ZodString;
1295
+ type: z.ZodLiteral<"other">;
1296
+ }, z.core.$strip>], "type">>;
1297
+ }, z.core.$strip>;
1298
+ export type EvaluationInput = z.output<typeof evaluationInputSchema>;
1299
+ export type EvaluationResult = z.output<typeof evaluationResultSchema>;
1225
1300
  /** Shared options accepted by the built-in image-generation action. */
1226
1301
  export declare const imageGenerationInputSchema: z.ZodObject<{
1227
1302
  aspectRatio: z.ZodOptional<z.ZodCustom<`${number}:${number}`, `${number}:${number}`>>;
@@ -2323,6 +2398,74 @@ export declare const runtimeContract: {
2323
2398
  output: z.ZodCustom<import("@automate.ax/codec/rpc").CodecEnvelope, import("@automate.ax/codec/rpc").CodecEnvelope>;
2324
2399
  status: z.ZodLiteral<"succeeded">;
2325
2400
  }, z.core.$strip>], "status">, z.ZodVoid, Record<never, never>, Record<never, never>>;
2401
+ evaluate: import("@orpc/contract").ContractProcedureBuilderWithInputOutput<z.ZodObject<{
2402
+ headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
2403
+ maxRetries: z.ZodOptional<z.ZodNumber>;
2404
+ model: z.ZodDefault<z.ZodCustom<GatewayEvaluationModelId, GatewayEvaluationModelId>>;
2405
+ providerOptions: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>>>;
2406
+ questions: z.ZodRecord<z.ZodString, z.ZodDiscriminatedUnion<[z.ZodObject<{
2407
+ criteria: z.ZodRecord<z.ZodString, z.ZodNullable<z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>, z.ZodArray<z.ZodJSONSchema>]>>>;
2408
+ instructions: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>, z.ZodArray<z.ZodJSONSchema>]>;
2409
+ type: z.ZodLiteral<"choice">;
2410
+ }, z.core.$strip>, z.ZodObject<{
2411
+ criteria: z.ZodArray<z.ZodNullable<z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>, z.ZodArray<z.ZodJSONSchema>]>>>;
2412
+ instructions: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>, z.ZodArray<z.ZodJSONSchema>]>;
2413
+ type: z.ZodLiteral<"score">;
2414
+ }, z.core.$strip>, z.ZodObject<{
2415
+ criteria: z.ZodOptional<z.ZodObject<{
2416
+ false: z.ZodOptional<z.ZodNullable<z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>, z.ZodArray<z.ZodJSONSchema>]>>>;
2417
+ true: z.ZodOptional<z.ZodNullable<z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>, z.ZodArray<z.ZodJSONSchema>]>>>;
2418
+ }, z.core.$strip>>;
2419
+ instructions: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>, z.ZodArray<z.ZodJSONSchema>]>;
2420
+ type: z.ZodLiteral<"boolean">;
2421
+ }, z.core.$strip>], "type">>;
2422
+ state: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>, z.ZodArray<z.ZodJSONSchema>]>;
2423
+ }, z.core.$strip>, z.ZodObject<{
2424
+ answers: z.ZodRecord<z.ZodString, z.ZodDiscriminatedUnion<[z.ZodObject<{
2425
+ choice: z.ZodString;
2426
+ probabilities: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
2427
+ type: z.ZodLiteral<"choice">;
2428
+ }, z.core.$strip>, z.ZodObject<{
2429
+ probabilities: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
2430
+ score: z.ZodNumber;
2431
+ type: z.ZodLiteral<"score">;
2432
+ }, z.core.$strip>, z.ZodObject<{
2433
+ probability: z.ZodNumber;
2434
+ type: z.ZodLiteral<"boolean">;
2435
+ }, z.core.$strip>], "type">>;
2436
+ providerMetadata: z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodJSONSchema>>;
2437
+ response: z.ZodObject<{
2438
+ headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
2439
+ id: z.ZodOptional<z.ZodString>;
2440
+ modelId: z.ZodString;
2441
+ timestamp: z.ZodISODateTime;
2442
+ }, z.core.$strip>;
2443
+ rounding: z.ZodOptional<z.ZodObject<{
2444
+ probabilityDecimals: z.ZodOptional<z.ZodNumber>;
2445
+ scoreDecimals: z.ZodOptional<z.ZodNumber>;
2446
+ }, z.core.$strip>>;
2447
+ usage: z.ZodObject<{
2448
+ inputTokens: z.ZodOptional<z.ZodNumber>;
2449
+ outputTokens: z.ZodOptional<z.ZodNumber>;
2450
+ totalTokens: z.ZodOptional<z.ZodNumber>;
2451
+ }, z.core.$strip>;
2452
+ warnings: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
2453
+ details: z.ZodOptional<z.ZodString>;
2454
+ feature: z.ZodString;
2455
+ type: z.ZodLiteral<"unsupported">;
2456
+ }, z.core.$strip>, z.ZodObject<{
2457
+ details: z.ZodOptional<z.ZodString>;
2458
+ feature: z.ZodString;
2459
+ type: z.ZodLiteral<"compatibility">;
2460
+ }, z.core.$strip>, z.ZodObject<{
2461
+ message: z.ZodString;
2462
+ setting: z.ZodString;
2463
+ type: z.ZodLiteral<"deprecated">;
2464
+ }, z.core.$strip>, z.ZodObject<{
2465
+ message: z.ZodString;
2466
+ type: z.ZodLiteral<"other">;
2467
+ }, z.core.$strip>], "type">>;
2468
+ }, z.core.$strip>, Record<never, never>, Record<never, never>>;
2326
2469
  generate: import("@orpc/contract").ContractProcedureBuilderWithInputOutput<z.ZodUnion<readonly [z.ZodObject<{
2327
2470
  frequencyPenalty: z.ZodOptional<z.ZodNumber>;
2328
2471
  instructions: z.ZodOptional<z.ZodString>;
package/dist/runtime.js CHANGED
@@ -7,6 +7,8 @@ import * as z from "zod";
7
7
  import { automationErrorSchema } from "./errors.js";
8
8
  /** Default model used by the platform Vercel AI Gateway account. */
9
9
  export const DEFAULT_GATEWAY_MODEL_ID = "openai/gpt-4.1-nano";
10
+ /** Default evaluation model used by the platform Vercel AI Gateway account. */
11
+ export const DEFAULT_GATEWAY_EVALUATION_MODEL_ID = "typesafe-ai/jev";
10
12
  /** Default image model used by the platform Vercel AI Gateway account. */
11
13
  export const DEFAULT_GATEWAY_IMAGE_MODEL_ID = "bfl/flux-2-flex";
12
14
  export const AUTOMATION_INVOCATION_DEFAULT_ENTRYPOINT = "default";
@@ -791,6 +793,95 @@ const RUNTIME_GENERATION_INPUT_SCHEMA = z.union([
791
793
  output: RUNTIME_GENERATION_OUTPUT_SCHEMA,
792
794
  }),
793
795
  ]);
796
+ const EVALUATION_CONTENT_SCHEMA = z.union([
797
+ z.string(),
798
+ z.record(z.string(), z.json()),
799
+ z.json().array(),
800
+ ]);
801
+ const EVALUATION_CRITERION_SCHEMA = EVALUATION_CONTENT_SCHEMA.nullable();
802
+ const EVALUATION_QUESTION_SCHEMA = z.discriminatedUnion("type", [
803
+ z.object({
804
+ criteria: z
805
+ .record(z.string(), EVALUATION_CRITERION_SCHEMA)
806
+ .refine((criteria) => Object.keys(criteria).length > 0, {
807
+ message: "Choice criteria must contain at least one option.",
808
+ }),
809
+ instructions: EVALUATION_CONTENT_SCHEMA,
810
+ type: z.literal("choice"),
811
+ }),
812
+ z.object({
813
+ criteria: EVALUATION_CRITERION_SCHEMA.array().min(2),
814
+ instructions: EVALUATION_CONTENT_SCHEMA,
815
+ type: z.literal("score"),
816
+ }),
817
+ z.object({
818
+ criteria: z
819
+ .object({
820
+ false: EVALUATION_CRITERION_SCHEMA.optional(),
821
+ true: EVALUATION_CRITERION_SCHEMA.optional(),
822
+ })
823
+ .optional(),
824
+ instructions: EVALUATION_CONTENT_SCHEMA,
825
+ type: z.literal("boolean"),
826
+ }),
827
+ ]);
828
+ const GATEWAY_EVALUATION_MODEL_ID_SCHEMA = z
829
+ .custom((value) => typeof value === "string" && value.length > 0, "Model must be a nonempty AI Gateway evaluation model ID.")
830
+ .default(DEFAULT_GATEWAY_EVALUATION_MODEL_ID);
831
+ /** Shared options accepted by the built-in evaluate action. */
832
+ export const evaluationInputSchema = z.object({
833
+ headers: z.record(z.string(), z.string()).optional(),
834
+ maxRetries: z.number().int().nonnegative().optional(),
835
+ model: GATEWAY_EVALUATION_MODEL_ID_SCHEMA,
836
+ providerOptions: z
837
+ .record(z.string(), z.record(z.string(), z.json()))
838
+ .optional(),
839
+ questions: z
840
+ .record(z.string(), EVALUATION_QUESTION_SCHEMA)
841
+ .refine((questions) => Object.keys(questions).length > 0, {
842
+ message: "Questions must contain at least one evaluation.",
843
+ }),
844
+ state: EVALUATION_CONTENT_SCHEMA,
845
+ });
846
+ const EVALUATION_ANSWER_SCHEMA = z.discriminatedUnion("type", [
847
+ z.object({
848
+ choice: z.string(),
849
+ probabilities: z.record(z.string(), z.number()).optional(),
850
+ type: z.literal("choice"),
851
+ }),
852
+ z.object({
853
+ probabilities: z.record(z.string(), z.number()).optional(),
854
+ score: z.number(),
855
+ type: z.literal("score"),
856
+ }),
857
+ z.object({
858
+ probability: z.number(),
859
+ type: z.literal("boolean"),
860
+ }),
861
+ ]);
862
+ /** Serializable subset of the AI SDK evaluation result. */
863
+ export const evaluationResultSchema = z.object({
864
+ answers: z.record(z.string(), EVALUATION_ANSWER_SCHEMA),
865
+ providerMetadata: z.record(z.string(), z.record(z.string(), z.json())),
866
+ response: z.object({
867
+ headers: z.record(z.string(), z.string()).optional(),
868
+ id: z.string().optional(),
869
+ modelId: z.string(),
870
+ timestamp: z.iso.datetime(),
871
+ }),
872
+ rounding: z
873
+ .object({
874
+ probabilityDecimals: z.number().int().min(0).max(15).optional(),
875
+ scoreDecimals: z.number().int().min(0).max(15).optional(),
876
+ })
877
+ .optional(),
878
+ usage: z.object({
879
+ inputTokens: z.number().optional(),
880
+ outputTokens: z.number().optional(),
881
+ totalTokens: z.number().optional(),
882
+ }),
883
+ warnings: AI_WARNING_SCHEMA.array(),
884
+ });
794
885
  const IMAGE_DATA_CONTENT_SCHEMA = z.union([
795
886
  z.string(),
796
887
  z.custom((value) => value instanceof Uint8Array),
@@ -995,6 +1086,7 @@ export const runtimeContract = {
995
1086
  completeActionAttempt: oc
996
1087
  .input(ACTION_ATTEMPT_COMPLETION_TRANSPORT_SCHEMA)
997
1088
  .output(z.void()),
1089
+ evaluate: oc.input(evaluationInputSchema).output(evaluationResultSchema),
998
1090
  generate: oc
999
1091
  .input(RUNTIME_GENERATION_INPUT_SCHEMA)
1000
1092
  .output(RUNTIME_GENERATION_RESULT_SCHEMA),
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@automate.ax/api-contract",
3
- "version": "0.157.0",
3
+ "version": "0.158.0",
4
4
  "description": "Public oRPC contract for the Automate.ax API.",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -30,10 +30,10 @@
30
30
  }
31
31
  },
32
32
  "dependencies": {
33
- "@ai-sdk/gateway": "4.0.46",
34
- "@automate.ax/catalog": "0.157.0",
35
- "@automate.ax/codec": "0.157.0",
36
- "@automate.ax/integration-contracts": "0.157.0",
33
+ "@ai-sdk/gateway": "4.0.87",
34
+ "@automate.ax/catalog": "0.158.0",
35
+ "@automate.ax/codec": "0.158.0",
36
+ "@automate.ax/integration-contracts": "0.158.0",
37
37
  "@orpc/contract": "1.15.0",
38
38
  "fast-json-stable-stringify": "^2.1.0",
39
39
  "zod": "^4.3.6"
package/src/index.ts CHANGED
@@ -53,6 +53,7 @@ export {
53
53
  AUTOMATION_LOG_MAX_MESSAGE_LENGTH,
54
54
  AUTOMATION_OBSERVABILITY_MAX_ENCODED_BYTES,
55
55
  AUTOMATION_OBSERVABILITY_MAX_FIELDS,
56
+ DEFAULT_GATEWAY_EVALUATION_MODEL_ID,
56
57
  DEFAULT_GATEWAY_IMAGE_MODEL_ID,
57
58
  DEFAULT_GATEWAY_MODEL_ID,
58
59
  DEFAULT_ACTION_MAX_ATTEMPTS,
@@ -61,6 +62,8 @@ export {
61
62
  automationInvocationInputSchema,
62
63
  automationInvocationOutputSchema,
63
64
  automationInvocationScheduleSchema,
65
+ evaluationInputSchema,
66
+ evaluationResultSchema,
64
67
  generationInputSchema,
65
68
  generationResultSchema,
66
69
  imageGenerationInputSchema,
@@ -72,6 +75,8 @@ export {
72
75
  runtimeContract,
73
76
  type GenerationInput,
74
77
  type GenerationResult,
78
+ type EvaluationInput,
79
+ type EvaluationResult,
75
80
  type ImageGenerationInput,
76
81
  type ImageGenerationCompletion,
77
82
  type ImageGenerationResult,
@@ -82,7 +87,11 @@ export {
82
87
  type RuntimeActionAttemptCompletion,
83
88
  type RuntimeActionAttemptStart,
84
89
  } from "./runtime"
85
- export type { GatewayImageModelId, GatewayModelId } from "@ai-sdk/gateway"
90
+ export type {
91
+ GatewayEvaluationModelId,
92
+ GatewayImageModelId,
93
+ GatewayModelId,
94
+ } from "@ai-sdk/gateway"
86
95
  export {
87
96
  createdOrganizationApiKeyOutputSchema,
88
97
  organizationApiKeyOutputSchema,
package/src/runtime-protocol.ts CHANGED
@@ -1,2 +1,2 @@
1
1
  /** Runtime protocol required by newly built automation bundles. */
2
- export const AUTOMATION_RUNTIME_PROTOCOL_VERSION = 6
2
+ export const AUTOMATION_RUNTIME_PROTOCOL_VERSION = 7
package/src/runtime.ts CHANGED
@@ -5,18 +5,30 @@ import {
5
5
  } from "@automate.ax/codec"
6
6
  import { codecEnvelopeSchema } from "@automate.ax/codec/rpc"
7
7
  import { CLOUDFLARE_BROWSER_MARKDOWN_REQUEST_SCHEMA } from "@automate.ax/integration-contracts/cloudflare"
8
- import type { GatewayImageModelId, GatewayModelId } from "@ai-sdk/gateway"
8
+ import type {
9
+ GatewayEvaluationModelId,
10
+ GatewayImageModelId,
11
+ GatewayModelId,
12
+ } from "@ai-sdk/gateway"
9
13
  import { oc } from "@orpc/contract"
10
14
  import stableStringify from "fast-json-stable-stringify"
11
15
  import * as z from "zod"
12
16
  import { automationErrorSchema } from "./errors"
13
17
 
14
- export type { GatewayImageModelId, GatewayModelId } from "@ai-sdk/gateway"
18
+ export type {
19
+ GatewayEvaluationModelId,
20
+ GatewayImageModelId,
21
+ GatewayModelId,
22
+ } from "@ai-sdk/gateway"
15
23
 
16
24
  /** Default model used by the platform Vercel AI Gateway account. */
17
25
  export const DEFAULT_GATEWAY_MODEL_ID =
18
26
  "openai/gpt-4.1-nano" satisfies GatewayModelId
19
27
 
28
+ /** Default evaluation model used by the platform Vercel AI Gateway account. */
29
+ export const DEFAULT_GATEWAY_EVALUATION_MODEL_ID =
30
+ "typesafe-ai/jev" satisfies GatewayEvaluationModelId
31
+
20
32
  /** Default image model used by the platform Vercel AI Gateway account. */
21
33
  export const DEFAULT_GATEWAY_IMAGE_MODEL_ID =
22
34
  "bfl/flux-2-flex" satisfies GatewayImageModelId
@@ -948,6 +960,105 @@ export type RuntimeGenerationInput = z.output<
948
960
  typeof RUNTIME_GENERATION_INPUT_SCHEMA
949
961
  >
950
962
 
963
+ const EVALUATION_CONTENT_SCHEMA = z.union([
964
+ z.string(),
965
+ z.record(z.string(), z.json()),
966
+ z.json().array(),
967
+ ])
968
+ const EVALUATION_CRITERION_SCHEMA = EVALUATION_CONTENT_SCHEMA.nullable()
969
+ const EVALUATION_QUESTION_SCHEMA = z.discriminatedUnion("type", [
970
+ z.object({
971
+ criteria: z
972
+ .record(z.string(), EVALUATION_CRITERION_SCHEMA)
973
+ .refine((criteria) => Object.keys(criteria).length > 0, {
974
+ message: "Choice criteria must contain at least one option.",
975
+ }),
976
+ instructions: EVALUATION_CONTENT_SCHEMA,
977
+ type: z.literal("choice"),
978
+ }),
979
+ z.object({
980
+ criteria: EVALUATION_CRITERION_SCHEMA.array().min(2),
981
+ instructions: EVALUATION_CONTENT_SCHEMA,
982
+ type: z.literal("score"),
983
+ }),
984
+ z.object({
985
+ criteria: z
986
+ .object({
987
+ false: EVALUATION_CRITERION_SCHEMA.optional(),
988
+ true: EVALUATION_CRITERION_SCHEMA.optional(),
989
+ })
990
+ .optional(),
991
+ instructions: EVALUATION_CONTENT_SCHEMA,
992
+ type: z.literal("boolean"),
993
+ }),
994
+ ])
995
+ const GATEWAY_EVALUATION_MODEL_ID_SCHEMA = z
996
+ .custom<GatewayEvaluationModelId>(
997
+ (value) => typeof value === "string" && value.length > 0,
998
+ "Model must be a nonempty AI Gateway evaluation model ID.",
999
+ )
1000
+ .default(DEFAULT_GATEWAY_EVALUATION_MODEL_ID)
1001
+
1002
+ /** Shared options accepted by the built-in evaluate action. */
1003
+ export const evaluationInputSchema = z.object({
1004
+ headers: z.record(z.string(), z.string()).optional(),
1005
+ maxRetries: z.number().int().nonnegative().optional(),
1006
+ model: GATEWAY_EVALUATION_MODEL_ID_SCHEMA,
1007
+ providerOptions: z
1008
+ .record(z.string(), z.record(z.string(), z.json()))
1009
+ .optional(),
1010
+ questions: z
1011
+ .record(z.string(), EVALUATION_QUESTION_SCHEMA)
1012
+ .refine((questions) => Object.keys(questions).length > 0, {
1013
+ message: "Questions must contain at least one evaluation.",
1014
+ }),
1015
+ state: EVALUATION_CONTENT_SCHEMA,
1016
+ })
1017
+
1018
+ const EVALUATION_ANSWER_SCHEMA = z.discriminatedUnion("type", [
1019
+ z.object({
1020
+ choice: z.string(),
1021
+ probabilities: z.record(z.string(), z.number()).optional(),
1022
+ type: z.literal("choice"),
1023
+ }),
1024
+ z.object({
1025
+ probabilities: z.record(z.string(), z.number()).optional(),
1026
+ score: z.number(),
1027
+ type: z.literal("score"),
1028
+ }),
1029
+ z.object({
1030
+ probability: z.number(),
1031
+ type: z.literal("boolean"),
1032
+ }),
1033
+ ])
1034
+
1035
+ /** Serializable subset of the AI SDK evaluation result. */
1036
+ export const evaluationResultSchema = z.object({
1037
+ answers: z.record(z.string(), EVALUATION_ANSWER_SCHEMA),
1038
+ providerMetadata: z.record(z.string(), z.record(z.string(), z.json())),
1039
+ response: z.object({
1040
+ headers: z.record(z.string(), z.string()).optional(),
1041
+ id: z.string().optional(),
1042
+ modelId: z.string(),
1043
+ timestamp: z.iso.datetime(),
1044
+ }),
1045
+ rounding: z
1046
+ .object({
1047
+ probabilityDecimals: z.number().int().min(0).max(15).optional(),
1048
+ scoreDecimals: z.number().int().min(0).max(15).optional(),
1049
+ })
1050
+ .optional(),
1051
+ usage: z.object({
1052
+ inputTokens: z.number().optional(),
1053
+ outputTokens: z.number().optional(),
1054
+ totalTokens: z.number().optional(),
1055
+ }),
1056
+ warnings: AI_WARNING_SCHEMA.array(),
1057
+ })
1058
+
1059
+ export type EvaluationInput = z.output<typeof evaluationInputSchema>
1060
+ export type EvaluationResult = z.output<typeof evaluationResultSchema>
1061
+
951
1062
  const IMAGE_DATA_CONTENT_SCHEMA = z.union([
952
1063
  z.string(),
953
1064
  z.custom<Uint8Array>((value) => value instanceof Uint8Array),
@@ -1224,6 +1335,7 @@ export const runtimeContract = {
1224
1335
  completeActionAttempt: oc
1225
1336
  .input(ACTION_ATTEMPT_COMPLETION_TRANSPORT_SCHEMA)
1226
1337
  .output(z.void()),
1338
+ evaluate: oc.input(evaluationInputSchema).output(evaluationResultSchema),
1227
1339
  generate: oc
1228
1340
  .input(RUNTIME_GENERATION_INPUT_SCHEMA)
1229
1341
  .output(RUNTIME_GENERATION_RESULT_SCHEMA),