@meetkai/mka1 0.55.32 → 0.55.34

package/src/models/components/evalcandidateobject.ts

@@ -0,0 +1,163 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import * as discriminatedUnionTypes from "../../types/discriminatedUnion.js";
+import { discriminatedUnion } from "../../types/discriminatedUnion.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+
+export type EvalCandidateObject2 = {
+  type: "saved_agent";
+  agentId: string;
+  version: number;
+};
+
+export type EvalCandidateObject1 = {
+  type: "model";
+  model: string;
+};
+
+export type EvalCandidateObject =
+  | EvalCandidateObject1
+  | EvalCandidateObject2
+  | discriminatedUnionTypes.Unknown<"type">;
+
+/** @internal */
+export const EvalCandidateObject2$inboundSchema: z.ZodType<
+  EvalCandidateObject2,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  type: z.literal("saved_agent"),
+  agent_id: z.string(),
+  version: z.number().int(),
+}).transform((v) => {
+  return remap$(v, {
+    "agent_id": "agentId",
+  });
+});
+/** @internal */
+export type EvalCandidateObject2$Outbound = {
+  type: "saved_agent";
+  agent_id: string;
+  version: number;
+};
+
+/** @internal */
+export const EvalCandidateObject2$outboundSchema: z.ZodType<
+  EvalCandidateObject2$Outbound,
+  z.ZodTypeDef,
+  EvalCandidateObject2
+> = z.object({
+  type: z.literal("saved_agent"),
+  agentId: z.string(),
+  version: z.number().int(),
+}).transform((v) => {
+  return remap$(v, {
+    agentId: "agent_id",
+  });
+});
+
+export function evalCandidateObject2ToJSON(
+  evalCandidateObject2: EvalCandidateObject2,
+): string {
+  return JSON.stringify(
+    EvalCandidateObject2$outboundSchema.parse(evalCandidateObject2),
+  );
+}
+export function evalCandidateObject2FromJSON(
+  jsonString: string,
+): SafeParseResult<EvalCandidateObject2, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => EvalCandidateObject2$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'EvalCandidateObject2' from JSON`,
+  );
+}
+
+/** @internal */
+export const EvalCandidateObject1$inboundSchema: z.ZodType<
+  EvalCandidateObject1,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  type: z.literal("model"),
+  model: z.string(),
+});
+/** @internal */
+export type EvalCandidateObject1$Outbound = {
+  type: "model";
+  model: string;
+};
+
+/** @internal */
+export const EvalCandidateObject1$outboundSchema: z.ZodType<
+  EvalCandidateObject1$Outbound,
+  z.ZodTypeDef,
+  EvalCandidateObject1
+> = z.object({
+  type: z.literal("model"),
+  model: z.string(),
+});
+
+export function evalCandidateObject1ToJSON(
+  evalCandidateObject1: EvalCandidateObject1,
+): string {
+  return JSON.stringify(
+    EvalCandidateObject1$outboundSchema.parse(evalCandidateObject1),
+  );
+}
+export function evalCandidateObject1FromJSON(
+  jsonString: string,
+): SafeParseResult<EvalCandidateObject1, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => EvalCandidateObject1$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'EvalCandidateObject1' from JSON`,
+  );
+}
+
+/** @internal */
+export const EvalCandidateObject$inboundSchema: z.ZodType<
+  EvalCandidateObject,
+  z.ZodTypeDef,
+  unknown
+> = discriminatedUnion("type", {
+  model: z.lazy(() => EvalCandidateObject1$inboundSchema),
+  saved_agent: z.lazy(() => EvalCandidateObject2$inboundSchema),
+});
+/** @internal */
+export type EvalCandidateObject$Outbound =
+  | EvalCandidateObject1$Outbound
+  | EvalCandidateObject2$Outbound;
+
+/** @internal */
+export const EvalCandidateObject$outboundSchema: z.ZodType<
+  EvalCandidateObject$Outbound,
+  z.ZodTypeDef,
+  EvalCandidateObject
+> = z.union([
+  z.lazy(() => EvalCandidateObject1$outboundSchema),
+  z.lazy(() => EvalCandidateObject2$outboundSchema),
+]);
+
+export function evalCandidateObjectToJSON(
+  evalCandidateObject: EvalCandidateObject,
+): string {
+  return JSON.stringify(
+    EvalCandidateObject$outboundSchema.parse(evalCandidateObject),
+  );
+}
+export function evalCandidateObjectFromJSON(
+  jsonString: string,
+): SafeParseResult<EvalCandidateObject, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => EvalCandidateObject$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'EvalCandidateObject' from JSON`,
+  );
+}

package/src/models/components/evalmetric.ts

@@ -19,6 +19,10 @@ export type EvalMetric = {
   id: string;
   aggregation?: Aggregation | undefined;
   higherIsBetter?: boolean | undefined;
+  /**
+   * Whether this metric requires a non-empty reference target. Defaults to true for standard reference metrics (accuracy, exact_match, F1, precision, recall, BLEU, ROUGE and transcription error rates), false otherwise. Set explicitly for custom Python metrics.
+   */
+  requiresTarget?: boolean | undefined;
   description?: string | undefined;
 };
 
@@ -38,10 +42,12 @@ export const EvalMetric$inboundSchema: z.ZodType<
   id: z.string(),
   aggregation: Aggregation$inboundSchema.default("mean"),
   higher_is_better: z.boolean().default(true),
+  requires_target: z.boolean().optional(),
   description: z.string().optional(),
 }).transform((v) => {
   return remap$(v, {
     "higher_is_better": "higherIsBetter",
+    "requires_target": "requiresTarget",
   });
 });
 /** @internal */
@@ -49,6 +55,7 @@ export type EvalMetric$Outbound = {
   id: string;
   aggregation: string;
   higher_is_better: boolean;
+  requires_target?: boolean | undefined;
   description?: string | undefined;
 };
 
@@ -61,10 +68,12 @@ export const EvalMetric$outboundSchema: z.ZodType<
   id: z.string(),
   aggregation: Aggregation$outboundSchema.default("mean"),
   higherIsBetter: z.boolean().default(true),
+  requiresTarget: z.boolean().optional(),
   description: z.string().optional(),
 }).transform((v) => {
   return remap$(v, {
     higherIsBetter: "higher_is_better",
+    requiresTarget: "requires_target",
   });
 });
 

package/src/models/components/index.ts

@@ -286,6 +286,8 @@ export * from "./environmentsetuprunmodel.js";
 export * from "./error.js";
 export * from "./errorevent.js";
 export * from "./evalartifactsobject.js";
+export * from "./evalcandidate.js";
+export * from "./evalcandidateobject.js";
 export * from "./evaldataset.js";
 export * from "./evaldatasetformat.js";
 export * from "./evalembeddingconfig.js";

package/src/models/operations/appendagentevaltrials.ts

@@ -126,6 +126,10 @@ export type AppendAgentEvalTrialsResponseBody = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -598,6 +602,7 @@ export const AppendAgentEvalTrialsResponseBody$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -653,6 +658,7 @@ export type AppendAgentEvalTrialsResponseBody$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -689,6 +695,7 @@ export const AppendAgentEvalTrialsResponseBody$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/cancelevalrun.ts

@@ -125,6 +125,10 @@ export type CancelEvalRunResponseBody = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -581,6 +585,7 @@ export const CancelEvalRunResponseBody$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -634,6 +639,7 @@ export type CancelEvalRunResponseBody$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -670,6 +676,7 @@ export const CancelEvalRunResponseBody$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/createevalrun.ts

@@ -121,6 +121,10 @@ export type CreateEvalRunResponseBody = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -544,6 +548,7 @@ export const CreateEvalRunResponseBody$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -597,6 +602,7 @@ export type CreateEvalRunResponseBody$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -633,6 +639,7 @@ export const CreateEvalRunResponseBody$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/getevalrun.ts

@@ -121,6 +121,10 @@ export type GetEvalRunResponseBody = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -573,6 +577,7 @@ export const GetEvalRunResponseBody$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -626,6 +631,7 @@ export type GetEvalRunResponseBody$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -662,6 +668,7 @@ export const GetEvalRunResponseBody$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/importagentevalrun.ts

@@ -125,6 +125,10 @@ export type ImportAgentEvalRunResponseBody = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -587,6 +591,7 @@ export const ImportAgentEvalRunResponseBody$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -640,6 +645,7 @@ export type ImportAgentEvalRunResponseBody$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -676,6 +682,7 @@ export const ImportAgentEvalRunResponseBody$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/listevalruns.ts

@@ -138,6 +138,10 @@ export type ListEvalRunsData = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -633,6 +637,7 @@ export const ListEvalRunsData$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -686,6 +691,7 @@ export type ListEvalRunsData$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -722,6 +728,7 @@ export const ListEvalRunsData$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/listevalscheduleruns.ts

@@ -140,6 +140,10 @@ export type ListEvalScheduleRunsData = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -644,6 +648,7 @@ export const ListEvalScheduleRunsData$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -697,6 +702,7 @@ export type ListEvalScheduleRunsData$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -733,6 +739,7 @@ export const ListEvalScheduleRunsData$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/listevalsuiteleaderboardruns.ts

@@ -126,6 +126,10 @@ export type ListEvalSuiteLeaderboardRunsData = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -657,6 +661,7 @@ export const ListEvalSuiteLeaderboardRunsData$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -714,6 +719,7 @@ export type ListEvalSuiteLeaderboardRunsData$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -750,6 +756,7 @@ export const ListEvalSuiteLeaderboardRunsData$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/rerunfailedevalsamples.ts

@@ -125,6 +125,10 @@ export type RerunFailedEvalSamplesResponseBody = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -593,6 +597,7 @@ export const RerunFailedEvalSamplesResponseBody$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -648,6 +653,7 @@ export type RerunFailedEvalSamplesResponseBody$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -684,6 +690,7 @@ export const RerunFailedEvalSamplesResponseBody$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/retryfailedevalrun.ts

@@ -125,6 +125,10 @@ export type RetryFailedEvalRunResponseBody = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -587,6 +591,7 @@ export const RetryFailedEvalRunResponseBody$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -640,6 +645,7 @@ export type RetryFailedEvalRunResponseBody$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -676,6 +682,7 @@ export const RetryFailedEvalRunResponseBody$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/react-query/llmEvalsCreateRun.ts

@@ -48,7 +48,7 @@ export type LlmEvalsCreateRunMutationError =
  * Create an eval run
  *
  * @remarks
- * Starts a durable eval run over the selected suite version, tasks, and models.
+ * Starts a durable eval run over the selected suite version and tasks, using model or pinned saved-agent candidates. Supply exactly one of candidates or the legacy models field.
  */
 export function useLlmEvalsCreateRunMutation(
   options?: MutationHookOptions<

package/src/sdk/evals.ts

@@ -269,7 +269,7 @@ export class Evals extends ClientSDK {
    * Create an eval run
    *
    * @remarks
-   * Starts a durable eval run over the selected suite version, tasks, and models.
+   * Starts a durable eval run over the selected suite version and tasks, using model or pinned saved-agent candidates. Supply exactly one of candidates or the legacy models field.
    */
   async createRun(
     request: operations.CreateEvalRunRequest,