@meetkai/mka1 0.55.33 → 0.55.35

package/src/models/components/evalcandidateobject.ts

@@ -0,0 +1,163 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import * as discriminatedUnionTypes from "../../types/discriminatedUnion.js";
+import { discriminatedUnion } from "../../types/discriminatedUnion.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
+
+export type EvalCandidateObject2 = {
+  type: "saved_agent";
+  agentId: string;
+  version: number;
+};
+
+export type EvalCandidateObject1 = {
+  type: "model";
+  model: string;
+};
+
+export type EvalCandidateObject =
+  | EvalCandidateObject1
+  | EvalCandidateObject2
+  | discriminatedUnionTypes.Unknown<"type">;
+
+/** @internal */
+export const EvalCandidateObject2$inboundSchema: z.ZodType<
+  EvalCandidateObject2,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  type: z.literal("saved_agent"),
+  agent_id: z.string(),
+  version: z.number().int(),
+}).transform((v) => {
+  return remap$(v, {
+    "agent_id": "agentId",
+  });
+});
+/** @internal */
+export type EvalCandidateObject2$Outbound = {
+  type: "saved_agent";
+  agent_id: string;
+  version: number;
+};
+
+/** @internal */
+export const EvalCandidateObject2$outboundSchema: z.ZodType<
+  EvalCandidateObject2$Outbound,
+  z.ZodTypeDef,
+  EvalCandidateObject2
+> = z.object({
+  type: z.literal("saved_agent"),
+  agentId: z.string(),
+  version: z.number().int(),
+}).transform((v) => {
+  return remap$(v, {
+    agentId: "agent_id",
+  });
+});
+
+export function evalCandidateObject2ToJSON(
+  evalCandidateObject2: EvalCandidateObject2,
+): string {
+  return JSON.stringify(
+    EvalCandidateObject2$outboundSchema.parse(evalCandidateObject2),
+  );
+}
+export function evalCandidateObject2FromJSON(
+  jsonString: string,
+): SafeParseResult<EvalCandidateObject2, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => EvalCandidateObject2$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'EvalCandidateObject2' from JSON`,
+  );
+}
+
+/** @internal */
+export const EvalCandidateObject1$inboundSchema: z.ZodType<
+  EvalCandidateObject1,
+  z.ZodTypeDef,
+  unknown
+> = z.object({
+  type: z.literal("model"),
+  model: z.string(),
+});
+/** @internal */
+export type EvalCandidateObject1$Outbound = {
+  type: "model";
+  model: string;
+};
+
+/** @internal */
+export const EvalCandidateObject1$outboundSchema: z.ZodType<
+  EvalCandidateObject1$Outbound,
+  z.ZodTypeDef,
+  EvalCandidateObject1
+> = z.object({
+  type: z.literal("model"),
+  model: z.string(),
+});
+
+export function evalCandidateObject1ToJSON(
+  evalCandidateObject1: EvalCandidateObject1,
+): string {
+  return JSON.stringify(
+    EvalCandidateObject1$outboundSchema.parse(evalCandidateObject1),
+  );
+}
+export function evalCandidateObject1FromJSON(
+  jsonString: string,
+): SafeParseResult<EvalCandidateObject1, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => EvalCandidateObject1$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'EvalCandidateObject1' from JSON`,
+  );
+}
+
+/** @internal */
+export const EvalCandidateObject$inboundSchema: z.ZodType<
+  EvalCandidateObject,
+  z.ZodTypeDef,
+  unknown
+> = discriminatedUnion("type", {
+  model: z.lazy(() => EvalCandidateObject1$inboundSchema),
+  saved_agent: z.lazy(() => EvalCandidateObject2$inboundSchema),
+});
+/** @internal */
+export type EvalCandidateObject$Outbound =
+  | EvalCandidateObject1$Outbound
+  | EvalCandidateObject2$Outbound;
+
+/** @internal */
+export const EvalCandidateObject$outboundSchema: z.ZodType<
+  EvalCandidateObject$Outbound,
+  z.ZodTypeDef,
+  EvalCandidateObject
+> = z.union([
+  z.lazy(() => EvalCandidateObject1$outboundSchema),
+  z.lazy(() => EvalCandidateObject2$outboundSchema),
+]);
+
+export function evalCandidateObjectToJSON(
+  evalCandidateObject: EvalCandidateObject,
+): string {
+  return JSON.stringify(
+    EvalCandidateObject$outboundSchema.parse(evalCandidateObject),
+  );
+}
+export function evalCandidateObjectFromJSON(
+  jsonString: string,
+): SafeParseResult<EvalCandidateObject, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => EvalCandidateObject$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'EvalCandidateObject' from JSON`,
+  );
+}

package/src/models/components/evalmetric.ts

@@ -19,6 +19,10 @@ export type EvalMetric = {
   id: string;
   aggregation?: Aggregation | undefined;
   higherIsBetter?: boolean | undefined;
+  /**
+   * Whether this metric requires a non-empty reference target. Defaults to true for standard reference metrics (accuracy, exact_match, F1, precision, recall, BLEU, ROUGE and transcription error rates), false otherwise. Set explicitly for custom Python metrics.
+   */
+  requiresTarget?: boolean | undefined;
   description?: string | undefined;
 };
 
@@ -38,10 +42,12 @@ export const EvalMetric$inboundSchema: z.ZodType<
   id: z.string(),
   aggregation: Aggregation$inboundSchema.default("mean"),
   higher_is_better: z.boolean().default(true),
+  requires_target: z.boolean().optional(),
   description: z.string().optional(),
 }).transform((v) => {
   return remap$(v, {
     "higher_is_better": "higherIsBetter",
+    "requires_target": "requiresTarget",
   });
 });
 /** @internal */
@@ -49,6 +55,7 @@ export type EvalMetric$Outbound = {
   id: string;
   aggregation: string;
   higher_is_better: boolean;
+  requires_target?: boolean | undefined;
   description?: string | undefined;
 };
 
@@ -61,10 +68,12 @@ export const EvalMetric$outboundSchema: z.ZodType<
   id: z.string(),
   aggregation: Aggregation$outboundSchema.default("mean"),
   higherIsBetter: z.boolean().default(true),
+  requiresTarget: z.boolean().optional(),
   description: z.string().optional(),
 }).transform((v) => {
   return remap$(v, {
     higherIsBetter: "higher_is_better",
+    requiresTarget: "requires_target",
   });
 });
 

package/src/models/components/index.ts

@@ -14,6 +14,7 @@ export * from "./agent.js";
 export * from "./agentcompactioncontextmanagement.js";
 export * from "./agentconnector.js";
 export * from "./agentconnectordeleted.js";
+export * from "./agentconnectorembed.js";
 export * from "./agentconnectorlist.js";
 export * from "./agentlist.js";
 export * from "./agentreasoningconfiguration.js";
@@ -243,6 +244,7 @@ export * from "./createvectorstorefilerequest.js";
 export * from "./createvectorstorepermissionrequest.js";
 export * from "./createvectorstorerequest.js";
 export * from "./createversionresponse.js";
+export * from "./createwebagentconnectorrequest.js";
 export * from "./createwhatsappagentconnectorrequest.js";
 export * from "./customtoolcall.js";
 export * from "./customtoolcalloutput.js";
@@ -286,6 +288,8 @@ export * from "./environmentsetuprunmodel.js";
 export * from "./error.js";
 export * from "./errorevent.js";
 export * from "./evalartifactsobject.js";
+export * from "./evalcandidate.js";
+export * from "./evalcandidateobject.js";
 export * from "./evaldataset.js";
 export * from "./evaldatasetformat.js";
 export * from "./evalembeddingconfig.js";

package/src/models/operations/appendagentevaltrials.ts

@@ -126,6 +126,10 @@ export type AppendAgentEvalTrialsResponseBody = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -598,6 +602,7 @@ export const AppendAgentEvalTrialsResponseBody$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -653,6 +658,7 @@ export type AppendAgentEvalTrialsResponseBody$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -689,6 +695,7 @@ export const AppendAgentEvalTrialsResponseBody$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/cancelevalrun.ts

@@ -125,6 +125,10 @@ export type CancelEvalRunResponseBody = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -581,6 +585,7 @@ export const CancelEvalRunResponseBody$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -634,6 +639,7 @@ export type CancelEvalRunResponseBody$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -670,6 +676,7 @@ export const CancelEvalRunResponseBody$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/createevalrun.ts

@@ -121,6 +121,10 @@ export type CreateEvalRunResponseBody = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -544,6 +548,7 @@ export const CreateEvalRunResponseBody$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -597,6 +602,7 @@ export type CreateEvalRunResponseBody$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -633,6 +639,7 @@ export const CreateEvalRunResponseBody$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/getevalrun.ts

@@ -121,6 +121,10 @@ export type GetEvalRunResponseBody = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -573,6 +577,7 @@ export const GetEvalRunResponseBody$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -626,6 +631,7 @@ export type GetEvalRunResponseBody$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -662,6 +668,7 @@ export const GetEvalRunResponseBody$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/importagentevalrun.ts

@@ -125,6 +125,10 @@ export type ImportAgentEvalRunResponseBody = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -587,6 +591,7 @@ export const ImportAgentEvalRunResponseBody$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -640,6 +645,7 @@ export type ImportAgentEvalRunResponseBody$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -676,6 +682,7 @@ export const ImportAgentEvalRunResponseBody$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/listevalruns.ts

@@ -138,6 +138,10 @@ export type ListEvalRunsData = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -633,6 +637,7 @@ export const ListEvalRunsData$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -686,6 +691,7 @@ export type ListEvalRunsData$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -722,6 +728,7 @@ export const ListEvalRunsData$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/listevalscheduleruns.ts

@@ -140,6 +140,10 @@ export type ListEvalScheduleRunsData = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -644,6 +648,7 @@ export const ListEvalScheduleRunsData$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -697,6 +702,7 @@ export type ListEvalScheduleRunsData$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -733,6 +739,7 @@ export const ListEvalScheduleRunsData$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/listevalsuiteleaderboardruns.ts

@@ -126,6 +126,10 @@ export type ListEvalSuiteLeaderboardRunsData = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -657,6 +661,7 @@ export const ListEvalSuiteLeaderboardRunsData$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -714,6 +719,7 @@ export type ListEvalSuiteLeaderboardRunsData$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -750,6 +756,7 @@ export const ListEvalSuiteLeaderboardRunsData$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/rerunfailedevalsamples.ts

@@ -125,6 +125,10 @@ export type RerunFailedEvalSamplesResponseBody = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -593,6 +597,7 @@ export const RerunFailedEvalSamplesResponseBody$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -648,6 +653,7 @@ export type RerunFailedEvalSamplesResponseBody$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -684,6 +690,7 @@ export const RerunFailedEvalSamplesResponseBody$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/models/operations/retryfailedevalrun.ts

@@ -125,6 +125,10 @@ export type RetryFailedEvalRunResponseBody = {
   teamId: string;
   status: components.EvalRunStatus;
   models: Array<string>;
+  /**
+   * Typed candidates, including pinned saved-agent versions. models remains the legacy storage/filter key list.
+   */
+  candidates?: Array<components.EvalCandidateObject> | undefined;
   taskIds: Array<string> | null;
   judgeModel: string | null;
   embeddingModel: string | null;
@@ -587,6 +591,7 @@ export const RetryFailedEvalRunResponseBody$inboundSchema: z.ZodType<
   team_id: z.string(),
   status: components.EvalRunStatus$inboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$inboundSchema).optional(),
   task_ids: z.nullable(z.array(z.string())),
   judge_model: z.nullable(z.string()),
   embedding_model: z.nullable(z.string()),
@@ -640,6 +645,7 @@ export type RetryFailedEvalRunResponseBody$Outbound = {
   team_id: string;
   status: string;
   models: Array<string>;
+  candidates?: Array<components.EvalCandidateObject$Outbound> | undefined;
   task_ids: Array<string> | null;
   judge_model: string | null;
   embedding_model: string | null;
@@ -676,6 +682,7 @@ export const RetryFailedEvalRunResponseBody$outboundSchema: z.ZodType<
   teamId: z.string(),
   status: components.EvalRunStatus$outboundSchema,
   models: z.array(z.string()),
+  candidates: z.array(components.EvalCandidateObject$outboundSchema).optional(),
   taskIds: z.nullable(z.array(z.string())),
   judgeModel: z.nullable(z.string()),
   embeddingModel: z.nullable(z.string()),

package/src/react-query/agentConnectorsActivateAgentConnector.ts

@@ -51,7 +51,7 @@ export type AgentConnectorsActivateAgentConnectorMutationError =
  * Activate a connector
  *
  * @remarks
- * Revalidates the stored provider credentials and registers the provider webhook. Use this to retry a connector left in an error state. API-key authentication with every listed scope is required; session authentication is rejected, and the key must belong to the connector's original user or service-account principal so existing conversations keep the same owner.
+ * Revalidates the stored provider credentials and registers the provider webhook. Use this to retry a connector left in an error state. API-key authentication with every listed scope is required; session authentication is rejected, and the key must belong to the connector's original user or service-account principal so existing conversations keep the same owner. Web connectors have no provider webhook; activation only re-authorizes the stored API-key identity.
  */
 export function useAgentConnectorsActivateAgentConnectorMutation(
   options?: MutationHookOptions<

package/src/react-query/agentConnectorsCreateAgentConnector.ts

@@ -51,7 +51,7 @@ export type AgentConnectorsCreateAgentConnectorMutationError =
  * Create a connector
  *
  * @remarks
- * Connects the agent to a Telegram bot or WhatsApp Business phone number, validates the provider credentials, and registers the agent's webhook. WhatsApp connector support is beta. Text, photos, and gateway-supported documents can invoke the agent; inbound media additionally requires write:files. Managed outbound files require read:files, and generated images require both file scopes so their transient provider URLs can be materialized before delivery. The optional write:feedback scope enables native thumbs reaction synchronization to gateway response feedback. These conditional scopes are deliberately not part of x-required-scopes because text-only connectors do not need them. An API key with every listed scope is required; its identity tuple and current scopes are stored as a non-widening ceiling, then revalidated with mk-authentication before every connector run; session and delegated end-user contexts are rejected. The required access policy controls which provider users or chats may invoke the agent.
+ * Connects the agent to a Telegram bot or WhatsApp Business phone number, validates the provider credentials, and registers the agent's webhook. WhatsApp connector support is beta. Text, photos, and gateway-supported documents can invoke the agent; inbound media additionally requires write:files. Managed outbound files require read:files, and generated images require both file scopes so their transient provider URLs can be materialized before delivery. The optional write:feedback scope enables native thumbs reaction synchronization to gateway response feedback. These conditional scopes are deliberately not part of x-required-scopes because text-only connectors do not need them. An API key with every listed scope is required; its identity tuple and current scopes are stored as a non-widening ceiling, then revalidated with mk-authentication before every connector run; session and delegated end-user contexts are rejected. The required access policy controls which provider users or chats may invoke the agent. A web connector (provider web) needs no provider credentials or access policy: it is active immediately and returns an embed object with an iframe URL and snippet for an anonymous web chat widget whose visitors are limited by signed sessions, rate limits and the optional allowed_origins list. Creating a web connector returns 409 when the deployment has not enabled web chat.
  */
 export function useAgentConnectorsCreateAgentConnectorMutation(
   options?: MutationHookOptions<

package/src/react-query/agentConnectorsDeleteAgentConnector.ts

@@ -51,7 +51,7 @@ export type AgentConnectorsDeleteAgentConnectorMutationError =
  * Delete a connector
  *
  * @remarks
- * Unregisters the connector-specific provider webhook and deletes the stored connector credentials. Telegram pending updates are dropped; WhatsApp deletion clears only the phone-number callback and does not unsubscribe other numbers in the business account.
+ * Unregisters the connector-specific provider webhook and deletes the stored connector credentials. Telegram pending updates are dropped; WhatsApp deletion clears only the phone-number callback and does not unsubscribe other numbers in the business account. Deleting a web connector disables its embed immediately.
  */
 export function useAgentConnectorsDeleteAgentConnectorMutation(
   options?: MutationHookOptions<

package/src/react-query/llmEvalsCreateRun.ts

@@ -48,7 +48,7 @@ export type LlmEvalsCreateRunMutationError =
  * Create an eval run
  *
  * @remarks
- * Starts a durable eval run over the selected suite version, tasks, and models.
+ * Starts a durable eval run over the selected suite version and tasks, using model or pinned saved-agent candidates. Supply exactly one of candidates or the legacy models field.
  */
 export function useLlmEvalsCreateRunMutation(
   options?: MutationHookOptions<

package/src/sdk/agentconnectors.ts

@@ -35,7 +35,7 @@ export class AgentConnectors extends ClientSDK {
    * Create a connector
    *
    * @remarks
-   * Connects the agent to a Telegram bot or WhatsApp Business phone number, validates the provider credentials, and registers the agent's webhook. WhatsApp connector support is beta. Text, photos, and gateway-supported documents can invoke the agent; inbound media additionally requires write:files. Managed outbound files require read:files, and generated images require both file scopes so their transient provider URLs can be materialized before delivery. The optional write:feedback scope enables native thumbs reaction synchronization to gateway response feedback. These conditional scopes are deliberately not part of x-required-scopes because text-only connectors do not need them. An API key with every listed scope is required; its identity tuple and current scopes are stored as a non-widening ceiling, then revalidated with mk-authentication before every connector run; session and delegated end-user contexts are rejected. The required access policy controls which provider users or chats may invoke the agent.
+   * Connects the agent to a Telegram bot or WhatsApp Business phone number, validates the provider credentials, and registers the agent's webhook. WhatsApp connector support is beta. Text, photos, and gateway-supported documents can invoke the agent; inbound media additionally requires write:files. Managed outbound files require read:files, and generated images require both file scopes so their transient provider URLs can be materialized before delivery. The optional write:feedback scope enables native thumbs reaction synchronization to gateway response feedback. These conditional scopes are deliberately not part of x-required-scopes because text-only connectors do not need them. An API key with every listed scope is required; its identity tuple and current scopes are stored as a non-widening ceiling, then revalidated with mk-authentication before every connector run; session and delegated end-user contexts are rejected. The required access policy controls which provider users or chats may invoke the agent. A web connector (provider web) needs no provider credentials or access policy: it is active immediately and returns an embed object with an iframe URL and snippet for an anonymous web chat widget whose visitors are limited by signed sessions, rate limits and the optional allowed_origins list. Creating a web connector returns 409 when the deployment has not enabled web chat.
    */
   async createAgentConnector(
     request: operations.CreateAgentConnectorRequest,
@@ -86,7 +86,7 @@ export class AgentConnectors extends ClientSDK {
    * Delete a connector
    *
    * @remarks
-   * Unregisters the connector-specific provider webhook and deletes the stored connector credentials. Telegram pending updates are dropped; WhatsApp deletion clears only the phone-number callback and does not unsubscribe other numbers in the business account.
+   * Unregisters the connector-specific provider webhook and deletes the stored connector credentials. Telegram pending updates are dropped; WhatsApp deletion clears only the phone-number callback and does not unsubscribe other numbers in the business account. Deleting a web connector disables its embed immediately.
    */
   async deleteAgentConnector(
     request: operations.DeleteAgentConnectorRequest,
@@ -103,7 +103,7 @@ export class AgentConnectors extends ClientSDK {
    * Activate a connector
    *
    * @remarks
-   * Revalidates the stored provider credentials and registers the provider webhook. Use this to retry a connector left in an error state. API-key authentication with every listed scope is required; session authentication is rejected, and the key must belong to the connector's original user or service-account principal so existing conversations keep the same owner.
+   * Revalidates the stored provider credentials and registers the provider webhook. Use this to retry a connector left in an error state. API-key authentication with every listed scope is required; session authentication is rejected, and the key must belong to the connector's original user or service-account principal so existing conversations keep the same owner. Web connectors have no provider webhook; activation only re-authorizes the stored API-key identity.
    */
   async activateAgentConnector(
     request: operations.ActivateAgentConnectorRequest,

package/src/sdk/evals.ts

@@ -269,7 +269,7 @@ export class Evals extends ClientSDK {
    * Create an eval run
    *
    * @remarks
-   * Starts a durable eval run over the selected suite version, tasks, and models.
+   * Starts a durable eval run over the selected suite version and tasks, using model or pinned saved-agent candidates. Supply exactly one of candidates or the legacy models field.
    */
   async createRun(
     request: operations.CreateEvalRunRequest,