@dalmia/calibrate-mcp 0.0.13 → 0.0.15

package/src/mcp-server/tools/evaluatorsList.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { evaluatorsList } from "../../funcs/evaluatorsList.js";
+import { ListEvaluatorsEvaluatorsGetRequest$zodSchema } from "../../models/listevaluatorsevaluatorsgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: ListEvaluatorsEvaluatorsGetRequest$zodSchema.optional(),
+};
+
+export const tool$evaluatorsList: ToolDefinition<typeof args> = {
+  name: "list-evaluators",
+  description:
+    `List your evaluators. Optionally filter by what they judge or their modality, and include the built-in defaults.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "List evaluators",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await evaluatorsList(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/models/agentcreate.ts

@@ -38,7 +38,7 @@ export type AgentCreate = {
 
 export const AgentCreate$zodSchema: z.ZodType<AgentCreate> = z.object({
   config: z.record(z.string(), z.any()).nullable().optional().describe(
-    "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`** (built inside Calibrate):\n- `system_prompt` (string): the agent's instructions\n- `llm.model` (string): `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider` (string): `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider` (string): `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first` (bool), `settings.max_assistant_turns` (int)\n- `system_tools.end_call` (bool, optional): let the agent end the call\n- `data_extraction_fields` (array, optional): `[{name, type, description, required}]`\n\n```json\n{\n  \"system_prompt\": \"You are a helpful support agent.\",\n  \"llm\": {\"model\": \"openai/gpt-4.1\"},\n  \"stt\": {\"provider\": \"deepgram\"},\n  \"tts\": {\"provider\": \"elevenlabs\"},\n  \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`** (your own HTTP endpoint):\n- `agent_url` (string, required): public HTTPS endpoint the agent is called at\n- `agent_headers` (object, optional): headers sent on each request, e.g. auth\n- `benchmark_provider` (string, optional): `openrouter` (default), `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n  \"agent_url\": \"https://api.example.com/agent\",\n  \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n  \"benchmark_provider\": \"openrouter\"\n}\n```\n\nFor `type=agent`, omitted keys inherit managed defaults (omit `config` entirely to use all defaults). For `type=connection`, `config` is stored as-is and must contain `agent_url`.",
+    "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n  \"system_prompt\": \"You are a helpful support agent.\",\n  \"llm\": {\"model\": \"openai/gpt-4.1\"},\n  \"stt\": {\"provider\": \"deepgram\"},\n  \"tts\": {\"provider\": \"elevenlabs\"},\n  \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n  \"agent_url\": \"https://api.example.com/agent\",\n  \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n  \"benchmark_provider\": \"openrouter\"\n}\n```\n\nFor `type=agent`, omitted keys inherit managed defaults. Omit `config` entirely to use all defaults. For `type=connection`, `config` is stored as-is and must contain `agent_url`",
   ),
   name: z.string().describe("Agent name, unique within the workspace"),
   type: AgentCreateType$zodSchema.default("agent").describe(

package/src/models/agenttestruncreateresponse.ts

@@ -6,14 +6,14 @@ import * as z from "zod";
 import { ClosedEnum } from "../types/enums.js";
 
 /**
- * Current status of the test run
+ * Current status of the run
  */
 export const AgentTestRunCreateResponseStatus = {
   Queued: "queued",
   InProgress: "in_progress",
 } as const;
 /**
- * Current status of the test run
+ * Current status of the run
  */
 export type AgentTestRunCreateResponseStatus = ClosedEnum<
   typeof AgentTestRunCreateResponseStatus
@@ -22,7 +22,7 @@ export type AgentTestRunCreateResponseStatus = ClosedEnum<
 export const AgentTestRunCreateResponseStatus$zodSchema = z.enum([
   "queued",
   "in_progress",
-]).describe("Current status of the test run");
+]).describe("Current status of the run");
 
 export type AgentTestRunCreateResponse = {
   task_id: string;
@@ -33,7 +33,9 @@ export const AgentTestRunCreateResponse$zodSchema: z.ZodType<
   AgentTestRunCreateResponse
 > = z.object({
   status: AgentTestRunCreateResponseStatus$zodSchema.describe(
-    "Current status of the test run",
+    "Current status of the run",
+  ),
+  task_id: z.string().describe(
+    "Test run job ID. Poll it for status and results",
   ),
-  task_id: z.string().describe("Test run job ID. Poll for status and results"),
 });

package/src/models/agenttestrunlistitem.ts

@@ -0,0 +1,116 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+import { ModelResult, ModelResult$zodSchema } from "./modelresult.js";
+import { TaskStatus, TaskStatus$zodSchema } from "./taskstatus.js";
+import { TestCaseResult, TestCaseResult$zodSchema } from "./testcaseresult.js";
+import {
+  TestRunEvaluator,
+  TestRunEvaluator$zodSchema,
+} from "./testrunevaluator.js";
+
+/**
+ * What kind of run this is:
+ *
+ * @remarks
+ * - `llm-unit-test`: a single run of the agent's tests
+ * - `llm-benchmark`: a multi-model comparison
+ */
+export const AgentTestRunListItemType = {
+  LlmUnitTest: "llm-unit-test",
+  LlmBenchmark: "llm-benchmark",
+} as const;
+/**
+ * What kind of run this is:
+ *
+ * @remarks
+ * - `llm-unit-test`: a single run of the agent's tests
+ * - `llm-benchmark`: a multi-model comparison
+ */
+export type AgentTestRunListItemType = ClosedEnum<
+  typeof AgentTestRunListItemType
+>;
+
+export const AgentTestRunListItemType$zodSchema = z.enum([
+  "llm-unit-test",
+  "llm-benchmark",
+]).describe(
+  "What kind of run this is:\n- `llm-unit-test`: a single run of the agent's tests\n- `llm-benchmark`: a multi-model comparison",
+);
+
+export type AgentTestRunListItem = {
+  uuid: string;
+  name: string;
+  status: TaskStatus;
+  type: AgentTestRunListItemType;
+  updated_at: string;
+  evaluators?: Array<TestRunEvaluator> | null | undefined;
+  total_tests?: number | null | undefined;
+  passed?: number | null | undefined;
+  failed?: number | null | undefined;
+  results?: Array<TestCaseResult> | null | undefined;
+  latency_ms?: { [k: string]: any } | null | undefined;
+  cost?: { [k: string]: any } | null | undefined;
+  total_tokens?: { [k: string]: any } | null | undefined;
+  model_results?: Array<ModelResult> | null | undefined;
+  leaderboard_summary?: Array<{ [k: string]: any }> | null | undefined;
+  error?: boolean | undefined;
+  is_public?: boolean | undefined;
+  share_token?: string | null | undefined;
+};
+
+export const AgentTestRunListItem$zodSchema: z.ZodType<AgentTestRunListItem> = z
+  .object({
+    cost: z.record(z.string(), z.any()).nullable().optional().describe(
+      "Aggregated cost as `{mean, min, max, count}` (USD)",
+    ),
+    error: z.boolean().default(false).describe("True if the run failed"),
+    evaluators: z.array(TestRunEvaluator$zodSchema).nullable().optional()
+      .describe(
+        "The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`",
+      ),
+    failed: z.int().nullable().optional().describe(
+      "Number of test cases that failed",
+    ),
+    is_public: z.boolean().default(false).describe(
+      "Whether the run is shared publicly",
+    ),
+    latency_ms: z.record(z.string(), z.any()).nullable().optional().describe(
+      "Aggregated latency in milliseconds, as `{p50, p95, p99, count}`",
+    ),
+    leaderboard_summary: z.array(z.record(z.string(), z.any())).nullable()
+      .optional().describe(
+        "Leaderboard comparing the models, one row per model. Columns vary by benchmark: a `model` column plus pass/fail counts, latency, cost, and one score column per evaluator, keyed by evaluator name",
+      ),
+    model_results: z.array(ModelResult$zodSchema).nullable().optional()
+      .describe("Results for each model in a benchmark run"),
+    name: z.string().describe(
+      "Display name, such as `Run 1` for a unit test or `Benchmark 1` for a benchmark",
+    ),
+    passed: z.int().nullable().optional().describe(
+      "Number of test cases that passed",
+    ),
+    results: z.array(TestCaseResult$zodSchema).nullable().optional().describe(
+      "Results for each test case",
+    ),
+    share_token: z.string().nullable().optional().describe(
+      "Token for building the public share URL",
+    ),
+    status: TaskStatus$zodSchema,
+    total_tests: z.int().nullable().optional().describe(
+      "Total number of test cases",
+    ),
+    total_tokens: z.record(z.string(), z.any()).nullable().optional().describe(
+      "Aggregated token usage as `{mean, min, max, count}`",
+    ),
+    type: AgentTestRunListItemType$zodSchema.describe(
+      "What kind of run this is:\n- `llm-unit-test`: a single run of the agent's tests\n- `llm-benchmark`: a multi-model comparison",
+    ),
+    updated_at: z.string().describe(
+      "When the run was last updated (ISO 8601 UTC)",
+    ),
+    uuid: z.string().describe("Test run job ID"),
+  });

package/src/models/agenttestrunsresponse.ts

@@ -0,0 +1,18 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  AgentTestRunListItem,
+  AgentTestRunListItem$zodSchema,
+} from "./agenttestrunlistitem.js";
+
+export type AgentTestRunsResponse = { runs: Array<AgentTestRunListItem> };
+
+export const AgentTestRunsResponse$zodSchema: z.ZodType<AgentTestRunsResponse> =
+  z.object({
+    runs: z.array(AgentTestRunListItem$zodSchema).describe(
+      "Test runs for the agent, most recently created first",
+    ),
+  });

package/src/models/agenttestscreate.ts

@@ -0,0 +1,19 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type AgentTestsCreate = {
+  agent_uuid: string;
+  test_uuids: Array<string>;
+};
+
+export const AgentTestsCreate$zodSchema: z.ZodType<AgentTestsCreate> = z.object(
+  {
+    agent_uuid: z.string().describe("Agent to link tests to"),
+    test_uuids: z.array(z.string()).describe(
+      "Tests to link. Any that are already linked are skipped",
+    ),
+  },
+);

package/src/models/agenttestscreateresponse.ts

@@ -0,0 +1,13 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type AgentTestsCreateResponse = { message: string };
+
+export const AgentTestsCreateResponse$zodSchema: z.ZodType<
+  AgentTestsCreateResponse
+> = z.object({
+  message: z.string().describe("Confirmation message"),
+});

package/src/models/agentupdate.ts

@@ -11,7 +11,7 @@ export type AgentUpdate = {
 
 export const AgentUpdate$zodSchema: z.ZodType<AgentUpdate> = z.object({
   config: z.record(z.string(), z.any()).nullable().optional().describe(
-    "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`** (built inside Calibrate):\n- `system_prompt` (string): the agent's instructions\n- `llm.model` (string): `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider` (string): `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider` (string): `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first` (bool), `settings.max_assistant_turns` (int)\n- `system_tools.end_call` (bool, optional): let the agent end the call\n- `data_extraction_fields` (array, optional): `[{name, type, description, required}]`\n\n```json\n{\n  \"system_prompt\": \"You are a helpful support agent.\",\n  \"llm\": {\"model\": \"openai/gpt-4.1\"},\n  \"stt\": {\"provider\": \"deepgram\"},\n  \"tts\": {\"provider\": \"elevenlabs\"},\n  \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`** (your own HTTP endpoint):\n- `agent_url` (string, required): public HTTPS endpoint the agent is called at\n- `agent_headers` (object, optional): headers sent on each request, e.g. auth\n- `benchmark_provider` (string, optional): `openrouter` (default), `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n  \"agent_url\": \"https://api.example.com/agent\",\n  \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n  \"benchmark_provider\": \"openrouter\"\n}\n```\n\nReplaces the stored config. Omit to leave unchanged.\n\nFor `type=connection`, changing `agent_url` or `agent_headers` resets the connection and benchmark verification flags.",
+    "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n  \"system_prompt\": \"You are a helpful support agent.\",\n  \"llm\": {\"model\": \"openai/gpt-4.1\"},\n  \"stt\": {\"provider\": \"deepgram\"},\n  \"tts\": {\"provider\": \"elevenlabs\"},\n  \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n  \"agent_url\": \"https://api.example.com/agent\",\n  \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n  \"benchmark_provider\": \"openrouter\"\n}\n```\n\nReplaces the stored config. Omit to leave unchanged\n\nFor `type=connection`, changing `agent_url` or `agent_headers` resets the connection and benchmark verification flags",
   ),
   name: z.string().nullable().optional().describe(
     "New agent name. Omit to leave the name unchanged",

package/src/models/agentverifyrequest.ts

@@ -0,0 +1,30 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+/**
+ * Body for verifying an existing agent by ID. The endpoint (`agent_url`,
+ *
+ * @remarks
+ * `agent_headers`) comes from the agent's stored config, so only the probe
+ * inputs are accepted here.
+ */
+export type AgentVerifyRequest = {
+  model?: string | null | undefined;
+  messages?: Array<{ [k: string]: string }> | null | undefined;
+};
+
+export const AgentVerifyRequest$zodSchema: z.ZodType<AgentVerifyRequest> = z
+  .object({
+    messages: z.array(z.record(z.string(), z.string())).nullable().optional()
+      .describe(
+        "Sample chat messages to send during verification. Omit to use the default probe",
+      ),
+    model: z.string().nullable().optional().describe(
+      "Model to verify. Omit for a basic connection check. Provide it for a model-specific check before benchmarking that model",
+    ),
+  }).describe(
+    "Body for verifying an existing agent by ID. The endpoint (`agent_url`,\n`agent_headers`) comes from the agent's stored config, so only the probe\ninputs are accepted here.",
+  );

package/src/models/annotationitempayload.ts

@@ -0,0 +1,20 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type AnnotationItemPayload = {
+  payload?: any | undefined;
+  annotations?: { [k: string]: any } | null | undefined;
+};
+
+export const AnnotationItemPayload$zodSchema: z.ZodType<AnnotationItemPayload> =
+  z.object({
+    annotations: z.record(z.string(), z.any()).nullable().optional().describe(
+      "Human annotations to seed, keyed by evaluator ID. Each evaluator ID must be linked to the task. Put the judgement in `value`, a bool for binary or a number for rating, with optional `reasoning`. Requires `annotator_id`",
+    ),
+    payload: z.any().optional().describe(
+      "The item's payload. Its shape depends on the task `type`. `payload['name']` is **required** for every type and must be unique within the task",
+    ),
+  });

package/src/models/annotationtaskcreate.ts

@@ -0,0 +1,68 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+/**
+ * Task type. Determines the shape of each item's payload.
+ *
+ * @remarks
+ * - `stt`: judge a transcript on its own
+ * - `tts`: judge synthesized speech output
+ * - `llm`: judge one response with its conversation history
+ * - `llm-general`: judge a standalone `input -> output` pair
+ * - `conversation`: judge a full conversation
+ */
+export const AnnotationTaskCreateType = {
+  Stt: "stt",
+  Tts: "tts",
+  Llm: "llm",
+  LlmGeneral: "llm-general",
+  Conversation: "conversation",
+} as const;
+/**
+ * Task type. Determines the shape of each item's payload.
+ *
+ * @remarks
+ * - `stt`: judge a transcript on its own
+ * - `tts`: judge synthesized speech output
+ * - `llm`: judge one response with its conversation history
+ * - `llm-general`: judge a standalone `input -> output` pair
+ * - `conversation`: judge a full conversation
+ */
+export type AnnotationTaskCreateType = ClosedEnum<
+  typeof AnnotationTaskCreateType
+>;
+
+export const AnnotationTaskCreateType$zodSchema = z.enum([
+  "stt",
+  "tts",
+  "llm",
+  "llm-general",
+  "conversation",
+]).describe(
+  "Task type. Determines the shape of each item's payload.\n- `stt`: judge a transcript on its own\n- `tts`: judge synthesized speech output\n- `llm`: judge one response with its conversation history\n- `llm-general`: judge a standalone `input -> output` pair\n- `conversation`: judge a full conversation",
+);
+
+export type AnnotationTaskCreate = {
+  name: string;
+  type: AnnotationTaskCreateType;
+  description?: string | null | undefined;
+  evaluator_ids?: Array<string> | null | undefined;
+};
+
+export const AnnotationTaskCreate$zodSchema: z.ZodType<AnnotationTaskCreate> = z
+  .object({
+    description: z.string().nullable().optional().describe(
+      "A description for the task. Omit for none",
+    ),
+    evaluator_ids: z.array(z.string()).nullable().optional().describe(
+      "IDs of evaluators to link when the task is created, in order. Each must be one you created or a built-in default. Omit to create with no linked evaluators",
+    ),
+    name: z.string().describe("Task name, unique within your workspace"),
+    type: AnnotationTaskCreateType$zodSchema.describe(
+      "Task type. Determines the shape of each item's payload.\n- `stt`: judge a transcript on its own\n- `tts`: judge synthesized speech output\n- `llm`: judge one response with its conversation history\n- `llm-general`: judge a standalone `input -> output` pair\n- `conversation`: judge a full conversation",
+    ),
+  });

package/src/models/annotationtaskcreateresponse.ts

@@ -0,0 +1,14 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type AnnotationTaskCreateResponse = { uuid: string; message: string };
+
+export const AnnotationTaskCreateResponse$zodSchema: z.ZodType<
+  AnnotationTaskCreateResponse
+> = z.object({
+  message: z.string().describe("Confirmation that the task was created"),
+  uuid: z.string().describe("ID of the newly created task"),
+});

package/src/models/annotationtaskresponse.ts

@@ -0,0 +1,89 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+/**
+ * Task type. Determines the shape of each item's payload.
+ *
+ * @remarks
+ * - `stt`: judge a transcript on its own
+ * - `tts`: judge synthesized speech output
+ * - `llm`: judge one response with its conversation history
+ * - `llm-general`: judge a standalone `input -> output` pair
+ * - `conversation`: judge a full conversation
+ */
+export const AnnotationTaskResponseType = {
+  Stt: "stt",
+  Tts: "tts",
+  Llm: "llm",
+  LlmGeneral: "llm-general",
+  Conversation: "conversation",
+} as const;
+/**
+ * Task type. Determines the shape of each item's payload.
+ *
+ * @remarks
+ * - `stt`: judge a transcript on its own
+ * - `tts`: judge synthesized speech output
+ * - `llm`: judge one response with its conversation history
+ * - `llm-general`: judge a standalone `input -> output` pair
+ * - `conversation`: judge a full conversation
+ */
+export type AnnotationTaskResponseType = ClosedEnum<
+  typeof AnnotationTaskResponseType
+>;
+
+export const AnnotationTaskResponseType$zodSchema = z.enum([
+  "stt",
+  "tts",
+  "llm",
+  "llm-general",
+  "conversation",
+]).describe(
+  "Task type. Determines the shape of each item's payload.\n- `stt`: judge a transcript on its own\n- `tts`: judge synthesized speech output\n- `llm`: judge one response with its conversation history\n- `llm-general`: judge a standalone `input -> output` pair\n- `conversation`: judge a full conversation",
+);
+
+export type AnnotationTaskResponse = {
+  uuid: string;
+  name: string;
+  type: AnnotationTaskResponseType;
+  description?: string | null | undefined;
+  created_at: string;
+  updated_at: string;
+  evaluators?: Array<{ [k: string]: any }> | undefined;
+  item_count?: number | null | undefined;
+  items?: Array<{ [k: string]: any }> | undefined;
+  jobs?: Array<{ [k: string]: any }> | undefined;
+};
+
+export const AnnotationTaskResponse$zodSchema: z.ZodType<
+  AnnotationTaskResponse
+> = z.object({
+  created_at: z.string().describe("When the task was created (ISO 8601 UTC)"),
+  description: z.string().nullable().optional().describe(
+    "The task's description",
+  ),
+  evaluators: z.array(z.record(z.string(), z.any())).optional().describe(
+    "The evaluators linked to this task, in display order, each with its live version's rubric",
+  ),
+  item_count: z.int().nullable().optional().describe(
+    "Number of items in the task",
+  ),
+  items: z.array(z.record(z.string(), z.any())).optional().describe(
+    "The task's items, each with its agreement stats. You get these when you fetch one task by ID, not when you list tasks",
+  ),
+  jobs: z.array(z.record(z.string(), z.any())).optional().describe(
+    "The task's labelling jobs. You get these when you fetch one task by ID, not when you list tasks",
+  ),
+  name: z.string().describe("Name of the task"),
+  type: AnnotationTaskResponseType$zodSchema.describe(
+    "Task type. Determines the shape of each item's payload.\n- `stt`: judge a transcript on its own\n- `tts`: judge synthesized speech output\n- `llm`: judge one response with its conversation history\n- `llm-general`: judge a standalone `input -> output` pair\n- `conversation`: judge a full conversation",
+  ),
+  updated_at: z.string().describe(
+    "When the task was last updated (ISO 8601 UTC)",
+  ),
+  uuid: z.string().describe("Unique ID for the task"),
+});

package/src/models/batchtestrun.ts

@@ -6,21 +6,21 @@ import * as z from "zod";
 import { ClosedEnum } from "../types/enums.js";
 
 /**
- * Initial status of the test run
+ * Current status of the run
  */
 export const BatchTestRunStatus = {
   Queued: "queued",
   InProgress: "in_progress",
 } as const;
 /**
- * Initial status of the test run
+ * Current status of the run
  */
 export type BatchTestRunStatus = ClosedEnum<typeof BatchTestRunStatus>;
 
 export const BatchTestRunStatus$zodSchema = z.enum([
   "queued",
   "in_progress",
-]).describe("Initial status of the test run");
+]).describe("Current status of the run");
 
 export type BatchTestRun = {
   agent_name: string;
@@ -32,8 +32,8 @@ export type BatchTestRun = {
 export const BatchTestRun$zodSchema: z.ZodType<BatchTestRun> = z.object({
   agent_name: z.string().describe("Name of the agent"),
   agent_uuid: z.string().describe("ID of the agent that was run"),
-  status: BatchTestRunStatus$zodSchema.describe(
-    "Initial status of the test run",
+  status: BatchTestRunStatus$zodSchema.describe("Current status of the run"),
+  task_id: z.string().describe(
+    "Test run job ID. Poll it for status and results",
   ),
-  task_id: z.string().describe("Test run job ID. Poll for status and results"),
 });

package/src/models/batchtestskip.ts

@@ -3,15 +3,45 @@
  */
 
 import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+/**
+ * Why this agent was not run:
+ *
+ * @remarks
+ * - `no_linked_tests`: the agent has no tests linked
+ * - `connection_not_verified`: the agent's connection is not verified
+ */
+export const Reason = {
+  NoLinkedTests: "no_linked_tests",
+  ConnectionNotVerified: "connection_not_verified",
+} as const;
+/**
+ * Why this agent was not run:
+ *
+ * @remarks
+ * - `no_linked_tests`: the agent has no tests linked
+ * - `connection_not_verified`: the agent's connection is not verified
+ */
+export type Reason = ClosedEnum<typeof Reason>;
+
+export const Reason$zodSchema = z.enum([
+  "no_linked_tests",
+  "connection_not_verified",
+]).describe(
+  "Why this agent was not run:\n- `no_linked_tests`: the agent has no tests linked\n- `connection_not_verified`: the agent's connection is not verified",
+);
 
 export type BatchTestSkip = {
   agent_name: string;
   agent_uuid: string;
-  reason: string;
+  reason: Reason;
 };
 
 export const BatchTestSkip$zodSchema: z.ZodType<BatchTestSkip> = z.object({
   agent_name: z.string().describe("Name of the skipped agent"),
   agent_uuid: z.string().describe("ID of the skipped agent"),
-  reason: z.string().describe("Why this agent was not run"),
+  reason: Reason$zodSchema.describe(
+    "Why this agent was not run:\n- `no_linked_tests`: the agent has no tests linked\n- `connection_not_verified`: the agent's connection is not verified",
+  ),
 });

package/src/models/benchmarkrequest.ts

@@ -0,0 +1,19 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type BenchmarkRequest = {
+  models: Array<string>;
+  test_uuids?: Array<string> | null | undefined;
+};
+
+export const BenchmarkRequest$zodSchema: z.ZodType<BenchmarkRequest> = z.object(
+  {
+    models: z.array(z.string()).describe("Model names to benchmark"),
+    test_uuids: z.array(z.string()).nullable().optional().describe(
+      "A subset of the agent's linked tests to benchmark. Each ID must be linked to the agent. Omit to run all linked tests",
+    ),
+  },
+);

package/src/models/benchmarkstatusresponse.ts

@@ -0,0 +1,47 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ModelResult, ModelResult$zodSchema } from "./modelresult.js";
+import { TaskStatus, TaskStatus$zodSchema } from "./taskstatus.js";
+import {
+  TestRunEvaluator,
+  TestRunEvaluator$zodSchema,
+} from "./testrunevaluator.js";
+
+export type BenchmarkStatusResponse = {
+  task_id: string;
+  status: TaskStatus;
+  evaluators?: Array<TestRunEvaluator> | null | undefined;
+  model_results?: Array<ModelResult> | null | undefined;
+  leaderboard_summary?: Array<{ [k: string]: any }> | null | undefined;
+  error?: boolean | undefined;
+  is_public?: boolean | undefined;
+  share_token?: string | null | undefined;
+};
+
+export const BenchmarkStatusResponse$zodSchema: z.ZodType<
+  BenchmarkStatusResponse
+> = z.object({
+  error: z.boolean().default(false).describe("True if the run failed"),
+  evaluators: z.array(TestRunEvaluator$zodSchema).nullable().optional()
+    .describe(
+      "The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`",
+    ),
+  is_public: z.boolean().default(false).describe(
+    "Whether the run is shared publicly",
+  ),
+  leaderboard_summary: z.array(z.record(z.string(), z.any())).nullable()
+    .optional().describe(
+      "Leaderboard comparing the models, one row per model. Columns vary by benchmark: a `model` column plus pass/fail counts, latency, cost, and one score column per evaluator, keyed by evaluator name",
+    ),
+  model_results: z.array(ModelResult$zodSchema).nullable().optional().describe(
+    "Results for each model",
+  ),
+  share_token: z.string().nullable().optional().describe(
+    "Token for building the public share URL",
+  ),
+  status: TaskStatus$zodSchema,
+  task_id: z.string().describe("Benchmark run job ID"),
+});