@dalmia/calibrate-mcp 0.0.13

package/src/models/agenttestruncreateresponse.ts

@@ -0,0 +1,39 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+/**
+ * Current status of the test run
+ */
+export const AgentTestRunCreateResponseStatus = {
+  Queued: "queued",
+  InProgress: "in_progress",
+} as const;
+/**
+ * Current status of the test run
+ */
+export type AgentTestRunCreateResponseStatus = ClosedEnum<
+  typeof AgentTestRunCreateResponseStatus
+>;
+
+export const AgentTestRunCreateResponseStatus$zodSchema = z.enum([
+  "queued",
+  "in_progress",
+]).describe("Current status of the test run");
+
+export type AgentTestRunCreateResponse = {
+  task_id: string;
+  status: AgentTestRunCreateResponseStatus;
+};
+
+export const AgentTestRunCreateResponse$zodSchema: z.ZodType<
+  AgentTestRunCreateResponse
+> = z.object({
+  status: AgentTestRunCreateResponseStatus$zodSchema.describe(
+    "Current status of the test run",
+  ),
+  task_id: z.string().describe("Test run job ID. Poll for status and results"),
+});

package/src/models/agentupdate.ts

@@ -0,0 +1,19 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type AgentUpdate = {
+  name?: string | null | undefined;
+  config?: { [k: string]: any } | null | undefined;
+};
+
+export const AgentUpdate$zodSchema: z.ZodType<AgentUpdate> = z.object({
+  config: z.record(z.string(), z.any()).nullable().optional().describe(
+    "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`** (built inside Calibrate):\n- `system_prompt` (string): the agent's instructions\n- `llm.model` (string): `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider` (string): `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider` (string): `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first` (bool), `settings.max_assistant_turns` (int)\n- `system_tools.end_call` (bool, optional): let the agent end the call\n- `data_extraction_fields` (array, optional): `[{name, type, description, required}]`\n\n```json\n{\n  \"system_prompt\": \"You are a helpful support agent.\",\n  \"llm\": {\"model\": \"openai/gpt-4.1\"},\n  \"stt\": {\"provider\": \"deepgram\"},\n  \"tts\": {\"provider\": \"elevenlabs\"},\n  \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`** (your own HTTP endpoint):\n- `agent_url` (string, required): public HTTPS endpoint the agent is called at\n- `agent_headers` (object, optional): headers sent on each request, e.g. auth\n- `benchmark_provider` (string, optional): `openrouter` (default), `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n  \"agent_url\": \"https://api.example.com/agent\",\n  \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n  \"benchmark_provider\": \"openrouter\"\n}\n```\n\nReplaces the stored config. Omit to leave unchanged.\n\nFor `type=connection`, changing `agent_url` or `agent_headers` resets the connection and benchmark verification flags.",
+  ),
+  name: z.string().nullable().optional().describe(
+    "New agent name. Omit to leave the name unchanged",
+  ),
+});

package/src/models/batchrunrequest.ts

@@ -0,0 +1,15 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type BatchRunRequest = {
+  agent_names?: Array<string> | null | undefined;
+};
+
+export const BatchRunRequest$zodSchema: z.ZodType<BatchRunRequest> = z.object({
+  agent_names: z.array(z.string()).nullable().optional().describe(
+    "Agents to run. Omit to run every agent in your workspace",
+  ),
+});

package/src/models/batchtestrun.ts

@@ -0,0 +1,39 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+/**
+ * Initial status of the test run
+ */
+export const BatchTestRunStatus = {
+  Queued: "queued",
+  InProgress: "in_progress",
+} as const;
+/**
+ * Initial status of the test run
+ */
+export type BatchTestRunStatus = ClosedEnum<typeof BatchTestRunStatus>;
+
+export const BatchTestRunStatus$zodSchema = z.enum([
+  "queued",
+  "in_progress",
+]).describe("Initial status of the test run");
+
+export type BatchTestRun = {
+  agent_name: string;
+  agent_uuid: string;
+  task_id: string;
+  status: BatchTestRunStatus;
+};
+
+export const BatchTestRun$zodSchema: z.ZodType<BatchTestRun> = z.object({
+  agent_name: z.string().describe("Name of the agent"),
+  agent_uuid: z.string().describe("ID of the agent that was run"),
+  status: BatchTestRunStatus$zodSchema.describe(
+    "Initial status of the test run",
+  ),
+  task_id: z.string().describe("Test run job ID. Poll for status and results"),
+});

package/src/models/batchtestrunresponse.ts

@@ -0,0 +1,22 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { BatchTestRun, BatchTestRun$zodSchema } from "./batchtestrun.js";
+import { BatchTestSkip, BatchTestSkip$zodSchema } from "./batchtestskip.js";
+
+export type BatchTestRunResponse = {
+  runs: Array<BatchTestRun>;
+  skipped?: Array<BatchTestSkip> | undefined;
+};
+
+export const BatchTestRunResponse$zodSchema: z.ZodType<BatchTestRunResponse> = z
+  .object({
+    runs: z.array(BatchTestRun$zodSchema).describe(
+      "Test runs that were launched",
+    ),
+    skipped: z.array(BatchTestSkip$zodSchema).optional().describe(
+      "Agents that were skipped instead of failing the batch",
+    ),
+  });

package/src/models/batchtestskip.ts

@@ -0,0 +1,17 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type BatchTestSkip = {
+  agent_name: string;
+  agent_uuid: string;
+  reason: string;
+};
+
+export const BatchTestSkip$zodSchema: z.ZodType<BatchTestSkip> = z.object({
+  agent_name: z.string().describe("Name of the skipped agent"),
+  agent_uuid: z.string().describe("ID of the skipped agent"),
+  reason: z.string().describe("Why this agent was not run"),
+});

package/src/models/bulktestitem.ts

@@ -0,0 +1,36 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ChatMessage, ChatMessage$zodSchema } from "./chatmessage.js";
+import {
+  ExpectedToolCall,
+  ExpectedToolCall$zodSchema,
+} from "./expectedtoolcall.js";
+import {
+  RoutersTestsEvaluatorRef,
+  RoutersTestsEvaluatorRef$zodSchema,
+} from "./routerstestsevaluatorref.js";
+
+export type BulkTestItem = {
+  name: string;
+  conversation_history: Array<ChatMessage>;
+  evaluators?: Array<RoutersTestsEvaluatorRef> | null | undefined;
+  tool_calls?: Array<ExpectedToolCall> | null | undefined;
+};
+
+export const BulkTestItem$zodSchema: z.ZodType<BulkTestItem> = z.object({
+  conversation_history: z.array(ChatMessage$zodSchema).describe(
+    "Ordered messages ending at the user turn the agent should answer (must be non-empty)",
+  ),
+  evaluators: z.array(RoutersTestsEvaluatorRef$zodSchema).nullable().optional()
+    .describe(
+      "Evaluators to link. Used by `response` and `conversation` tests",
+    ),
+  name: z.string().describe(
+    "Name of the test, unique within the workspace and within the batch",
+  ),
+  tool_calls: z.array(ExpectedToolCall$zodSchema).nullable().optional()
+    .describe("Expected tool calls. **Required for `tool_call` batches**"),
+});

package/src/models/bulktestupload.ts

@@ -0,0 +1,66 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+import { BulkTestItem, BulkTestItem$zodSchema } from "./bulktestitem.js";
+
+/**
+ * What the test judges:
+ *
+ * @remarks
+ *
+ * - `response`: judges the generated reply
+ * - `tool_call`: diffs the generated tool calls
+ * - `conversation`: judges the full conversation
+ *
+ * Applied to every test in the batch.
+ */
+export const BulkTestUploadType = {
+  Response: "response",
+  ToolCall: "tool_call",
+  Conversation: "conversation",
+} as const;
+/**
+ * What the test judges:
+ *
+ * @remarks
+ *
+ * - `response`: judges the generated reply
+ * - `tool_call`: diffs the generated tool calls
+ * - `conversation`: judges the full conversation
+ *
+ * Applied to every test in the batch.
+ */
+export type BulkTestUploadType = ClosedEnum<typeof BulkTestUploadType>;
+
+export const BulkTestUploadType$zodSchema = z.enum([
+  "response",
+  "tool_call",
+  "conversation",
+]).describe(
+  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\nApplied to every test in the batch.",
+);
+
+export type BulkTestUpload = {
+  type: BulkTestUploadType;
+  tests: Array<BulkTestItem>;
+  agent_uuids?: Array<string> | null | undefined;
+  language?: string | null | undefined;
+};
+
+export const BulkTestUpload$zodSchema: z.ZodType<BulkTestUpload> = z.object({
+  agent_uuids: z.array(z.string()).nullable().optional().describe(
+    "Agents (IDs) to link every created test to. Omit to link none",
+  ),
+  language: z.string().nullable().optional().describe(
+    "Language written to each test's `config.settings.language`. Omit to leave unset",
+  ),
+  tests: z.array(BulkTestItem$zodSchema).describe(
+    "Test items to create (non-empty, max 500 per request, names unique within the batch)",
+  ),
+  type: BulkTestUploadType$zodSchema.describe(
+    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\nApplied to every test in the batch.",
+  ),
+});

package/src/models/bulktestuploadresponse.ts

@@ -0,0 +1,25 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type BulkTestUploadResponse = {
+  uuids: Array<string>;
+  count: number;
+  message: string;
+  warnings?: Array<string> | null | undefined;
+};
+
+export const BulkTestUploadResponse$zodSchema: z.ZodType<
+  BulkTestUploadResponse
+> = z.object({
+  count: z.int().describe("Number of tests created"),
+  message: z.string().describe("Confirmation message"),
+  uuids: z.array(z.string()).describe(
+    "IDs of the created tests, in request order",
+  ),
+  warnings: z.array(z.string()).nullable().optional().describe(
+    "Non-fatal issues (e.g. agents some tests couldn't link to). Null when there were none",
+  ),
+});

package/src/models/bulkuploadteststestsbulkpostop.ts

@@ -0,0 +1,39 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { BulkTestUpload, BulkTestUpload$zodSchema } from "./bulktestupload.js";
+import {
+  BulkTestUploadResponse,
+  BulkTestUploadResponse$zodSchema,
+} from "./bulktestuploadresponse.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+
+export type BulkUploadTestsTestsBulkPostRequest = {
+  xAPIKey?: string | null | undefined;
+  xOrgUUID?: string | null | undefined;
+  body: BulkTestUpload;
+};
+
+export const BulkUploadTestsTestsBulkPostRequest$zodSchema: z.ZodType<
+  BulkUploadTestsTestsBulkPostRequest
+> = z.object({
+  body: BulkTestUpload$zodSchema,
+  xAPIKey: z.string().nullable().optional(),
+  xOrgUUID: z.string().nullable().optional(),
+});
+
+export type BulkUploadTestsTestsBulkPostResponse =
+  | BulkTestUploadResponse
+  | HTTPValidationError;
+
+export const BulkUploadTestsTestsBulkPostResponse$zodSchema: z.ZodType<
+  BulkUploadTestsTestsBulkPostResponse
+> = z.union([
+  BulkTestUploadResponse$zodSchema,
+  HTTPValidationError$zodSchema,
+]);

package/src/models/chatmessage.ts

@@ -0,0 +1,50 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+/**
+ * Message author role in the conversation history
+ */
+export const Role = {
+  User: "user",
+  Assistant: "assistant",
+  Tool: "tool",
+} as const;
+/**
+ * Message author role in the conversation history
+ */
+export type Role = ClosedEnum<typeof Role>;
+
+export const Role$zodSchema = z.enum([
+  "user",
+  "assistant",
+  "tool",
+]).describe("Message author role in the conversation history");
+
+export type ChatMessage = {
+  role: Role;
+  content?: string | null | undefined;
+  tool_calls?: Array<{ [k: string]: any }> | null | undefined;
+  tool_call_id?: string | null | undefined;
+  name?: string | null | undefined;
+};
+
+export const ChatMessage$zodSchema: z.ZodType<ChatMessage> = z.object({
+  content: z.string().nullable().optional().describe(
+    "Message text. Omit for assistant messages that only carry `tool_calls`",
+  ),
+  name: z.string().nullable().optional().describe(
+    "Tool name for `role=tool` messages. Omit otherwise",
+  ),
+  role: Role$zodSchema.describe(
+    "Message author role in the conversation history",
+  ),
+  tool_call_id: z.string().nullable().optional().describe(
+    "ID of the tool call this message responds to. **Required for `role=tool`**",
+  ),
+  tool_calls: z.array(z.record(z.string(), z.any())).nullable().optional()
+    .describe("Tool calls issued by the assistant. Omit for plain text turns"),
+});

package/src/models/createagentendpointagentspostop.ts

@@ -0,0 +1,39 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { AgentCreate, AgentCreate$zodSchema } from "./agentcreate.js";
+import {
+  AgentCreateResponse,
+  AgentCreateResponse$zodSchema,
+} from "./agentcreateresponse.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+
+export type CreateAgentEndpointAgentsPostRequest = {
+  xAPIKey?: string | null | undefined;
+  xOrgUUID?: string | null | undefined;
+  body: AgentCreate;
+};
+
+export const CreateAgentEndpointAgentsPostRequest$zodSchema: z.ZodType<
+  CreateAgentEndpointAgentsPostRequest
+> = z.object({
+  body: AgentCreate$zodSchema,
+  xAPIKey: z.string().nullable().optional(),
+  xOrgUUID: z.string().nullable().optional(),
+});
+
+export type CreateAgentEndpointAgentsPostResponse =
+  | AgentCreateResponse
+  | HTTPValidationError;
+
+export const CreateAgentEndpointAgentsPostResponse$zodSchema: z.ZodType<
+  CreateAgentEndpointAgentsPostResponse
+> = z.union([
+  AgentCreateResponse$zodSchema,
+  HTTPValidationError$zodSchema,
+]);

package/src/models/createtestendpointtestspostop.ts

@@ -0,0 +1,39 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import { TestCreate, TestCreate$zodSchema } from "./testcreate.js";
+import {
+  TestCreateResponse,
+  TestCreateResponse$zodSchema,
+} from "./testcreateresponse.js";
+
+export type CreateTestEndpointTestsPostRequest = {
+  xAPIKey?: string | null | undefined;
+  xOrgUUID?: string | null | undefined;
+  body: TestCreate;
+};
+
+export const CreateTestEndpointTestsPostRequest$zodSchema: z.ZodType<
+  CreateTestEndpointTestsPostRequest
+> = z.object({
+  body: TestCreate$zodSchema,
+  xAPIKey: z.string().nullable().optional(),
+  xOrgUUID: z.string().nullable().optional(),
+});
+
+export type CreateTestEndpointTestsPostResponse =
+  | TestCreateResponse
+  | HTTPValidationError;
+
+export const CreateTestEndpointTestsPostResponse$zodSchema: z.ZodType<
+  CreateTestEndpointTestsPostResponse
+> = z.union([
+  TestCreateResponse$zodSchema,
+  HTTPValidationError$zodSchema,
+]);

package/src/models/errors/apierror.ts

@@ -0,0 +1,21 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+export class APIError extends Error {
+  constructor(
+    message: string,
+    public readonly httpMeta: {
+      response: Response;
+      request: Request;
+      body: string;
+    },
+  ) {
+    super(
+      `${message}: Status ${httpMeta.response.status} Content-Type ${
+        httpMeta.response.headers.get("content-type") || ""
+      } Body: ${httpMeta.body}`,
+    );
+    this.name = "APIError";
+  }
+}

package/src/models/errors/httpclienterrors.ts

@@ -0,0 +1,62 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+/**
+ * Base class for all HTTP errors.
+ */
+export class HTTPClientError extends Error {
+  /** The underlying cause of the error. */
+  override readonly cause: unknown;
+  override name = "HTTPClientError";
+  constructor(message: string, opts?: { cause?: unknown }) {
+    let msg = message;
+    if (opts?.cause) {
+      msg += `: ${opts.cause}`;
+    }
+
+    super(msg, opts);
+    // In older runtimes, the cause field would not have been assigned through
+    // the super() call.
+    if (typeof this.cause === "undefined") {
+      this.cause = opts?.cause;
+    }
+  }
+}
+
+/**
+ * An error to capture unrecognised or unexpected errors when making HTTP calls.
+ */
+export class UnexpectedClientError extends HTTPClientError {
+  override name = "UnexpectedClientError";
+}
+
+/**
+ * An error that is raised when any inputs used to create a request are invalid.
+ */
+export class InvalidRequestError extends HTTPClientError {
+  override name = "InvalidRequestError";
+}
+
+/**
+ * An error that is raised when a HTTP request was aborted by the client error.
+ */
+export class RequestAbortedError extends HTTPClientError {
+  override readonly name = "RequestAbortedError";
+}
+
+/**
+ * An error that is raised when a HTTP request timed out due to an AbortSignal
+ * signal timeout.
+ */
+export class RequestTimeoutError extends HTTPClientError {
+  override readonly name = "RequestTimeoutError";
+}
+
+/**
+ * An error that is raised when a HTTP client is unable to make a request to
+ * a server.
+ */
+export class ConnectionError extends HTTPClientError {
+  override readonly name = "ConnectionError";
+}

package/src/models/errors/sdkvalidationerror.ts

@@ -0,0 +1,42 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export class SDKValidationError extends Error {
+  /**
+   * The raw value that failed validation.
+   */
+  public readonly rawValue: unknown;
+
+  /**
+   * The raw message that failed validation.
+   */
+  public readonly rawMessage: unknown;
+
+  constructor(message: string, cause: unknown, rawValue: unknown) {
+    super(`${message}: ${cause}`);
+    this.name = "SDKValidationError";
+    this.cause = cause;
+    this.rawValue = rawValue;
+    this.rawMessage = message;
+  }
+
+  /**
+   * Return a pretty-formatted error message if the underlying validation error
+   * is a ZodError or some other recognized error type, otherwise return the
+   * default error message.
+   */
+  public pretty(): string {
+    if (this.cause instanceof z.ZodError) {
+      return `${this.rawMessage}\n${formatZodError(this.cause)}`;
+    } else {
+      return this.toString();
+    }
+  }
+}
+
+export function formatZodError(err: z.ZodError): string {
+  return z.prettifyError(err);
+}

package/src/models/expectedtoolcall.ts

@@ -0,0 +1,23 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type ExpectedToolCall = {
+  tool: string;
+  arguments?: { [k: string]: any } | null | undefined;
+  accept_any_arguments?: boolean | undefined;
+};
+
+export const ExpectedToolCall$zodSchema: z.ZodType<ExpectedToolCall> = z.object(
+  {
+    accept_any_arguments: z.boolean().default(false).describe(
+      "When `true`, only the tool name must match and `arguments` is ignored",
+    ),
+    arguments: z.record(z.string(), z.any()).nullable().optional().describe(
+      "Expected argument values, diffed against the generated call. Omit to expect no arguments",
+    ),
+    tool: z.string().describe("Name of the tool the agent is expected to call"),
+  },
+);

package/src/models/getagentendpointagentsagentuuidgetop.ts

@@ -0,0 +1,38 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import {
+  RoutersAgentsAgentResponse,
+  RoutersAgentsAgentResponse$zodSchema,
+} from "./routersagentsagentresponse.js";
+
+export type GetAgentEndpointAgentsAgentUuidGetRequest = {
+  agent_uuid: string;
+  xAPIKey?: string | null | undefined;
+  xOrgUUID?: string | null | undefined;
+};
+
+export const GetAgentEndpointAgentsAgentUuidGetRequest$zodSchema: z.ZodType<
+  GetAgentEndpointAgentsAgentUuidGetRequest
+> = z.object({
+  agent_uuid: z.string().describe("The agent to retrieve."),
+  xAPIKey: z.string().nullable().optional(),
+  xOrgUUID: z.string().nullable().optional(),
+});
+
+export type GetAgentEndpointAgentsAgentUuidGetResponse =
+  | RoutersAgentsAgentResponse
+  | HTTPValidationError;
+
+export const GetAgentEndpointAgentsAgentUuidGetResponse$zodSchema: z.ZodType<
+  GetAgentEndpointAgentsAgentUuidGetResponse
+> = z.union([
+  RoutersAgentsAgentResponse$zodSchema,
+  HTTPValidationError$zodSchema,
+]);