@dalmia/calibrate-mcp 0.0.13

package/src/mcp-server/tools/testsGet.ts

@@ -0,0 +1,44 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { testsGet } from "../../funcs/testsGet.js";
+import { GetTestEndpointTestsTestUuidGetRequest$zodSchema } from "../../models/gettestendpointteststestuuidgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: GetTestEndpointTestsTestUuidGetRequest$zodSchema,
+};
+
+export const tool$testsGet: ToolDefinition<typeof args> = {
+  name: "get-test",
+  description: `Fetch a single test by UUID within the caller's org. Returns the
+test's name, type, and config. Fails with 404 if the test does not
+exist or belongs to another org.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "Get test",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await testsGet(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/testsList.ts

@@ -0,0 +1,45 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { testsList } from "../../funcs/testsList.js";
+import { ListTestsTestsGetRequest$zodSchema } from "../../models/listteststestsgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: ListTestsTestsGetRequest$zodSchema.optional(),
+};
+
+export const tool$testsList: ToolDefinition<typeof args> = {
+  name: "list-tests",
+  description:
+    `List every test in the caller's organization. Returns each test's
+UUID, name, type, and config. Use this to discover which tests exist
+before creating runs.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "List tests",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await testsList(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/testsUpdate.ts

@@ -0,0 +1,45 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { testsUpdate } from "../../funcs/testsUpdate.js";
+import { UpdateTestEndpointTestsTestUuidPutRequest$zodSchema } from "../../models/updatetestendpointteststestuuidputop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: UpdateTestEndpointTestsTestUuidPutRequest$zodSchema,
+};
+
+export const tool$testsUpdate: ToolDefinition<typeof args> = {
+  name: "update-test",
+  description:
+    `Update an existing test by UUID within the caller's org. Supply the
+fields to change (name, config). Returns the updated test. Fails with
+404 if the test does not exist or belongs to another org.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Update test",
+    "destructiveHint": false,
+    "idempotentHint": true,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await testsUpdate(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools.ts

@@ -0,0 +1,498 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import {
+  ShapeOutput,
+  ZodRawShapeCompat,
+} from "@modelcontextprotocol/sdk/server/zod-compat.js";
+import { RequestHandlerExtra } from "@modelcontextprotocol/sdk/shared/protocol.js";
+import {
+  CallToolResult,
+  ServerNotification,
+  ServerRequest,
+} from "@modelcontextprotocol/sdk/types.js";
+import * as z from "zod";
+import { CalibrateMcpCore } from "../core.js";
+import { ConsoleLogger } from "./console-logger.js";
+import { MCPServerFlags } from "./flags.js";
+import { MCPScope, mcpScopes } from "./scopes.js";
+import { valueToBase64 } from "./shared.js";
+
+export type MCPToolAnnotationFilter = {
+  readOnlyHint?: boolean;
+  destructiveHint?: boolean;
+  idempotentHint?: boolean;
+  openWorldHint?: boolean;
+};
+
+const VALID_ANNOTATIONS = [
+  "readOnly",
+  "destructive",
+  "idempotent",
+  "openWorld",
+] as const;
+type AnnotationName = typeof VALID_ANNOTATIONS[number];
+
+const annotationToKey: Record<AnnotationName, keyof MCPToolAnnotationFilter> = {
+  readOnly: "readOnlyHint",
+  destructive: "destructiveHint",
+  idempotent: "idempotentHint",
+  openWorld: "openWorldHint",
+};
+
+export function buildAnnotationFilter(
+  annotations: string[] | undefined,
+): MCPToolAnnotationFilter {
+  const filter: MCPToolAnnotationFilter = {};
+  if (!annotations || annotations.length === 0) {
+    return filter;
+  }
+  for (const a of annotations) {
+    if (!VALID_ANNOTATIONS.includes(a as AnnotationName)) {
+      throw new Error(
+        `Invalid annotation filter: "${a}". Valid values are: ${
+          VALID_ANNOTATIONS.join(", ")
+        }`,
+      );
+    }
+  }
+  for (const name of VALID_ANNOTATIONS) {
+    filter[annotationToKey[name]] = annotations.includes(name);
+  }
+  return filter;
+}
+
+export type ToolDefinition<
+  Args extends undefined | ZodRawShapeCompat = undefined,
+> = Args extends ZodRawShapeCompat ? {
+    name: string;
+    description: string;
+    scopes?: MCPScope[];
+    args: Args;
+    annotations: {
+      title: string;
+      destructiveHint: boolean;
+      idempotentHint: boolean;
+      openWorldHint: boolean;
+      readOnlyHint: boolean;
+    };
+    tool: (
+      client: CalibrateMcpCore,
+      args: ShapeOutput<Args>,
+      extra: RequestHandlerExtra<ServerRequest, ServerNotification>,
+    ) => CallToolResult | Promise<CallToolResult>;
+  }
+  : {
+    name: string;
+    description: string;
+    scopes?: MCPScope[];
+    args?: undefined;
+    annotations: {
+      title: string;
+      destructiveHint: boolean;
+      idempotentHint: boolean;
+      openWorldHint: boolean;
+      readOnlyHint: boolean;
+    };
+    tool: (
+      client: CalibrateMcpCore,
+      extra: RequestHandlerExtra<ServerRequest, ServerNotification>,
+    ) => CallToolResult | Promise<CallToolResult>;
+  };
+
+// Optional function to assist with formatting tool results
+export async function formatResult(
+  response: Response,
+): Promise<CallToolResult> {
+  let content: CallToolResult["content"] = [];
+  const contentType = response?.headers.get("content-type") ?? "";
+
+  if (contentType.startsWith("image/")) {
+    const data = await valueToBase64(await response.arrayBuffer());
+    content = data == null
+      ? []
+      : [{ type: "image", data, mimeType: contentType }];
+  } else if (contentType.startsWith("audio/")) {
+    const data = await valueToBase64(await response.arrayBuffer());
+    content = data == null
+      ? []
+      : [{ type: "audio", data, mimeType: contentType }];
+  } else {
+    const text = await response.text();
+    content = [{ type: "text", text }];
+  }
+
+  return response.ok ? { content } : { content, isError: true };
+}
+
+export function createRegisterTool(
+  logger: ConsoleLogger,
+  server: McpServer,
+  getSDK: () => CalibrateMcpCore,
+  allowedScopes: Set<MCPScope>,
+  allowedTools?: Set<string>,
+  dynamic?: boolean,
+  annotationFilter?: MCPToolAnnotationFilter,
+): [
+  <A extends ZodRawShapeCompat | undefined>(tool: ToolDefinition<A>) => void,
+  Array<{ name: string; description: string }>,
+  Map<string, ToolDefinition<ZodRawShapeCompat | undefined>>,
+] {
+  const tools: Array<{ name: string; description: string }> = [];
+  const toolMap = new Map<
+    string,
+    ToolDefinition<ZodRawShapeCompat | undefined>
+  >();
+  const registerTool = <A extends ZodRawShapeCompat | undefined>(
+    tool: ToolDefinition<A>,
+  ): void => {
+    if (allowedTools && !allowedTools.has(tool.name)) {
+      return;
+    }
+
+    const scopes = tool.scopes ?? [];
+    if (allowedScopes.size > 0 && scopes.length === 0) {
+      return;
+    }
+
+    if (
+      allowedScopes.size > 0
+      && !scopes.every((s: MCPScope) => allowedScopes.has(s))
+    ) {
+      return;
+    }
+
+    if (annotationFilter) {
+      const a = tool.annotations;
+      if (
+        annotationFilter.readOnlyHint !== undefined
+        && a.readOnlyHint !== annotationFilter.readOnlyHint
+      ) {
+        return;
+      }
+      if (
+        annotationFilter.destructiveHint !== undefined
+        && a.destructiveHint !== annotationFilter.destructiveHint
+      ) {
+        return;
+      }
+      if (
+        annotationFilter.idempotentHint !== undefined
+        && a.idempotentHint !== annotationFilter.idempotentHint
+      ) {
+        return;
+      }
+      if (
+        annotationFilter.openWorldHint !== undefined
+        && a.openWorldHint !== annotationFilter.openWorldHint
+      ) {
+        return;
+      }
+    }
+
+    toolMap.set(
+      tool.name,
+      tool as ToolDefinition<ZodRawShapeCompat | undefined>,
+    );
+
+    if (dynamic) {
+      logger.debug("Collected tool for dynamic mode", { name: tool.name });
+      tools.push({ name: tool.name, description: tool.description });
+      return;
+    }
+
+    if (tool.args) {
+      server.registerTool(
+        tool.name,
+        {
+          description: tool.description,
+          inputSchema: tool.args,
+          annotations: tool.annotations,
+        },
+        async (args, ctx) => {
+          return tool.tool(getSDK(), args, ctx);
+        },
+      );
+    } else {
+      server.registerTool(
+        tool.name,
+        {
+          description: tool.description,
+          annotations: tool.annotations,
+        },
+        async (ctx) => {
+          return tool.tool(getSDK(), ctx);
+        },
+      );
+    }
+
+    logger.debug("Registered tool", { name: tool.name });
+    tools.push({ name: tool.name, description: tool.description });
+  };
+
+  return [registerTool, tools, toolMap];
+}
+
+function matchesSearchTerms(
+  terms: string[],
+  name: string,
+  def: ToolDefinition<ZodRawShapeCompat | undefined>,
+): boolean {
+  if (terms.length === 0) {
+    return true;
+  }
+
+  const nameLower = name.toLowerCase();
+  const descLower = def.description.toLowerCase();
+  const scopesLower = def.scopes?.flatMap((s: unknown) => {
+    return typeof s === "string" ? [s.toLowerCase()] : [];
+  }) || [];
+
+  return terms.some((term) =>
+    nameLower.includes(term)
+    || descLower.includes(term)
+    || scopesLower.some((s) => s.includes(term))
+  );
+}
+
+export function registerDynamicTools(
+  logger: ConsoleLogger,
+  server: McpServer,
+  getSDK: () => CalibrateMcpCore,
+  toolMap: Map<string, ToolDefinition<ZodRawShapeCompat | undefined>>,
+  allowedScopes: Set<MCPScope>,
+): void {
+  // 1. list_tools
+  server.registerTool("list_tools", {
+    description:
+      "List available tools. Optionally filter by search terms that match against tool name, description, and scopes.",
+    inputSchema: {
+      search_terms: z.array(z.string()).optional().describe(
+        [
+          "Filter the list of tools.",
+          "Each term is matched case-insensitively as a substring against tool name, description, and scopes.",
+          "Multiple terms are combined with a logical OR: a tool is included if ANY term matches ANY field.",
+          "If not provided, all tools are returned.",
+        ].join("\n"),
+      ),
+    },
+    annotations: {
+      title: "List Tools",
+      readOnlyHint: true,
+      destructiveHint: false,
+      idempotentHint: true,
+      openWorldHint: false,
+    },
+  }, async (args) => {
+    const terms = (args.search_terms ?? []).map((t) => t.toLowerCase());
+
+    const tools: Array<
+      { name: string; description: string; scopes?: MCPScope[] | undefined }
+    > = [];
+    for (const [name, def] of toolMap) {
+      if (!matchesSearchTerms(terms, name, def)) {
+        continue;
+      }
+
+      tools.push({
+        name: name,
+        description: def.description,
+        scopes: def.scopes,
+      });
+    }
+    return {
+      content: [{ type: "text", text: JSON.stringify(tools, null, 2) }],
+    };
+  });
+  logger.debug("Registered dynamic meta-tool", { name: "list_tools" });
+
+  // 2. describe_tool
+  server.registerTool("describe_tool_input", {
+    description:
+      "Get the input schema for one or more tools. It is a good idea to call this tool first to understand how to successfully call execute_tool.",
+    inputSchema: {
+      tool_names: z.array(z.string()).describe(
+        "The names of the tools to describe",
+      ),
+    },
+    annotations: {
+      title: "Describe Tool",
+      readOnlyHint: true,
+      destructiveHint: false,
+      idempotentHint: true,
+      openWorldHint: false,
+    },
+  }, async (args) => {
+    const parts: string[] = [];
+    const unknown: string[] = [];
+
+    if (args.tool_names.length === 0) {
+      return { content: [{ type: "text", text: "No tool names provided." }] };
+    }
+
+    for (const toolName of args.tool_names) {
+      const def = toolMap.get(toolName);
+      if (!def) {
+        unknown.push(toolName);
+        continue;
+      }
+
+      let schemaText = `<input_schema tool="${toolName}">\n\n`;
+      if (def.args) {
+        const jsonSchema = z.toJSONSchema(z.object(def.args), {
+          target: "draft-2020-12",
+          io: "input",
+          unrepresentable: "any",
+        });
+        schemaText += JSON.stringify(jsonSchema, null, 2);
+      } else {
+        schemaText += "This tool takes no input parameters.";
+      }
+      schemaText += `\n\n</input_schema>`;
+      parts.push(schemaText);
+    }
+
+    if (unknown.length > 0) {
+      parts.push(`Unknown tools: ${unknown.join(", ")}`);
+    }
+
+    return { content: [{ type: "text", text: parts.join("\n\n") }] };
+  });
+  logger.debug("Registered dynamic meta-tool", { name: "describe_tool_input" });
+
+  // 3. execute_tool
+  server.registerTool("execute_tool", {
+    description:
+      "Execute a tool by name with its arguments. If executing a given tool for the first time, it is recommended to call describe_tool_input first to understand the expected `arguments` shape.",
+    inputSchema: {
+      name: z.string().describe("The name of the tool to execute"),
+      arguments: z.looseObject({}).optional().describe(
+        "Arguments for the target tool as a JSON object, matching the schema returned by describe_tool_input.",
+      ),
+    },
+    annotations: {
+      title: "Execute Tool",
+      readOnlyHint: false,
+      destructiveHint: true,
+      idempotentHint: false,
+      openWorldHint: true,
+    },
+  }, async (args, ctx) => {
+    const def = toolMap.get(args.name);
+    if (!def) {
+      return {
+        content: [{ type: "text", text: `Unknown tool: ${args.name}` }],
+        isError: true,
+      };
+    }
+
+    let validatedInput: Record<string, unknown> = {};
+    if (def.args) {
+      const vres = z.object(def.args).safeParse(args.arguments ?? {});
+      if (vres.success) {
+        validatedInput = vres.data;
+      } else {
+        const issues = z.prettifyError(vres.error);
+
+        return {
+          content: [{
+            type: "text",
+            text:
+              `Invalid input for tool ${args.name}:\n<issues>\n${issues}\n</issues>`,
+          }],
+          isError: true,
+        };
+      }
+    }
+
+    try {
+      if (def.args) {
+        return await def.tool(getSDK(), validatedInput, ctx);
+      } else {
+        return await def.tool(getSDK(), ctx);
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      return {
+        content: [{
+          type: "text",
+          text: `Error executing tool ${args.name}: ${message}`,
+        }],
+        isError: true,
+      };
+    }
+  });
+  logger.debug("Registered dynamic meta-tool", { name: "execute_tool" });
+
+  // 4. list_scopes (only when the server has scopes defined)
+  if (mcpScopes.length > 0) {
+    const scopes = allowedScopes.size > 0
+      ? [...allowedScopes].sort()
+      : [...mcpScopes];
+
+    server.registerTool("list_scopes", {
+      description: [
+        "List the scopes available on this server.",
+        "Scopes are categories that group related tools together.",
+        "They can be used as search terms when filtering the list of tools.",
+      ].join("\n"),
+      annotations: {
+        title: "List Scopes",
+        readOnlyHint: true,
+        destructiveHint: false,
+        idempotentHint: true,
+        openWorldHint: false,
+      },
+    }, async () => {
+      const lines = scopes.map((s) => `- ${s}`);
+      return { content: [{ type: "text", text: lines.join("\n") }] };
+    });
+    logger.debug("Registered dynamic meta-tool", { name: "list_scopes" });
+  }
+}
+function resolveHeader<T>(
+  headers: Headers,
+  headerName: string,
+  schema: z.ZodType<T>,
+  cliFlagValue: T | undefined,
+  disableStaticAuth: boolean,
+): T | undefined {
+  const val = headers.get(headerName);
+  if (val != null) {
+    return schema.parse(val);
+  }
+  return disableStaticAuth ? undefined : schema.parse(cliFlagValue);
+}
+
+export function buildSDK(
+  headers: Headers,
+  cliFlags: MCPServerFlags,
+  disableStaticAuth: boolean,
+  logger: { level: string },
+) {
+  const flags = {
+    ...cliFlags,
+    "api-key-auth": resolveHeader(
+      headers,
+      "api-key-auth",
+      z.string(),
+      cliFlags["api-key-auth"],
+      disableStaticAuth,
+    ),
+  };
+
+  return new CalibrateMcpCore({
+    security: { ApiKeyAuth: flags["api-key-auth"] ?? "" },
+    serverURL: cliFlags["server-url"],
+    serverIdx: cliFlags["server-index"],
+    debugLogger: logger.level === "debug"
+      ? {
+        log: (...args) => console.log(...args),
+        group: (...args) => console.group(...args),
+        groupEnd: (...args) => console.groupEnd(...args),
+      }
+      : undefined,
+  });
+}

package/src/models/agentcreate.ts

@@ -0,0 +1,47 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+/**
+ * - `agent`: built inside Calibrate
+ *
+ * @remarks
+ * - `connection`: your existing agent connected to Calibrate
+ */
+export const AgentCreateType = {
+  Agent: "agent",
+  Connection: "connection",
+} as const;
+/**
+ * - `agent`: built inside Calibrate
+ *
+ * @remarks
+ * - `connection`: your existing agent connected to Calibrate
+ */
+export type AgentCreateType = ClosedEnum<typeof AgentCreateType>;
+
+export const AgentCreateType$zodSchema = z.enum([
+  "agent",
+  "connection",
+]).describe(
+  "- `agent`: built inside Calibrate\n- `connection`: your existing agent connected to Calibrate",
+);
+
+export type AgentCreate = {
+  name: string;
+  type?: AgentCreateType | undefined;
+  config?: { [k: string]: any } | null | undefined;
+};
+
+export const AgentCreate$zodSchema: z.ZodType<AgentCreate> = z.object({
+  config: z.record(z.string(), z.any()).nullable().optional().describe(
+    "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`** (built inside Calibrate):\n- `system_prompt` (string): the agent's instructions\n- `llm.model` (string): `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider` (string): `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider` (string): `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first` (bool), `settings.max_assistant_turns` (int)\n- `system_tools.end_call` (bool, optional): let the agent end the call\n- `data_extraction_fields` (array, optional): `[{name, type, description, required}]`\n\n```json\n{\n  \"system_prompt\": \"You are a helpful support agent.\",\n  \"llm\": {\"model\": \"openai/gpt-4.1\"},\n  \"stt\": {\"provider\": \"deepgram\"},\n  \"tts\": {\"provider\": \"elevenlabs\"},\n  \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`** (your own HTTP endpoint):\n- `agent_url` (string, required): public HTTPS endpoint the agent is called at\n- `agent_headers` (object, optional): headers sent on each request, e.g. auth\n- `benchmark_provider` (string, optional): `openrouter` (default), `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n  \"agent_url\": \"https://api.example.com/agent\",\n  \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n  \"benchmark_provider\": \"openrouter\"\n}\n```\n\nFor `type=agent`, omitted keys inherit managed defaults (omit `config` entirely to use all defaults). For `type=connection`, `config` is stored as-is and must contain `agent_url`.",
+  ),
+  name: z.string().describe("Agent name, unique within the workspace"),
+  type: AgentCreateType$zodSchema.default("agent").describe(
+    "- `agent`: built inside Calibrate\n- `connection`: your existing agent connected to Calibrate",
+  ),
+});

package/src/models/agentcreateresponse.ts

@@ -0,0 +1,13 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type AgentCreateResponse = { uuid: string; message: string };
+
+export const AgentCreateResponse$zodSchema: z.ZodType<AgentCreateResponse> = z
+  .object({
+    message: z.string().describe("Confirmation message"),
+    uuid: z.string().describe("ID of the newly created agent"),
+  });