@dalmia/calibrate-mcp 0.0.13

package/src/mcp-server/server.ts

@@ -0,0 +1,111 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { CalibrateMcpCore } from "../core.js";
+import { SDKOptions } from "../lib/config.js";
+import type { ConsoleLogger } from "./console-logger.js";
+import { createRegisterPrompt } from "./prompts.js";
+import {
+  createRegisterResource,
+  createRegisterResourceTemplate,
+} from "./resources.js";
+import { MCPScope } from "./scopes.js";
+import {
+  createRegisterTool,
+  MCPToolAnnotationFilter,
+  registerDynamicTools,
+} from "./tools.js";
+import { tool$agentsCreate } from "./tools/agentsCreate.js";
+import { tool$agentsGet } from "./tools/agentsGet.js";
+import { tool$agentsList } from "./tools/agentsList.js";
+import { tool$agentsResolve } from "./tools/agentsResolve.js";
+import { tool$agentsUpdate } from "./tools/agentsUpdate.js";
+import { tool$agentTestsGetRun } from "./tools/agentTestsGetRun.js";
+import { tool$agentTestsRun } from "./tools/agentTestsRun.js";
+import { tool$agentTestsRunBatch } from "./tools/agentTestsRunBatch.js";
+import { tool$testsBulkCreate } from "./tools/testsBulkCreate.js";
+import { tool$testsCreate } from "./tools/testsCreate.js";
+import { tool$testsGet } from "./tools/testsGet.js";
+import { tool$testsList } from "./tools/testsList.js";
+import { tool$testsUpdate } from "./tools/testsUpdate.js";
+
+export function createMCPServer(deps: {
+  logger: ConsoleLogger;
+  allowedTools?: string[] | undefined;
+  dynamic?: boolean | undefined;
+  scopes?: MCPScope[] | undefined;
+  annotationFilter?: MCPToolAnnotationFilter | undefined;
+  getSDK?: () => CalibrateMcpCore;
+  serverURL?: string | undefined;
+  security?: SDKOptions["security"] | undefined;
+  serverIdx?: SDKOptions["serverIdx"] | undefined;
+}) {
+  const server = new McpServer({
+    name: "CalibrateMcp",
+    version: "0.0.13",
+  });
+
+  const getClient = deps.getSDK || (() =>
+    new CalibrateMcpCore({
+      security: deps.security,
+      serverURL: deps.serverURL,
+      serverIdx: deps.serverIdx,
+      debugLogger: deps.logger.level === "debug"
+        ? {
+          log: (...args) => console.log(...args),
+          group: (...args) => console.group(...args),
+          groupEnd: (...args) => console.groupEnd(...args),
+        }
+        : undefined,
+    }));
+
+  const scopes = new Set(deps.scopes);
+
+  const allowedTools = deps.allowedTools && new Set(deps.allowedTools);
+  const [tool, tools, toolMap] = createRegisterTool(
+    deps.logger,
+    server,
+    getClient,
+    scopes,
+    allowedTools,
+    deps.dynamic,
+    deps.annotationFilter,
+  );
+  const resource = createRegisterResource(
+    deps.logger,
+    server,
+    getClient,
+    scopes,
+  );
+  const resourceTemplate = createRegisterResourceTemplate(
+    deps.logger,
+    server,
+    getClient,
+    scopes,
+  );
+  const prompt = createRegisterPrompt(deps.logger, server, getClient, scopes);
+  const register = { tool, resource, resourceTemplate, prompt };
+  void register; // suppress unused warnings
+
+  tool(tool$agentsResolve);
+  tool(tool$agentsCreate);
+  tool(tool$agentsList);
+  tool(tool$agentsGet);
+  tool(tool$agentsUpdate);
+  tool(tool$testsBulkCreate);
+  tool(tool$testsCreate);
+  tool(tool$testsList);
+  tool(tool$testsGet);
+  tool(tool$testsUpdate);
+  tool(tool$agentTestsRun);
+  tool(tool$agentTestsRunBatch);
+  tool(tool$agentTestsGetRun);
+
+  if (deps.dynamic) {
+    registerDynamicTools(deps.logger, server, getClient, toolMap, scopes);
+  }
+
+  return { server, tools };
+}

package/src/mcp-server/shared.ts

@@ -0,0 +1,78 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+type BinaryData =
+  | Uint8Array
+  | ArrayBuffer
+  | Blob
+  | ReadableStream
+  | Response
+  | string;
+
+const base64Schema = z.string().base64();
+
+export function bytesToBase64(u8arr: Uint8Array): string {
+  return btoa(String.fromCodePoint(...u8arr));
+}
+
+export async function consumeStream(
+  stream: ReadableStream<Uint8Array>,
+): Promise<Uint8Array> {
+  const reader = stream.getReader();
+  const chunks: Uint8Array[] = [];
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (value != null) chunks.push(value);
+      if (done) break;
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  return new Uint8Array(await new Blob(chunks).arrayBuffer());
+}
+
+export function isAsyncIterable(
+  value: unknown,
+): value is AsyncIterable<string> {
+  return (
+    typeof value === "object" && value != null && Symbol.asyncIterator in value
+  );
+}
+
+export function isBinaryData(value: unknown): value is BinaryData {
+  return (
+    value instanceof Uint8Array
+    || value instanceof ArrayBuffer
+    || value instanceof Blob
+    || value instanceof ReadableStream
+    || value instanceof Response
+    || typeof value === "string"
+  );
+}
+
+export async function valueToBase64(
+  value: BinaryData | null | undefined,
+): Promise<string | null> {
+  if (value == null) {
+    return null;
+  } else if (value instanceof Uint8Array) {
+    return bytesToBase64(value);
+  } else if (value instanceof ArrayBuffer) {
+    return bytesToBase64(new Uint8Array(value));
+  } else if (value instanceof Response || value instanceof Blob) {
+    return bytesToBase64(new Uint8Array(await value.arrayBuffer()));
+  } else if (value instanceof ReadableStream) {
+    return bytesToBase64(await consumeStream(value));
+  } else if (typeof value === "string") {
+    return base64Schema.parse(value);
+  } else {
+    value satisfies never;
+    throw new Error(`Unsupported image value type: ${typeof value}`);
+  }
+}

package/src/mcp-server/tools/agentTestsGetRun.ts

@@ -0,0 +1,47 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { agentTestsGetRun } from "../../funcs/agentTestsGetRun.js";
+import { GetAgentTestRunStatusAgentTestsRunTaskIdGetRequest$zodSchema } from "../../models/getagenttestrunstatusagenttestsruntaskidgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: GetAgentTestRunStatusAgentTestsRunTaskIdGetRequest$zodSchema,
+};
+
+export const tool$agentTestsGetRun: ToolDefinition<typeof args> = {
+  name: "get-agent-test-run",
+  description:
+    `Poll the status and results of an agent test run by task_id. Returns
+queued, in_progress, completed, failed, or aborted status. When
+completed, includes per-test-case pass/fail results and evaluator
+judgments. Use the task_id from run-agent-tests or
+run-agent-tests-batch.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "Get agent test run status",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await agentTestsGetRun(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/agentTestsRun.ts

@@ -0,0 +1,47 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { agentTestsRun } from "../../funcs/agentTestsRun.js";
+import { RunAgentTestAgentTestsAgentAgentUuidRunPostRequest$zodSchema } from "../../models/runagenttestagenttestsagentagentuuidrunpostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: RunAgentTestAgentTestsAgentAgentUuidRunPostRequest$zodSchema,
+};
+
+export const tool$agentTestsRun: ToolDefinition<typeof args> = {
+  name: "run-agent-tests",
+  description:
+    `Start a background LLM unit-test job for one agent. Runs all tests
+linked to the agent unless test_uuids is provided. Returns a task_id
+to poll with get-agent-test-run. The agent must belong to the caller's
+org and have a verified connection; otherwise the call fails with 400
+or 404.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Run agent tests",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await agentTestsRun(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/agentTestsRunBatch.ts

@@ -0,0 +1,47 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { agentTestsRunBatch } from "../../funcs/agentTestsRunBatch.js";
+import { RunTestsBatchAgentTestsRunPostRequest$zodSchema } from "../../models/runtestsbatchagenttestsrunpostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: RunTestsBatchAgentTestsRunPostRequest$zodSchema.optional(),
+};
+
+export const tool$agentTestsRunBatch: ToolDefinition<typeof args> = {
+  name: "run-agent-tests-batch",
+  description:
+    `Launch one test job per agent in the caller's org. Optionally restrict
+to agent_names; omit or pass an empty list to run every org agent.
+Agents with no linked tests or an unverified connection are skipped
+(reported under skipped) rather than failing the batch. Returns
+task_id values to poll with get-agent-test-run.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Run agent tests (batch)",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await agentTestsRunBatch(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/agentsCreate.ts

@@ -0,0 +1,45 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { agentsCreate } from "../../funcs/agentsCreate.js";
+import { CreateAgentEndpointAgentsPostRequest$zodSchema } from "../../models/createagentendpointagentspostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: CreateAgentEndpointAgentsPostRequest$zodSchema,
+};
+
+export const tool$agentsCreate: ToolDefinition<typeof args> = {
+  name: "create-agent",
+  description:
+    `Create a new agent in the caller's organization. Provide the agent's
+name, type, and config. Returns the created agent including its UUID.
+Requires a Calibrate API key (CALIBRATE_API_KEY).
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Create agent",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await agentsCreate(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/agentsGet.ts

@@ -0,0 +1,45 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { agentsGet } from "../../funcs/agentsGet.js";
+import { GetAgentEndpointAgentsAgentUuidGetRequest$zodSchema } from "../../models/getagentendpointagentsagentuuidgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: GetAgentEndpointAgentsAgentUuidGetRequest$zodSchema,
+};
+
+export const tool$agentsGet: ToolDefinition<typeof args> = {
+  name: "get-agent",
+  description:
+    `Fetch a single agent by UUID within the caller's org. Returns the
+agent's name, type, and config. Fails with 404 if the agent does not
+exist or belongs to another org.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "Get agent",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await agentsGet(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/agentsList.ts

@@ -0,0 +1,46 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { agentsList } from "../../funcs/agentsList.js";
+import { ListAgentsAgentsGetRequest$zodSchema } from "../../models/listagentsagentsgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: ListAgentsAgentsGetRequest$zodSchema.optional(),
+};
+
+export const tool$agentsList: ToolDefinition<typeof args> = {
+  name: "list-agents",
+  description:
+    `List every agent in the caller's organization. Returns each agent's
+UUID, name, type, and config. Use this to discover which agents exist
+before resolving names or launching test runs. Requires a Calibrate
+API key (CALIBRATE_API_KEY).
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "List agents",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await agentsList(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/agentsResolve.ts

@@ -0,0 +1,46 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { agentsResolve } from "../../funcs/agentsResolve.js";
+import { ResolveAgentNamesAgentsResolvePostRequest$zodSchema } from "../../models/resolveagentnamesagentsresolvepostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: ResolveAgentNamesAgentsResolvePostRequest$zodSchema,
+};
+
+export const tool$agentsResolve: ToolDefinition<typeof args> = {
+  name: "resolve-agent-names",
+  description:
+    `Map human-friendly agent names to UUIDs within the caller's org. Each
+name resolves to at most one agent; unknown names are returned under
+not_found. Use this before run-agent-tests when you have names instead
+of UUIDs. Does not create or modify agents.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Resolve agent names to UUIDs",
+    "destructiveHint": false,
+    "idempotentHint": true,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await agentsResolve(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/agentsUpdate.ts

@@ -0,0 +1,45 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { agentsUpdate } from "../../funcs/agentsUpdate.js";
+import { UpdateAgentEndpointAgentsAgentUuidPutRequest$zodSchema } from "../../models/updateagentendpointagentsagentuuidputop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: UpdateAgentEndpointAgentsAgentUuidPutRequest$zodSchema,
+};
+
+export const tool$agentsUpdate: ToolDefinition<typeof args> = {
+  name: "update-agent",
+  description:
+    `Update an existing agent by UUID within the caller's org. Supply the
+fields to change (name, type, config). Returns the updated agent.
+Fails with 404 if the agent does not exist or belongs to another org.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Update agent",
+    "destructiveHint": false,
+    "idempotentHint": true,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await agentsUpdate(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/testsBulkCreate.ts

@@ -0,0 +1,45 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { testsBulkCreate } from "../../funcs/testsBulkCreate.js";
+import { BulkUploadTestsTestsBulkPostRequest$zodSchema } from "../../models/bulkuploadteststestsbulkpostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: BulkUploadTestsTestsBulkPostRequest$zodSchema,
+};
+
+export const tool$testsBulkCreate: ToolDefinition<typeof args> = {
+  name: "bulk-create-tests",
+  description:
+    `Create multiple tests in the caller's organization in one call. Provide
+a list of test definitions (name, type, config). Returns the created
+tests including their UUIDs.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Create tests (bulk)",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await testsBulkCreate(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/testsCreate.ts

@@ -0,0 +1,45 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { testsCreate } from "../../funcs/testsCreate.js";
+import { CreateTestEndpointTestsPostRequest$zodSchema } from "../../models/createtestendpointtestspostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: CreateTestEndpointTestsPostRequest$zodSchema,
+};
+
+export const tool$testsCreate: ToolDefinition<typeof args> = {
+  name: "create-test",
+  description:
+    `Create a new test in the caller's organization. Provide the test's
+name, type, and config. Returns the created test including its UUID.
+Requires a Calibrate API key (CALIBRATE_API_KEY).
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Create test",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await testsCreate(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};