@dalmia/calibrate-mcp 0.0.13 → 0.0.15

package/src/funcs/evaluatorsGet.ts

@@ -0,0 +1,159 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { CalibrateMcpCore } from "../core.js";
+import { encodeSimple } from "../lib/encodings.js";
+import { compactMap } from "../lib/primitives.js";
+import { safeParse } from "../lib/schemas.js";
+import { RequestOptions } from "../lib/sdks.js";
+import { extractSecurity, resolveGlobalSecurity } from "../lib/security.js";
+import { pathToFunc } from "../lib/url.js";
+import { APIError } from "../models/errors/apierror.js";
+import {
+  ConnectionError,
+  InvalidRequestError,
+  RequestAbortedError,
+  RequestTimeoutError,
+  UnexpectedClientError,
+} from "../models/errors/httpclienterrors.js";
+import { SDKValidationError } from "../models/errors/sdkvalidationerror.js";
+import {
+  GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetRequest,
+  GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetRequest$zodSchema,
+} from "../models/getevaluatorendpointevaluatorsevaluatoruuidgetop.js";
+import { APICall, APIPromise } from "../types/async.js";
+import { Result } from "../types/fp.js";
+
+/**
+ * Get evaluator
+ *
+ * @remarks
+ * Get one evaluator with its full version history
+ */
+export function evaluatorsGet(
+  client$: CalibrateMcpCore,
+  request: GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetRequest,
+  options?: RequestOptions,
+): APIPromise<
+  Result<
+    Response,
+    | APIError
+    | SDKValidationError
+    | UnexpectedClientError
+    | InvalidRequestError
+    | RequestAbortedError
+    | RequestTimeoutError
+    | ConnectionError
+  >
+> {
+  return new APIPromise($do(
+    client$,
+    request,
+    options,
+  ));
+}
+
+async function $do(
+  client$: CalibrateMcpCore,
+  request: GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetRequest,
+  options?: RequestOptions,
+): Promise<
+  [
+    Result<
+      Response,
+      | APIError
+      | SDKValidationError
+      | UnexpectedClientError
+      | InvalidRequestError
+      | RequestAbortedError
+      | RequestTimeoutError
+      | ConnectionError
+    >,
+    APICall,
+  ]
+> {
+  const parsed$ = safeParse(
+    request,
+    (value$) =>
+      GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetRequest$zodSchema.parse(
+        value$,
+      ),
+    "Input validation failed",
+  );
+  if (!parsed$.ok) {
+    return [parsed$, { status: "invalid" }];
+  }
+  const payload$ = parsed$.value;
+  const body$ = null;
+
+  const pathParams$ = {
+    evaluator_uuid: encodeSimple("evaluator_uuid", payload$.evaluator_uuid, {
+      explode: false,
+      charEncoding: "percent",
+    }),
+  };
+  const path$ = pathToFunc("/evaluators/{evaluator_uuid}")(
+    pathParams$,
+  );
+
+  const headers$ = new Headers(compactMap({
+    Accept: "application/json",
+    "X-API-Key": encodeSimple("X-API-Key", payload$.xAPIKey, {
+      explode: false,
+      charEncoding: "none",
+    }),
+  }));
+  const securityInput = await extractSecurity(client$._options.security);
+  const requestSecurity = resolveGlobalSecurity(securityInput);
+
+  const context = {
+    options: client$._options,
+    baseURL: options?.serverURL ?? client$._baseURL ?? "",
+    operationID: "get_evaluator_endpoint_evaluators__evaluator_uuid__get",
+    oAuth2Scopes: null,
+    resolvedSecurity: requestSecurity,
+    securitySource: client$._options.security,
+    retryConfig: options?.retries
+      || client$._options.retryConfig
+      || { strategy: "none" },
+    retryCodes: options?.retryCodes || [
+      "429",
+      "500",
+      "502",
+      "503",
+      "504",
+    ],
+  };
+
+  const requestRes = client$._createRequest(context, {
+    security: requestSecurity,
+    method: "GET",
+    baseURL: options?.serverURL,
+    path: path$,
+    headers: headers$,
+    body: body$,
+    userAgent: client$._options.userAgent,
+    timeoutMs: options?.timeoutMs || client$._options.timeoutMs
+      || -1,
+  }, options);
+  if (!requestRes.ok) {
+    return [requestRes, { status: "invalid" }];
+  }
+  const req$ = requestRes.value;
+
+  const doResult = await client$._do(req$, {
+    context,
+    errorCodes: [],
+    retryConfig: context.retryConfig,
+    retryCodes: context.retryCodes,
+  });
+  if (!doResult.ok) {
+    return [doResult, { status: "request-error", request: req$ }];
+  }
+  return [doResult, {
+    status: "complete",
+    "request": req$,
+    response: doResult.value,
+  }];
+}

package/src/funcs/evaluatorsList.ts

@@ -0,0 +1,154 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { CalibrateMcpCore } from "../core.js";
+import { encodeFormQuery, encodeSimple } from "../lib/encodings.js";
+import { compactMap } from "../lib/primitives.js";
+import { safeParse } from "../lib/schemas.js";
+import { RequestOptions } from "../lib/sdks.js";
+import { extractSecurity, resolveGlobalSecurity } from "../lib/security.js";
+import { pathToFunc } from "../lib/url.js";
+import { APIError } from "../models/errors/apierror.js";
+import {
+  ConnectionError,
+  InvalidRequestError,
+  RequestAbortedError,
+  RequestTimeoutError,
+  UnexpectedClientError,
+} from "../models/errors/httpclienterrors.js";
+import { SDKValidationError } from "../models/errors/sdkvalidationerror.js";
+import {
+  ListEvaluatorsEvaluatorsGetRequest,
+  ListEvaluatorsEvaluatorsGetRequest$zodSchema,
+} from "../models/listevaluatorsevaluatorsgetop.js";
+import { APICall, APIPromise } from "../types/async.js";
+import { Result } from "../types/fp.js";
+
+/**
+ * List evaluators
+ *
+ * @remarks
+ * List your evaluators
+ */
+export function evaluatorsList(
+  client$: CalibrateMcpCore,
+  request?: ListEvaluatorsEvaluatorsGetRequest | undefined,
+  options?: RequestOptions,
+): APIPromise<
+  Result<
+    Response,
+    | APIError
+    | SDKValidationError
+    | UnexpectedClientError
+    | InvalidRequestError
+    | RequestAbortedError
+    | RequestTimeoutError
+    | ConnectionError
+  >
+> {
+  return new APIPromise($do(
+    client$,
+    request,
+    options,
+  ));
+}
+
+async function $do(
+  client$: CalibrateMcpCore,
+  request?: ListEvaluatorsEvaluatorsGetRequest | undefined,
+  options?: RequestOptions,
+): Promise<
+  [
+    Result<
+      Response,
+      | APIError
+      | SDKValidationError
+      | UnexpectedClientError
+      | InvalidRequestError
+      | RequestAbortedError
+      | RequestTimeoutError
+      | ConnectionError
+    >,
+    APICall,
+  ]
+> {
+  const parsed$ = safeParse(
+    request,
+    (value$) =>
+      ListEvaluatorsEvaluatorsGetRequest$zodSchema.optional().parse(value$),
+    "Input validation failed",
+  );
+  if (!parsed$.ok) {
+    return [parsed$, { status: "invalid" }];
+  }
+  const payload$ = parsed$.value;
+  const body$ = null;
+  const path$ = pathToFunc("/evaluators")();
+  const query$ = encodeFormQuery({
+    "data_type": payload$?.data_type,
+    "evaluator_type": payload$?.evaluator_type,
+    "include_defaults": payload$?.include_defaults,
+  });
+
+  const headers$ = new Headers(compactMap({
+    Accept: "application/json",
+    "X-API-Key": encodeSimple("X-API-Key", payload$?.xAPIKey, {
+      explode: false,
+      charEncoding: "none",
+    }),
+  }));
+  const securityInput = await extractSecurity(client$._options.security);
+  const requestSecurity = resolveGlobalSecurity(securityInput);
+
+  const context = {
+    options: client$._options,
+    baseURL: options?.serverURL ?? client$._baseURL ?? "",
+    operationID: "list_evaluators_evaluators_get",
+    oAuth2Scopes: null,
+    resolvedSecurity: requestSecurity,
+    securitySource: client$._options.security,
+    retryConfig: options?.retries
+      || client$._options.retryConfig
+      || { strategy: "none" },
+    retryCodes: options?.retryCodes || [
+      "429",
+      "500",
+      "502",
+      "503",
+      "504",
+    ],
+  };
+
+  const requestRes = client$._createRequest(context, {
+    security: requestSecurity,
+    method: "GET",
+    baseURL: options?.serverURL,
+    path: path$,
+    headers: headers$,
+    query: query$,
+    body: body$,
+    userAgent: client$._options.userAgent,
+    timeoutMs: options?.timeoutMs || client$._options.timeoutMs
+      || -1,
+  }, options);
+  if (!requestRes.ok) {
+    return [requestRes, { status: "invalid" }];
+  }
+  const req$ = requestRes.value;
+
+  const doResult = await client$._do(req$, {
+    context,
+    errorCodes: [],
+    retryConfig: context.retryConfig,
+    retryCodes: context.retryCodes,
+  });
+  if (!doResult.ok) {
+    return [doResult, { status: "request-error", request: req$ }];
+  }
+  return [doResult, {
+    status: "complete",
+    "request": req$,
+    response: doResult.value,
+  }];
+}

package/src/funcs/testsBulkCreate.ts

@@ -92,10 +92,6 @@ async function $do(
       explode: false,
       charEncoding: "none",
     }),
-    "X-Org-UUID": encodeSimple("X-Org-UUID", payload$.xOrgUUID, {
-      explode: false,
-      charEncoding: "none",
-    }),
   }));
   const securityInput = await extractSecurity(client$._options.security);
   const requestSecurity = resolveGlobalSecurity(securityInput);

package/src/funcs/testsCreate.ts

@@ -92,10 +92,6 @@ async function $do(
       explode: false,
       charEncoding: "none",
     }),
-    "X-Org-UUID": encodeSimple("X-Org-UUID", payload$.xOrgUUID, {
-      explode: false,
-      charEncoding: "none",
-    }),
   }));
   const securityInput = await extractSecurity(client$._options.security);
   const requestSecurity = resolveGlobalSecurity(securityInput);

package/src/funcs/testsGet.ts

@@ -100,10 +100,6 @@ async function $do(
       explode: false,
       charEncoding: "none",
     }),
-    "X-Org-UUID": encodeSimple("X-Org-UUID", payload$.xOrgUUID, {
-      explode: false,
-      charEncoding: "none",
-    }),
   }));
   const securityInput = await extractSecurity(client$._options.security);
   const requestSecurity = resolveGlobalSecurity(securityInput);

package/src/funcs/testsList.ts

@@ -91,10 +91,6 @@ async function $do(
       explode: false,
       charEncoding: "none",
     }),
-    "X-Org-UUID": encodeSimple("X-Org-UUID", payload$?.xOrgUUID, {
-      explode: false,
-      charEncoding: "none",
-    }),
   }));
   const securityInput = await extractSecurity(client$._options.security);
   const requestSecurity = resolveGlobalSecurity(securityInput);

package/src/funcs/testsUpdate.ts

@@ -102,10 +102,6 @@ async function $do(
       explode: false,
       charEncoding: "none",
     }),
-    "X-Org-UUID": encodeSimple("X-Org-UUID", payload$.xOrgUUID, {
-      explode: false,
-      charEncoding: "none",
-    }),
   }));
   const securityInput = await extractSecurity(client$._options.security);
   const requestSecurity = resolveGlobalSecurity(securityInput);

package/src/landing-page.ts

@@ -930,7 +930,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
         <h1>Instructions</h1>
         <p>One-click installation for Claude Desktop users</p>
         <div class="instruction-item">
-          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.13/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
+          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.15/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
             📥 Download MCP Bundle
           </a>
         </div>

package/src/lib/config.ts

@@ -65,8 +65,8 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
 export const SDK_METADATA = {
   language: "typescript",
   openapiDocVersion: "0.1.0",
-  sdkVersion: "0.0.13",
+  sdkVersion: "0.0.15",
   genVersion: "2.915.1",
   userAgent:
-    "speakeasy-sdk/mcp-typescript 0.0.13 2.915.1 0.1.0 @dalmia/calibrate-mcp",
+    "speakeasy-sdk/mcp-typescript 0.0.15 2.915.1 0.1.0 @dalmia/calibrate-mcp",
 } as const;

package/src/mcp-server/mcp-server.ts

@@ -21,7 +21,7 @@ const routes = buildRouteMap({
 export const app = buildApplication(routes, {
   name: "mcp",
   versionInfo: {
-    currentVersion: "0.0.13",
+    currentVersion: "0.0.15",
   },
 });
 

package/src/mcp-server/server.ts

@@ -22,9 +22,29 @@ import { tool$agentsGet } from "./tools/agentsGet.js";
 import { tool$agentsList } from "./tools/agentsList.js";
 import { tool$agentsResolve } from "./tools/agentsResolve.js";
 import { tool$agentsUpdate } from "./tools/agentsUpdate.js";
+import { tool$agentsVerifyConnection } from "./tools/agentsVerifyConnection.js";
+import { tool$agentTestsBenchmark } from "./tools/agentTestsBenchmark.js";
+import { tool$agentTestsGetBenchmark } from "./tools/agentTestsGetBenchmark.js";
 import { tool$agentTestsGetRun } from "./tools/agentTestsGetRun.js";
+import { tool$agentTestsLink } from "./tools/agentTestsLink.js";
+import { tool$agentTestsListForAgent } from "./tools/agentTestsListForAgent.js";
+import { tool$agentTestsListRunsForAgent } from "./tools/agentTestsListRunsForAgent.js";
 import { tool$agentTestsRun } from "./tools/agentTestsRun.js";
 import { tool$agentTestsRunBatch } from "./tools/agentTestsRunBatch.js";
+import { tool$annotationTasksAddItems } from "./tools/annotationTasksAddItems.js";
+import { tool$annotationTasksCreate } from "./tools/annotationTasksCreate.js";
+import { tool$annotationTasksCreateEvaluatorRun } from "./tools/annotationTasksCreateEvaluatorRun.js";
+import { tool$annotationTasksGet } from "./tools/annotationTasksGet.js";
+import { tool$annotationTasksGetAgreement } from "./tools/annotationTasksGetAgreement.js";
+import { tool$annotationTasksGetEvaluatorRun } from "./tools/annotationTasksGetEvaluatorRun.js";
+import { tool$annotationTasksGetSummary } from "./tools/annotationTasksGetSummary.js";
+import { tool$annotationTasksLinkEvaluator } from "./tools/annotationTasksLinkEvaluator.js";
+import { tool$annotationTasksList } from "./tools/annotationTasksList.js";
+import { tool$annotationTasksUpdateItems } from "./tools/annotationTasksUpdateItems.js";
+import { tool$evaluatorsCreate } from "./tools/evaluatorsCreate.js";
+import { tool$evaluatorsCreateVersion } from "./tools/evaluatorsCreateVersion.js";
+import { tool$evaluatorsGet } from "./tools/evaluatorsGet.js";
+import { tool$evaluatorsList } from "./tools/evaluatorsList.js";
 import { tool$testsBulkCreate } from "./tools/testsBulkCreate.js";
 import { tool$testsCreate } from "./tools/testsCreate.js";
 import { tool$testsGet } from "./tools/testsGet.js";
@@ -44,7 +64,7 @@ export function createMCPServer(deps: {
 }) {
   const server = new McpServer({
     name: "CalibrateMcp",
-    version: "0.0.13",
+    version: "0.0.15",
   });
 
   const getClient = deps.getSDK || (() =>
@@ -89,6 +109,7 @@ export function createMCPServer(deps: {
   const register = { tool, resource, resourceTemplate, prompt };
   void register; // suppress unused warnings
 
+  tool(tool$agentsVerifyConnection);
   tool(tool$agentsResolve);
   tool(tool$agentsCreate);
   tool(tool$agentsList);
@@ -99,9 +120,28 @@ export function createMCPServer(deps: {
   tool(tool$testsList);
   tool(tool$testsGet);
   tool(tool$testsUpdate);
+  tool(tool$agentTestsLink);
+  tool(tool$agentTestsListForAgent);
+  tool(tool$agentTestsListRunsForAgent);
   tool(tool$agentTestsRun);
   tool(tool$agentTestsRunBatch);
   tool(tool$agentTestsGetRun);
+  tool(tool$agentTestsBenchmark);
+  tool(tool$agentTestsGetBenchmark);
+  tool(tool$evaluatorsCreate);
+  tool(tool$evaluatorsList);
+  tool(tool$evaluatorsGet);
+  tool(tool$evaluatorsCreateVersion);
+  tool(tool$annotationTasksCreate);
+  tool(tool$annotationTasksList);
+  tool(tool$annotationTasksGet);
+  tool(tool$annotationTasksLinkEvaluator);
+  tool(tool$annotationTasksAddItems);
+  tool(tool$annotationTasksUpdateItems);
+  tool(tool$annotationTasksCreateEvaluatorRun);
+  tool(tool$annotationTasksGetEvaluatorRun);
+  tool(tool$annotationTasksGetAgreement);
+  tool(tool$annotationTasksGetSummary);
 
   if (deps.dynamic) {
     registerDynamicTools(deps.logger, server, getClient, toolMap, scopes);

package/src/mcp-server/tools/agentTestsBenchmark.ts

@@ -0,0 +1,44 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { agentTestsBenchmark } from "../../funcs/agentTestsBenchmark.js";
+import { RunAgentBenchmarkAgentTestsAgentAgentUuidBenchmarkPostRequest$zodSchema } from "../../models/runagentbenchmarkagenttestsagentagentuuidbenchmarkpostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request:
+    RunAgentBenchmarkAgentTestsAgentAgentUuidBenchmarkPostRequest$zodSchema,
+};
+
+export const tool$agentTestsBenchmark: ToolDefinition<typeof args> = {
+  name: "run-agent-benchmark",
+  description:
+    `Start a multi-model benchmark over an agent's linked tests as a background job. Optionally restrict to a subset of linked tests. Returns a task_id to poll with get-agent-benchmark.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Run agent benchmark",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await agentTestsBenchmark(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/agentTestsGetBenchmark.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { agentTestsGetBenchmark } from "../../funcs/agentTestsGetBenchmark.js";
+import { GetBenchmarkStatusAgentTestsBenchmarkTaskIdGetRequest$zodSchema } from "../../models/getbenchmarkstatusagenttestsbenchmarktaskidgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: GetBenchmarkStatusAgentTestsBenchmarkTaskIdGetRequest$zodSchema,
+};
+
+export const tool$agentTestsGetBenchmark: ToolDefinition<typeof args> = {
+  name: "get-agent-benchmark",
+  description:
+    `Get the results of a benchmark run, including per-model scores and the leaderboard.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "Get benchmark status",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await agentTestsGetBenchmark(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/agentTestsLink.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { agentTestsLink } from "../../funcs/agentTestsLink.js";
+import { CreateAgentTestLinksAgentTestsPostRequest$zodSchema } from "../../models/createagenttestlinksagenttestspostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: CreateAgentTestLinksAgentTestsPostRequest$zodSchema,
+};
+
+export const tool$agentTestsLink: ToolDefinition<typeof args> = {
+  name: "link-tests-to-agent",
+  description:
+    `Link one or more existing tests to an agent so they run together. Tests already linked are skipped. The agent and every test must be in your workspace.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Link tests to agent",
+    "destructiveHint": false,
+    "idempotentHint": true,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await agentTestsLink(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/agentTestsListForAgent.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { agentTestsListForAgent } from "../../funcs/agentTestsListForAgent.js";
+import { GetAgentTestsEndpointAgentTestsAgentAgentUuidTestsGetRequest$zodSchema } from "../../models/getagenttestsendpointagenttestsagentagentuuidtestsgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request:
+    GetAgentTestsEndpointAgentTestsAgentAgentUuidTestsGetRequest$zodSchema,
+};
+
+export const tool$agentTestsListForAgent: ToolDefinition<typeof args> = {
+  name: "list-agent-tests",
+  description: `List the tests linked to an agent.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "List tests for an agent",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await agentTestsListForAgent(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};