@dalmia/calibrate-mcp 0.0.44 → 0.0.46

package/src/funcs/agentTestsBulkUnlink.ts

@@ -0,0 +1,151 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { CalibrateMcpCore } from "../core.js";
+import { encodeJSON, encodeSimple } from "../lib/encodings.js";
+import { compactMap } from "../lib/primitives.js";
+import { safeParse } from "../lib/schemas.js";
+import { RequestOptions } from "../lib/sdks.js";
+import { extractSecurity, resolveGlobalSecurity } from "../lib/security.js";
+import { pathToFunc } from "../lib/url.js";
+import {
+  BulkDeleteAgentTestLinksAgentTestsBulkUnlinkPostRequest,
+  BulkDeleteAgentTestLinksAgentTestsBulkUnlinkPostRequest$zodSchema,
+} from "../models/bulkdeleteagenttestlinksagenttestsbulkunlinkpostop.js";
+import { APIError } from "../models/errors/apierror.js";
+import {
+  ConnectionError,
+  InvalidRequestError,
+  RequestAbortedError,
+  RequestTimeoutError,
+  UnexpectedClientError,
+} from "../models/errors/httpclienterrors.js";
+import { SDKValidationError } from "../models/errors/sdkvalidationerror.js";
+import { APICall, APIPromise } from "../types/async.js";
+import { Result } from "../types/fp.js";
+
+/**
+ * Bulk unlink tests from agent
+ *
+ * @remarks
+ * Unlink one or more tests from an agent. Tests that are not linked are skipped.
+ */
+export function agentTestsBulkUnlink(
+  client$: CalibrateMcpCore,
+  request: BulkDeleteAgentTestLinksAgentTestsBulkUnlinkPostRequest,
+  options?: RequestOptions,
+): APIPromise<
+  Result<
+    Response,
+    | APIError
+    | SDKValidationError
+    | UnexpectedClientError
+    | InvalidRequestError
+    | RequestAbortedError
+    | RequestTimeoutError
+    | ConnectionError
+  >
+> {
+  return new APIPromise($do(
+    client$,
+    request,
+    options,
+  ));
+}
+
+async function $do(
+  client$: CalibrateMcpCore,
+  request: BulkDeleteAgentTestLinksAgentTestsBulkUnlinkPostRequest,
+  options?: RequestOptions,
+): Promise<
+  [
+    Result<
+      Response,
+      | APIError
+      | SDKValidationError
+      | UnexpectedClientError
+      | InvalidRequestError
+      | RequestAbortedError
+      | RequestTimeoutError
+      | ConnectionError
+    >,
+    APICall,
+  ]
+> {
+  const parsed$ = safeParse(
+    request,
+    (value$) =>
+      BulkDeleteAgentTestLinksAgentTestsBulkUnlinkPostRequest$zodSchema.parse(
+        value$,
+      ),
+    "Input validation failed",
+  );
+  if (!parsed$.ok) {
+    return [parsed$, { status: "invalid" }];
+  }
+  const payload$ = parsed$.value;
+  const body$ = encodeJSON("body", payload$.body, { explode: true });
+  const path$ = pathToFunc("/agent-tests/bulk-unlink")();
+
+  const headers$ = new Headers(compactMap({
+    "Content-Type": "application/json",
+    Accept: "application/json",
+    "X-API-Key": encodeSimple("X-API-Key", payload$.xAPIKey, {
+      explode: false,
+      charEncoding: "none",
+    }),
+  }));
+  const securityInput = await extractSecurity(client$._options.security);
+  const requestSecurity = resolveGlobalSecurity(securityInput);
+
+  const context = {
+    options: client$._options,
+    baseURL: options?.serverURL ?? client$._baseURL ?? "",
+    operationID: "bulk_delete_agent_test_links_agent_tests_bulk_unlink_post",
+    oAuth2Scopes: null,
+    resolvedSecurity: requestSecurity,
+    securitySource: client$._options.security,
+    retryConfig: options?.retries
+      || client$._options.retryConfig
+      || { strategy: "none" },
+    retryCodes: options?.retryCodes || [
+      "429",
+      "500",
+      "502",
+      "503",
+      "504",
+    ],
+  };
+
+  const requestRes = client$._createRequest(context, {
+    security: requestSecurity,
+    method: "POST",
+    baseURL: options?.serverURL,
+    path: path$,
+    headers: headers$,
+    body: body$,
+    userAgent: client$._options.userAgent,
+    timeoutMs: options?.timeoutMs || client$._options.timeoutMs
+      || -1,
+  }, options);
+  if (!requestRes.ok) {
+    return [requestRes, { status: "invalid" }];
+  }
+  const req$ = requestRes.value;
+
+  const doResult = await client$._do(req$, {
+    context,
+    errorCodes: [],
+    retryConfig: context.retryConfig,
+    retryCodes: context.retryCodes,
+  });
+  if (!doResult.ok) {
+    return [doResult, { status: "request-error", request: req$ }];
+  }
+  return [doResult, {
+    status: "complete",
+    "request": req$,
+    response: doResult.value,
+  }];
+}

package/src/funcs/agentTestsListForAgent.ts

@@ -99,6 +99,8 @@ async function $do(
     "limit": payload$.limit,
     "offset": payload$.offset,
     "q": payload$.q,
+    "q_mode": payload$.q_mode,
+    "type": payload$.type,
   });
 
   const headers$ = new Headers(compactMap({

package/src/landing-page.ts

@@ -930,7 +930,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
         <h1>Instructions</h1>
         <p>One-click installation for Claude Desktop users</p>
         <div class="instruction-item">
-          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.44/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
+          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.46/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
             📥 Download MCP Bundle
           </a>
         </div>

package/src/lib/config.ts

@@ -65,8 +65,8 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
 export const SDK_METADATA = {
   language: "typescript",
   openapiDocVersion: "0.1.0",
-  sdkVersion: "0.0.44",
+  sdkVersion: "0.0.46",
   genVersion: "2.915.1",
   userAgent:
-    "speakeasy-sdk/mcp-typescript 0.0.44 2.915.1 0.1.0 @dalmia/calibrate-mcp",
+    "speakeasy-sdk/mcp-typescript 0.0.46 2.915.1 0.1.0 @dalmia/calibrate-mcp",
 } as const;

package/src/mcp-server/mcp-server.ts

@@ -21,7 +21,7 @@ const routes = buildRouteMap({
 export const app = buildApplication(routes, {
   name: "mcp",
   versionInfo: {
-    currentVersion: "0.0.44",
+    currentVersion: "0.0.46",
   },
 });
 

package/src/mcp-server/server.ts

@@ -26,6 +26,7 @@ import { tool$agentsResolve } from "./tools/agentsResolve.js";
 import { tool$agentsUpdate } from "./tools/agentsUpdate.js";
 import { tool$agentsVerifyConnection } from "./tools/agentsVerifyConnection.js";
 import { tool$agentTestsBenchmark } from "./tools/agentTestsBenchmark.js";
+import { tool$agentTestsBulkUnlink } from "./tools/agentTestsBulkUnlink.js";
 import { tool$agentTestsGetBenchmark } from "./tools/agentTestsGetBenchmark.js";
 import { tool$agentTestsGetRun } from "./tools/agentTestsGetRun.js";
 import { tool$agentTestsLink } from "./tools/agentTestsLink.js";
@@ -71,7 +72,7 @@ export function createMCPServer(deps: {
 }) {
   const server = new McpServer({
     name: "CalibrateMcp",
-    version: "0.0.44",
+    version: "0.0.46",
   });
 
   const getClient = deps.getSDK || (() =>
@@ -132,6 +133,7 @@ export function createMCPServer(deps: {
   tool(tool$agentTestsLink);
   tool(tool$agentTestsListForAgent);
   tool(tool$agentTestsListRunsForAgent);
+  tool(tool$agentTestsBulkUnlink);
   tool(tool$agentTestsRun);
   tool(tool$agentTestsRunBatch);
   tool(tool$agentTestsGetRun);

package/src/mcp-server/tools/agentTestsBulkUnlink.ts

@@ -0,0 +1,48 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { agentTestsBulkUnlink } from "../../funcs/agentTestsBulkUnlink.js";
+import { BulkDeleteAgentTestLinksAgentTestsBulkUnlinkPostRequest$zodSchema } from "../../models/bulkdeleteagenttestlinksagenttestsbulkunlinkpostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: BulkDeleteAgentTestLinksAgentTestsBulkUnlinkPostRequest$zodSchema,
+};
+
+export const tool$agentTestsBulkUnlink: ToolDefinition<typeof args> = {
+  name: "bulk-unlink-agent-tests",
+  description:
+    `Remove one or more tests from an agent in a single call. The tests
+themselves are kept and stay available to other agents; only the links
+to this agent go. Tests that were not linked are skipped, so
+deleted_count can be lower than the number of ids sent. Fails with 404
+when the agent is not in the caller's org and 400 when test_uuids is
+empty.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Unlink tests from an agent",
+    "destructiveHint": true,
+    "idempotentHint": true,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await agentTestsBulkUnlink(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/models/agenttestbulkdelete.ts

@@ -0,0 +1,16 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type AgentTestBulkDelete = {
+  agent_uuid: string;
+  test_uuids: Array<string>;
+};
+
+export const AgentTestBulkDelete$zodSchema: z.ZodType<AgentTestBulkDelete> = z
+  .object({
+    agent_uuid: z.string().describe("Agent to unlink tests from"),
+    test_uuids: z.array(z.string()).describe("Tests to unlink from the agent"),
+  });

package/src/models/agenttestsbulkunlinkresponse.ts

@@ -0,0 +1,19 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type AgentTestsBulkUnlinkResponse = {
+  deleted_count: number;
+  message: string;
+};
+
+export const AgentTestsBulkUnlinkResponse$zodSchema: z.ZodType<
+  AgentTestsBulkUnlinkResponse
+> = z.object({
+  deleted_count: z.int().describe(
+    "Number of links removed. Tests that were not linked are excluded",
+  ),
+  message: z.string().describe("Confirmation message"),
+});

package/src/models/annotationitempayload.ts

@@ -7,6 +7,7 @@ import * as z from "zod";
 export type AnnotationItemPayload = {
   payload?: any | undefined;
   annotations?: { [k: string]: any } | null | undefined;
+  evaluator_results?: { [k: string]: any } | null | undefined;
 };
 
 export const AnnotationItemPayload$zodSchema: z.ZodType<AnnotationItemPayload> =
@@ -14,6 +15,10 @@ export const AnnotationItemPayload$zodSchema: z.ZodType<AnnotationItemPayload> =
     annotations: z.record(z.string(), z.any()).nullable().optional().describe(
       "Human annotations to seed, keyed by evaluator ID. Each evaluator ID must be linked to the task. Put the judgement in `value`, a bool for binary or a number for rating, with optional `reasoning`. Requires `annotator_id`",
     ),
+    evaluator_results: z.record(z.string(), z.any()).nullable().optional()
+      .describe(
+        "Evaluator scores to record, keyed by evaluator ID. Each evaluator ID must be linked to the task. Put the score in `value`, a bool for binary or a number within the scale for rating, with optional `reasoning` and `version_number`. Omit `version_number` to record against the evaluator's live version. An item recording a tool call takes the tool-call evaluator and no other, and every other item takes any evaluator except that one",
+      ),
     payload: z.any().optional().describe(
       "The item's payload. `name` is required and unique within the task for every type. The other fields depend on the task `type`:\n\n- `stt`: `name`, `reference_transcript`, `predicted_transcript`\n- `llm`: `name`, `chat_history` (list of `{role, content}` turns ending at the user turn), `agent_response` (the reply to judge), optional `evaluator_variables`\n- `llm-general`: `name`, `input`, `output`, optional `evaluator_variables`\n- `conversation`: `name`, `transcript` (list of `{role, content}` turns), optional `evaluator_variables`\n\n`evaluator_variables` maps an evaluator ID to that evaluator's `{{\"{{\"}}variable}}` values for this item",
     ),

package/src/models/bulkcreateitemsresponse.ts

@@ -10,6 +10,8 @@ export type BulkCreateItemsResponse = {
   new_item_ids?: Array<string> | null | undefined;
   existing_item_ids?: Array<string> | null | undefined;
   annotation_job_id?: string | null | undefined;
+  evaluator_result_count?: number | null | undefined;
+  evaluator_run_job_id?: string | null | undefined;
 };
 
 export const BulkCreateItemsResponse$zodSchema: z.ZodType<
@@ -19,6 +21,12 @@ export const BulkCreateItemsResponse$zodSchema: z.ZodType<
     "ID of the labelling job that holds the annotations you provided, present only when you send annotations with the items",
   ),
   count: z.int().describe("How many items the request resolved to"),
+  evaluator_result_count: z.int().nullable().optional().describe(
+    "How many evaluator scores were stored, present only when you send `evaluator_results` with the items",
+  ),
+  evaluator_run_job_id: z.string().nullable().optional().describe(
+    "ID of the evaluator-run job that holds the scores you provided, present only when you send `evaluator_results` with the items",
+  ),
   existing_item_ids: z.array(z.string()).nullable().optional().describe(
     "IDs of items that already existed by name and were reused",
   ),

package/src/models/bulkdeleteagenttestlinksagenttestsbulkunlinkpostop.ts

@@ -0,0 +1,42 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  AgentTestBulkDelete,
+  AgentTestBulkDelete$zodSchema,
+} from "./agenttestbulkdelete.js";
+import {
+  AgentTestsBulkUnlinkResponse,
+  AgentTestsBulkUnlinkResponse$zodSchema,
+} from "./agenttestsbulkunlinkresponse.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+
+export type BulkDeleteAgentTestLinksAgentTestsBulkUnlinkPostRequest = {
+  xAPIKey?: string | null | undefined;
+  body: AgentTestBulkDelete;
+};
+
+export const BulkDeleteAgentTestLinksAgentTestsBulkUnlinkPostRequest$zodSchema:
+  z.ZodType<BulkDeleteAgentTestLinksAgentTestsBulkUnlinkPostRequest> = z.object(
+    {
+      body: AgentTestBulkDelete$zodSchema,
+      xAPIKey: z.string().nullable().optional(),
+    },
+  );
+
+export type BulkDeleteAgentTestLinksAgentTestsBulkUnlinkPostResponse =
+  | AgentTestsBulkUnlinkResponse
+  | HTTPValidationError;
+
+export const BulkDeleteAgentTestLinksAgentTestsBulkUnlinkPostResponse$zodSchema:
+  z.ZodType<BulkDeleteAgentTestLinksAgentTestsBulkUnlinkPostResponse> = z.union(
+    [
+      AgentTestsBulkUnlinkResponse$zodSchema,
+      HTTPValidationError$zodSchema,
+    ],
+  );

package/src/models/getagenttestsendpointagenttestsagentagentuuidtestsgetop.ts

@@ -3,6 +3,7 @@
  */
 
 import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
 import {
   HTTPValidationError,
   HTTPValidationError$zodSchema,
@@ -12,9 +13,32 @@ import {
   PaginatedResponseTestListResponse$zodSchema,
 } from "./paginatedresponsetestlistresponse.js";
 
+/**
+ * How to match `q` against the searched fields
+ */
+export const QMode = {
+  Contains: "contains",
+  StartsWith: "starts_with",
+  EndsWith: "ends_with",
+  Exact: "exact",
+} as const;
+/**
+ * How to match `q` against the searched fields
+ */
+export type QMode = ClosedEnum<typeof QMode>;
+
+export const QMode$zodSchema = z.enum([
+  "contains",
+  "starts_with",
+  "ends_with",
+  "exact",
+]).describe("How to match `q` against the searched fields");
+
 export type GetAgentTestsEndpointAgentTestsAgentAgentUuidTestsGetRequest = {
   agent_uuid: string;
+  type?: Array<string> | null | undefined;
   q?: string | null | undefined;
+  q_mode?: QMode | undefined;
   limit?: number | null | undefined;
   offset?: number | undefined;
   xAPIKey?: string | null | undefined;
@@ -31,7 +55,13 @@ export const GetAgentTestsEndpointAgentTestsAgentAgentUuidTestsGetRequest$zodSch
         "Number of items to skip before returning results",
       ),
       q: z.string().describe(
-        "Case-insensitive substring search on `name`. Blank is a no-op",
+        "Case-insensitive search on `name`. Blank is a no-op",
+      ).nullable().optional(),
+      q_mode: QMode$zodSchema.default("contains").describe(
+        "How to match `q` against the searched fields",
+      ),
+      type: z.array(z.string()).describe(
+        "Keep only tests of these types. Repeat the parameter or pass one comma-separated value. Accepts `response`, `tool_call`, `conversation`, `general`",
       ).nullable().optional(),
       xAPIKey: z.string().nullable().optional(),
     });

package/src/tool-names.ts

@@ -64,6 +64,10 @@ export const toolNames: Array<{ name: string; description: string }>= [
     "name": "list-agent-test-runs",
     "description": "List an agent's test and benchmark runs, most recently created first, with their status and results.\n"
   },
+  {
+    "name": "bulk-unlink-agent-tests",
+    "description": "Remove one or more tests from an agent in a single call. The tests\nthemselves are kept and stay available to other agents; only the links\nto this agent go. Tests that were not linked are skipped, so\ndeleted_count can be lower than the number of ids sent. Fails with 404\nwhen the agent is not in the caller's org and 400 when test_uuids is\nempty.\n"
+  },
   {
     "name": "run-agent-tests",
     "description": "Start a background LLM unit-test job for one agent. Runs all tests\nlinked to the agent unless test_uuids is provided. Returns a task_id\nto poll with get-agent-test-run. The agent must belong to the caller's\norg and have a verified connection; otherwise the call fails with 400\nor 404.\n"