@dalmia/calibrate-mcp 0.0.14 → 0.0.16

package/src/mcp-server/tools/annotationTasksCreateEvaluatorRun.ts

@@ -0,0 +1,46 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksCreateEvaluatorRun } from "../../funcs/annotationTasksCreateEvaluatorRun.js";
+import { StartEvaluatorRunAnnotationTasksTaskUuidEvaluatorRunsPostRequest$zodSchema } from "../../models/startevaluatorrunannotationtaskstaskuuidevaluatorrunspostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request:
+    StartEvaluatorRunAnnotationTasksTaskUuidEvaluatorRunsPostRequest$zodSchema,
+};
+
+export const tool$annotationTasksCreateEvaluatorRun: ToolDefinition<
+  typeof args
+> = {
+  name: "run-annotation-evaluators",
+  description:
+    `Run one or more of a task's evaluators over its items as a background job. Returns a job_uuid to poll with get-annotation-evaluator-run.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Run evaluators on items",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await annotationTasksCreateEvaluatorRun(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/annotationTasksGet.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksGet } from "../../funcs/annotationTasksGet.js";
+import { GetAnnotationTaskEndpointAnnotationTasksTaskUuidGetRequest$zodSchema } from "../../models/getannotationtaskendpointannotationtaskstaskuuidgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: GetAnnotationTaskEndpointAnnotationTasksTaskUuidGetRequest$zodSchema,
+};
+
+export const tool$annotationTasksGet: ToolDefinition<typeof args> = {
+  name: "get-annotation-task",
+  description:
+    `Get one annotation task with its evaluators, items, and labelling jobs.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "Get annotation task",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await annotationTasksGet(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/annotationTasksGetAgreement.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksGetAgreement } from "../../funcs/annotationTasksGetAgreement.js";
+import { TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest$zodSchema } from "../../models/taskagreementannotationtaskstaskuuidagreementgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest$zodSchema,
+};
+
+export const tool$annotationTasksGetAgreement: ToolDefinition<typeof args> = {
+  name: "get-annotation-agreement",
+  description:
+    `Get human-vs-human and human-vs-evaluator agreement metrics for a task, with a trend series.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "Get task agreement",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await annotationTasksGetAgreement(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/annotationTasksGetEvaluatorRun.ts

@@ -0,0 +1,47 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksGetEvaluatorRun } from "../../funcs/annotationTasksGetEvaluatorRun.js";
+import {
+  GetEvaluatorRunJobAnnotationTasksTaskUuidEvaluatorRunsJobUuidGetRequest$zodSchema,
+} from "../../models/getevaluatorrunjobannotationtaskstaskuuidevaluatorrunsjobuuidgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request:
+    GetEvaluatorRunJobAnnotationTasksTaskUuidEvaluatorRunsJobUuidGetRequest$zodSchema,
+};
+
+export const tool$annotationTasksGetEvaluatorRun: ToolDefinition<typeof args> =
+  {
+    name: "get-annotation-evaluator-run",
+    description:
+      `Get one evaluator-run job with its results and human-agreement summary.
+`,
+    scopes: ["read"],
+    annotations: {
+      "title": "Get evaluator run",
+      "destructiveHint": false,
+      "idempotentHint": false,
+      "openWorldHint": true,
+      "readOnlyHint": true,
+    },
+    args,
+    tool: async (client, args, ctx) => {
+      const [result] = await annotationTasksGetEvaluatorRun(
+        client,
+        args.request,
+        { fetchOptions: { signal: ctx.signal } },
+      ).$inspect();
+
+      if (!result.ok) {
+        return {
+          content: [{ type: "text", text: result.error.message }],
+          isError: true,
+        };
+      }
+
+      return formatResult(result.value);
+    },
+  };

package/src/mcp-server/tools/annotationTasksGetSummary.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksGetSummary } from "../../funcs/annotationTasksGetSummary.js";
+import { TaskSummaryAnnotationTasksTaskUuidSummaryGetRequest$zodSchema } from "../../models/tasksummaryannotationtaskstaskuuidsummarygetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: TaskSummaryAnnotationTasksTaskUuidSummaryGetRequest$zodSchema,
+};
+
+export const tool$annotationTasksGetSummary: ToolDefinition<typeof args> = {
+  name: "get-annotation-summary",
+  description:
+    `Get a paginated summary table for a task, with each item's evaluator values and human labels.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "Get task summary",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await annotationTasksGetSummary(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/annotationTasksLinkEvaluator.ts

@@ -0,0 +1,44 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksLinkEvaluator } from "../../funcs/annotationTasksLinkEvaluator.js";
+import { LinkEvaluatorToTaskAnnotationTasksTaskUuidEvaluatorsPostRequest$zodSchema } from "../../models/linkevaluatortotaskannotationtaskstaskuuidevaluatorspostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request:
+    LinkEvaluatorToTaskAnnotationTasksTaskUuidEvaluatorsPostRequest$zodSchema,
+};
+
+export const tool$annotationTasksLinkEvaluator: ToolDefinition<typeof args> = {
+  name: "link-evaluator-to-task",
+  description:
+    `Link an evaluator to an annotation task, appending it to the display order.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Link evaluator to task",
+    "destructiveHint": false,
+    "idempotentHint": true,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await annotationTasksLinkEvaluator(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/annotationTasksList.ts

@@ -0,0 +1,42 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksList } from "../../funcs/annotationTasksList.js";
+import { ListAnnotationTasksAnnotationTasksGetRequest$zodSchema } from "../../models/listannotationtasksannotationtasksgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: ListAnnotationTasksAnnotationTasksGetRequest$zodSchema.optional(),
+};
+
+export const tool$annotationTasksList: ToolDefinition<typeof args> = {
+  name: "list-annotation-tasks",
+  description: `List your annotation tasks with their linked evaluators.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "List annotation tasks",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await annotationTasksList(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/annotationTasksUpdateItems.ts

@@ -0,0 +1,42 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksUpdateItems } from "../../funcs/annotationTasksUpdateItems.js";
+import { BulkUpdateItemsAnnotationTasksTaskUuidItemsPutRequest$zodSchema } from "../../models/bulkupdateitemsannotationtaskstaskuuiditemsputop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: BulkUpdateItemsAnnotationTasksTaskUuidItemsPutRequest$zodSchema,
+};
+
+export const tool$annotationTasksUpdateItems: ToolDefinition<typeof args> = {
+  name: "update-annotation-items",
+  description: `Bulk-update item payloads in an annotation task.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Bulk update items",
+    "destructiveHint": false,
+    "idempotentHint": true,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await annotationTasksUpdateItems(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/evaluatorsCreate.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { evaluatorsCreate } from "../../funcs/evaluatorsCreate.js";
+import { CreateEvaluatorEndpointEvaluatorsPostRequest$zodSchema } from "../../models/createevaluatorendpointevaluatorspostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: CreateEvaluatorEndpointEvaluatorsPostRequest$zodSchema,
+};
+
+export const tool$evaluatorsCreate: ToolDefinition<typeof args> = {
+  name: "create-evaluator",
+  description:
+    `Create an evaluator and its first version, which is set live. An evaluator judges an output against a prompt and rubric.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Create evaluator",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await evaluatorsCreate(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/evaluatorsCreateVersion.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { evaluatorsCreateVersion } from "../../funcs/evaluatorsCreateVersion.js";
+import { CreateVersionEvaluatorsEvaluatorUuidVersionsPostRequest$zodSchema } from "../../models/createversionevaluatorsevaluatoruuidversionspostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: CreateVersionEvaluatorsEvaluatorUuidVersionsPostRequest$zodSchema,
+};
+
+export const tool$evaluatorsCreateVersion: ToolDefinition<typeof args> = {
+  name: "create-evaluator-version",
+  description:
+    `Add a new version to an evaluator you created, optionally making it live. The variable names must match the previous version.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Create evaluator version",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await evaluatorsCreateVersion(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/evaluatorsGet.ts

@@ -0,0 +1,42 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { evaluatorsGet } from "../../funcs/evaluatorsGet.js";
+import { GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetRequest$zodSchema } from "../../models/getevaluatorendpointevaluatorsevaluatoruuidgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetRequest$zodSchema,
+};
+
+export const tool$evaluatorsGet: ToolDefinition<typeof args> = {
+  name: "get-evaluator",
+  description: `Get one evaluator with its full version history.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "Get evaluator",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await evaluatorsGet(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/evaluatorsList.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { evaluatorsList } from "../../funcs/evaluatorsList.js";
+import { ListEvaluatorsEvaluatorsGetRequest$zodSchema } from "../../models/listevaluatorsevaluatorsgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: ListEvaluatorsEvaluatorsGetRequest$zodSchema.optional(),
+};
+
+export const tool$evaluatorsList: ToolDefinition<typeof args> = {
+  name: "list-evaluators",
+  description:
+    `List your evaluators. Optionally filter by what they judge or their modality, and include the built-in defaults.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "List evaluators",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await evaluatorsList(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/models/agentcreate.ts

@@ -38,7 +38,7 @@ export type AgentCreate = {
 
 export const AgentCreate$zodSchema: z.ZodType<AgentCreate> = z.object({
   config: z.record(z.string(), z.any()).nullable().optional().describe(
-    "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`** (built inside Calibrate):\n- `system_prompt` (string): the agent's instructions\n- `llm.model` (string): `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider` (string): `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider` (string): `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first` (bool), `settings.max_assistant_turns` (int)\n- `system_tools.end_call` (bool, optional): let the agent end the call\n- `data_extraction_fields` (array, optional): `[{name, type, description, required}]`\n\n```json\n{\n  \"system_prompt\": \"You are a helpful support agent.\",\n  \"llm\": {\"model\": \"openai/gpt-4.1\"},\n  \"stt\": {\"provider\": \"deepgram\"},\n  \"tts\": {\"provider\": \"elevenlabs\"},\n  \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`** (your own HTTP endpoint):\n- `agent_url` (string, required): public HTTPS endpoint the agent is called at\n- `agent_headers` (object, optional): headers sent on each request, e.g. auth\n- `benchmark_provider` (string, optional): `openrouter` (default), `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n  \"agent_url\": \"https://api.example.com/agent\",\n  \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n  \"benchmark_provider\": \"openrouter\"\n}\n```\n\nFor `type=agent`, omitted keys inherit managed defaults (omit `config` entirely to use all defaults). For `type=connection`, `config` is stored as-is and must contain `agent_url`.",
+    "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n  \"system_prompt\": \"You are a helpful support agent.\",\n  \"llm\": {\"model\": \"openai/gpt-4.1\"},\n  \"stt\": {\"provider\": \"deepgram\"},\n  \"tts\": {\"provider\": \"elevenlabs\"},\n  \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n  \"agent_url\": \"https://api.example.com/agent\",\n  \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n  \"benchmark_provider\": \"openrouter\"\n}\n```\n\nFor `type=agent`, omitted keys inherit managed defaults. Omit `config` entirely to use all defaults. For `type=connection`, `config` is stored as-is and must contain `agent_url`",
   ),
   name: z.string().describe("Agent name, unique within the workspace"),
   type: AgentCreateType$zodSchema.default("agent").describe(

package/src/models/agenttestruncreateresponse.ts

@@ -6,14 +6,14 @@ import * as z from "zod";
 import { ClosedEnum } from "../types/enums.js";
 
 /**
- * Current status of the test run
+ * Current status of the run
  */
 export const AgentTestRunCreateResponseStatus = {
   Queued: "queued",
   InProgress: "in_progress",
 } as const;
 /**
- * Current status of the test run
+ * Current status of the run
  */
 export type AgentTestRunCreateResponseStatus = ClosedEnum<
   typeof AgentTestRunCreateResponseStatus
@@ -22,7 +22,7 @@ export type AgentTestRunCreateResponseStatus = ClosedEnum<
 export const AgentTestRunCreateResponseStatus$zodSchema = z.enum([
   "queued",
   "in_progress",
-]).describe("Current status of the test run");
+]).describe("Current status of the run");
 
 export type AgentTestRunCreateResponse = {
   task_id: string;
@@ -33,7 +33,9 @@ export const AgentTestRunCreateResponse$zodSchema: z.ZodType<
   AgentTestRunCreateResponse
 > = z.object({
   status: AgentTestRunCreateResponseStatus$zodSchema.describe(
-    "Current status of the test run",
+    "Current status of the run",
+  ),
+  task_id: z.string().describe(
+    "Test run job ID. Poll it for status and results",
   ),
-  task_id: z.string().describe("Test run job ID. Poll for status and results"),
 });