@dalmia/calibrate-mcp 0.0.14 → 0.0.15

package/src/mcp-server/tools/agentTestsListRunsForAgent.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { agentTestsListRunsForAgent } from "../../funcs/agentTestsListRunsForAgent.js";
+import { GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetRequest$zodSchema } from "../../models/getagenttestrunsagenttestsagentagentuuidrunsgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetRequest$zodSchema,
+};
+
+export const tool$agentTestsListRunsForAgent: ToolDefinition<typeof args> = {
+  name: "list-agent-test-runs",
+  description:
+    `List an agent's test and benchmark runs, most recently created first, with their status and results.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "List test runs for an agent",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await agentTestsListRunsForAgent(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/agentsVerifyConnection.ts

@@ -0,0 +1,44 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { agentsVerifyConnection } from "../../funcs/agentsVerifyConnection.js";
+import { VerifyAgentConnectionAgentsAgentUuidVerifyConnectionPostRequest$zodSchema } from "../../models/verifyagentconnectionagentsagentuuidverifyconnectionpostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request:
+    VerifyAgentConnectionAgentsAgentUuidVerifyConnectionPostRequest$zodSchema,
+};
+
+export const tool$agentsVerifyConnection: ToolDefinition<typeof args> = {
+  name: "verify-agent-connection",
+  description:
+    `Verify that a connection-type agent responds. Optionally pass a model to verify it before benchmarking. The endpoint and headers come from the agent's stored config; a successful check is recorded on the agent.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Verify agent connection",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await agentsVerifyConnection(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/annotationTasksAddItems.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksAddItems } from "../../funcs/annotationTasksAddItems.js";
+import { BulkCreateItemsAnnotationTasksTaskUuidItemsPostRequest$zodSchema } from "../../models/bulkcreateitemsannotationtaskstaskuuiditemspostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: BulkCreateItemsAnnotationTasksTaskUuidItemsPostRequest$zodSchema,
+};
+
+export const tool$annotationTasksAddItems: ToolDefinition<typeof args> = {
+  name: "add-annotation-items",
+  description:
+    `Bulk-create items in an annotation task, optionally seeding human annotations.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Bulk create items",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await annotationTasksAddItems(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/annotationTasksCreate.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksCreate } from "../../funcs/annotationTasksCreate.js";
+import { CreateAnnotationTaskEndpointAnnotationTasksPostRequest$zodSchema } from "../../models/createannotationtaskendpointannotationtaskspostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: CreateAnnotationTaskEndpointAnnotationTasksPostRequest$zodSchema,
+};
+
+export const tool$annotationTasksCreate: ToolDefinition<typeof args> = {
+  name: "create-annotation-task",
+  description:
+    `Create an annotation task for labelling items against evaluators. Optionally link evaluators in order.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Create annotation task",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await annotationTasksCreate(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/annotationTasksCreateEvaluatorRun.ts

@@ -0,0 +1,46 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksCreateEvaluatorRun } from "../../funcs/annotationTasksCreateEvaluatorRun.js";
+import { StartEvaluatorRunAnnotationTasksTaskUuidEvaluatorRunsPostRequest$zodSchema } from "../../models/startevaluatorrunannotationtaskstaskuuidevaluatorrunspostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request:
+    StartEvaluatorRunAnnotationTasksTaskUuidEvaluatorRunsPostRequest$zodSchema,
+};
+
+export const tool$annotationTasksCreateEvaluatorRun: ToolDefinition<
+  typeof args
+> = {
+  name: "run-annotation-evaluators",
+  description:
+    `Run one or more of a task's evaluators over its items as a background job. Returns a job_uuid to poll with get-annotation-evaluator-run.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Run evaluators on items",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await annotationTasksCreateEvaluatorRun(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/annotationTasksGet.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksGet } from "../../funcs/annotationTasksGet.js";
+import { GetAnnotationTaskEndpointAnnotationTasksTaskUuidGetRequest$zodSchema } from "../../models/getannotationtaskendpointannotationtaskstaskuuidgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: GetAnnotationTaskEndpointAnnotationTasksTaskUuidGetRequest$zodSchema,
+};
+
+export const tool$annotationTasksGet: ToolDefinition<typeof args> = {
+  name: "get-annotation-task",
+  description:
+    `Get one annotation task with its evaluators, items, and labelling jobs.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "Get annotation task",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await annotationTasksGet(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/annotationTasksGetAgreement.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksGetAgreement } from "../../funcs/annotationTasksGetAgreement.js";
+import { TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest$zodSchema } from "../../models/taskagreementannotationtaskstaskuuidagreementgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest$zodSchema,
+};
+
+export const tool$annotationTasksGetAgreement: ToolDefinition<typeof args> = {
+  name: "get-annotation-agreement",
+  description:
+    `Get human-vs-human and human-vs-evaluator agreement metrics for a task, with a trend series.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "Get task agreement",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await annotationTasksGetAgreement(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/annotationTasksGetEvaluatorRun.ts

@@ -0,0 +1,47 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksGetEvaluatorRun } from "../../funcs/annotationTasksGetEvaluatorRun.js";
+import {
+  GetEvaluatorRunJobAnnotationTasksTaskUuidEvaluatorRunsJobUuidGetRequest$zodSchema,
+} from "../../models/getevaluatorrunjobannotationtaskstaskuuidevaluatorrunsjobuuidgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request:
+    GetEvaluatorRunJobAnnotationTasksTaskUuidEvaluatorRunsJobUuidGetRequest$zodSchema,
+};
+
+export const tool$annotationTasksGetEvaluatorRun: ToolDefinition<typeof args> =
+  {
+    name: "get-annotation-evaluator-run",
+    description:
+      `Get one evaluator-run job with its results and human-agreement summary.
+`,
+    scopes: ["read"],
+    annotations: {
+      "title": "Get evaluator run",
+      "destructiveHint": false,
+      "idempotentHint": false,
+      "openWorldHint": true,
+      "readOnlyHint": true,
+    },
+    args,
+    tool: async (client, args, ctx) => {
+      const [result] = await annotationTasksGetEvaluatorRun(
+        client,
+        args.request,
+        { fetchOptions: { signal: ctx.signal } },
+      ).$inspect();
+
+      if (!result.ok) {
+        return {
+          content: [{ type: "text", text: result.error.message }],
+          isError: true,
+        };
+      }
+
+      return formatResult(result.value);
+    },
+  };

package/src/mcp-server/tools/annotationTasksGetSummary.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksGetSummary } from "../../funcs/annotationTasksGetSummary.js";
+import { TaskSummaryAnnotationTasksTaskUuidSummaryGetRequest$zodSchema } from "../../models/tasksummaryannotationtaskstaskuuidsummarygetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: TaskSummaryAnnotationTasksTaskUuidSummaryGetRequest$zodSchema,
+};
+
+export const tool$annotationTasksGetSummary: ToolDefinition<typeof args> = {
+  name: "get-annotation-summary",
+  description:
+    `Get a paginated summary table for a task, with each item's evaluator values and human labels.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "Get task summary",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await annotationTasksGetSummary(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/annotationTasksLinkEvaluator.ts

@@ -0,0 +1,44 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksLinkEvaluator } from "../../funcs/annotationTasksLinkEvaluator.js";
+import { LinkEvaluatorToTaskAnnotationTasksTaskUuidEvaluatorsPostRequest$zodSchema } from "../../models/linkevaluatortotaskannotationtaskstaskuuidevaluatorspostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request:
+    LinkEvaluatorToTaskAnnotationTasksTaskUuidEvaluatorsPostRequest$zodSchema,
+};
+
+export const tool$annotationTasksLinkEvaluator: ToolDefinition<typeof args> = {
+  name: "link-evaluator-to-task",
+  description:
+    `Link an evaluator to an annotation task, appending it to the display order.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Link evaluator to task",
+    "destructiveHint": false,
+    "idempotentHint": true,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await annotationTasksLinkEvaluator(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/annotationTasksList.ts

@@ -0,0 +1,42 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksList } from "../../funcs/annotationTasksList.js";
+import { ListAnnotationTasksAnnotationTasksGetRequest$zodSchema } from "../../models/listannotationtasksannotationtasksgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: ListAnnotationTasksAnnotationTasksGetRequest$zodSchema.optional(),
+};
+
+export const tool$annotationTasksList: ToolDefinition<typeof args> = {
+  name: "list-annotation-tasks",
+  description: `List your annotation tasks with their linked evaluators.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "List annotation tasks",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await annotationTasksList(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/annotationTasksUpdateItems.ts

@@ -0,0 +1,42 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { annotationTasksUpdateItems } from "../../funcs/annotationTasksUpdateItems.js";
+import { BulkUpdateItemsAnnotationTasksTaskUuidItemsPutRequest$zodSchema } from "../../models/bulkupdateitemsannotationtaskstaskuuiditemsputop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: BulkUpdateItemsAnnotationTasksTaskUuidItemsPutRequest$zodSchema,
+};
+
+export const tool$annotationTasksUpdateItems: ToolDefinition<typeof args> = {
+  name: "update-annotation-items",
+  description: `Bulk-update item payloads in an annotation task.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Bulk update items",
+    "destructiveHint": false,
+    "idempotentHint": true,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await annotationTasksUpdateItems(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/evaluatorsCreate.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { evaluatorsCreate } from "../../funcs/evaluatorsCreate.js";
+import { CreateEvaluatorEndpointEvaluatorsPostRequest$zodSchema } from "../../models/createevaluatorendpointevaluatorspostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: CreateEvaluatorEndpointEvaluatorsPostRequest$zodSchema,
+};
+
+export const tool$evaluatorsCreate: ToolDefinition<typeof args> = {
+  name: "create-evaluator",
+  description:
+    `Create an evaluator and its first version, which is set live. An evaluator judges an output against a prompt and rubric.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Create evaluator",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await evaluatorsCreate(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/evaluatorsCreateVersion.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { evaluatorsCreateVersion } from "../../funcs/evaluatorsCreateVersion.js";
+import { CreateVersionEvaluatorsEvaluatorUuidVersionsPostRequest$zodSchema } from "../../models/createversionevaluatorsevaluatoruuidversionspostop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: CreateVersionEvaluatorsEvaluatorUuidVersionsPostRequest$zodSchema,
+};
+
+export const tool$evaluatorsCreateVersion: ToolDefinition<typeof args> = {
+  name: "create-evaluator-version",
+  description:
+    `Add a new version to an evaluator you created, optionally making it live. The variable names must match the previous version.
+`,
+  scopes: ["write"],
+  annotations: {
+    "title": "Create evaluator version",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await evaluatorsCreateVersion(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/evaluatorsGet.ts

@@ -0,0 +1,42 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { evaluatorsGet } from "../../funcs/evaluatorsGet.js";
+import { GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetRequest$zodSchema } from "../../models/getevaluatorendpointevaluatorsevaluatoruuidgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetRequest$zodSchema,
+};
+
+export const tool$evaluatorsGet: ToolDefinition<typeof args> = {
+  name: "get-evaluator",
+  description: `Get one evaluator with its full version history.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "Get evaluator",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await evaluatorsGet(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};