@dalmia/calibrate-mcp 0.0.14 → 0.0.15

package/src/models/testoutput.ts

@@ -12,9 +12,9 @@ export type TestOutput = {
 
 export const TestOutput$zodSchema: z.ZodType<TestOutput> = z.object({
   response: z.string().nullable().optional().describe(
-    "The agent's generated reply. Null for tool-call-only cases",
+    "The reply the agent generated",
   ),
   tool_calls: z.array(ToolCallOutput$zodSchema).nullable().optional().describe(
-    "Tool calls the agent generated. Null when it made none",
+    "Tool calls the agent generated",
   ),
 });

package/src/models/testrunevaluator.ts

@@ -5,13 +5,15 @@
 import * as z from "zod";
 import { ClosedEnum } from "../types/enums.js";
 
-export const OutputType = {
+export const TestRunEvaluatorOutputType = {
   Binary: "binary",
   Rating: "rating",
 } as const;
-export type OutputType = ClosedEnum<typeof OutputType>;
+export type TestRunEvaluatorOutputType = ClosedEnum<
+  typeof TestRunEvaluatorOutputType
+>;
 
-export const OutputType$zodSchema = z.enum([
+export const TestRunEvaluatorOutputType$zodSchema = z.enum([
   "binary",
   "rating",
 ]);
@@ -20,7 +22,7 @@ export type TestRunEvaluator = {
   uuid?: string | null | undefined;
   name?: string | null | undefined;
   description?: string | null | undefined;
-  output_type?: OutputType | null | undefined;
+  output_type?: TestRunEvaluatorOutputType | null | undefined;
   output_config?: { [k: string]: any } | null | undefined;
   scale_min?: number | null | undefined;
   scale_max?: number | null | undefined;
@@ -34,20 +36,21 @@ export const TestRunEvaluator$zodSchema: z.ZodType<TestRunEvaluator> = z.object(
     ),
     name: z.string().nullable().optional().describe("Name of the evaluator"),
     output_config: z.record(z.string(), z.any()).nullable().optional().describe(
-      "Rubric: the scale values, labels, and colors a verdict maps to",
-    ),
-    output_type: OutputType$zodSchema.nullable().optional().describe(
-      "Verdict shape: pass/fail or a numeric rating",
+      "The rubric: the scale values, labels, and colors a verdict maps to",
     ),
+    output_type: TestRunEvaluatorOutputType$zodSchema.nullable().optional()
+      .describe(
+        "The shape of the verdict:\n- `binary`: a pass/fail verdict\n- `rating`: a numeric score",
+      ),
     scale_max: z.number().nullable().optional().describe(
-      "Highest rating value. Set for rating evaluators",
+      "Highest value on a rating scale",
     ),
     scale_min: z.number().nullable().optional().describe(
-      "Lowest rating value. Set for rating evaluators",
+      "Lowest value on a rating scale",
     ),
     uuid: z.string().nullable().optional().describe("ID of the evaluator"),
     version_number: z.int().nullable().optional().describe(
-      "Evaluator version this run used",
+      "The evaluator version this run used",
     ),
   },
 );

package/src/models/testrunstatusresponse.ts

@@ -29,7 +29,7 @@ export type TestRunStatusResponse = {
 export const TestRunStatusResponse$zodSchema: z.ZodType<TestRunStatusResponse> =
   z.object({
     cost: z.record(z.string(), z.any()).nullable().optional().describe(
-      "Aggregated cost `{mean, min, max, count}` (USD)",
+      "Aggregated cost as `{mean, min, max, count}` (USD)",
     ),
     error: z.boolean().default(false).describe("True if the run failed"),
     evaluators: z.array(TestRunEvaluator$zodSchema).nullable().optional()
@@ -37,29 +37,29 @@ export const TestRunStatusResponse$zodSchema: z.ZodType<TestRunStatusResponse> =
         "The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`",
       ),
     failed: z.int().nullable().optional().describe(
-      "Number of test cases that failed. Null until done",
+      "Number of test cases that failed",
     ),
     is_public: z.boolean().default(false).describe(
       "Whether the run is shared publicly",
     ),
     latency_ms: z.record(z.string(), z.any()).nullable().optional().describe(
-      "Aggregated response latency in milliseconds: `{p50, p95, p99, count}`",
+      "Aggregated response latency in milliseconds, as `{p50, p95, p99, count}`",
     ),
     passed: z.int().nullable().optional().describe(
-      "Number of test cases that passed. Null until done",
+      "Number of test cases that passed",
     ),
     results: z.array(TestCaseResult$zodSchema).nullable().optional().describe(
-      "Results for each test case. Null until available",
+      "Results for each test case",
     ),
     share_token: z.string().nullable().optional().describe(
-      "Public share token. Null unless the run is public",
+      "Token for building the public share URL",
     ),
     status: TaskStatus$zodSchema,
     task_id: z.string().describe("Test run job ID"),
     total_tests: z.int().nullable().optional().describe(
-      "Total number of test cases. Null until known",
+      "Total number of test cases",
     ),
     total_tokens: z.record(z.string(), z.any()).nullable().optional().describe(
-      "Aggregated token usage `{mean, min, max, count}`",
+      "Aggregated token usage as `{mean, min, max, count}`",
     ),
   });

package/src/models/testupdate.ts

@@ -31,7 +31,7 @@ export type TestUpdate = {
 
 export const TestUpdate$zodSchema: z.ZodType<TestUpdate> = z.object({
   config: z.record(z.string(), z.any()).nullable().optional().describe(
-    "The calibrate test config. Three top-level keys.\n\n- `history` (array, required): the conversation up to the agent's turn. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `evaluation` (object, required): `{type, ...}`, where `type` matches the test's `type` (below).\n- `settings` (object, optional): e.g. `{\"language\": \"en\"}`.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nReplaces the stored config. Omit to leave unchanged.",
+    "The calibrate test config. Three top-level keys.\n\n- `history`: the required conversation up to the agent's turn. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `evaluation`: the required `{type, ...}`, where `type` matches the test's `type` below.\n- `settings`: an optional object, e.g. `{\"language\": \"en\"}`.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nReplaces the stored config. Omit to leave unchanged",
   ),
   evaluators: z.array(RoutersTestsEvaluatorRef$zodSchema).nullable().optional()
     .describe(
@@ -41,6 +41,6 @@ export const TestUpdate$zodSchema: z.ZodType<TestUpdate> = z.object({
     "New test name. Omit to leave unchanged",
   ),
   type: TestUpdateType$zodSchema.nullable().optional().describe(
-    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\nImmutable. Omit, or send the existing value. A different value is rejected (400).",
+    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\n\nImmutable. Omit it, or send the current value",
   ),
 });

package/src/models/toolcalloutput.ts

@@ -12,10 +12,10 @@ export type ToolCallOutput = {
 
 export const ToolCallOutput$zodSchema: z.ZodType<ToolCallOutput> = z.object({
   arguments: z.record(z.string(), z.any()).nullable().optional().describe(
-    "Arguments the agent passed to the tool. Null if none",
+    "Arguments the agent passed to the tool",
   ),
   output: z.any().nullable().optional().describe(
-    "Tool execution result (any JSON value). Present only when the agent runs the tool and returns its result",
+    "Tool execution result, when the agent ran the tool and returned its result",
   ),
   tool: z.string().describe("Name of the tool the agent called"),
 });

package/src/models/updateagentendpointagentsagentuuidputop.ts

@@ -16,17 +16,15 @@ import {
 export type UpdateAgentEndpointAgentsAgentUuidPutRequest = {
   agent_uuid: string;
   xAPIKey?: string | null | undefined;
-  xOrgUUID?: string | null | undefined;
   body: AgentUpdate;
 };
 
 export const UpdateAgentEndpointAgentsAgentUuidPutRequest$zodSchema: z.ZodType<
   UpdateAgentEndpointAgentsAgentUuidPutRequest
 > = z.object({
-  agent_uuid: z.string().describe("The agent to update."),
+  agent_uuid: z.string().describe("The agent to update"),
   body: AgentUpdate$zodSchema,
   xAPIKey: z.string().nullable().optional(),
-  xOrgUUID: z.string().nullable().optional(),
 });
 
 export type UpdateAgentEndpointAgentsAgentUuidPutResponse =

package/src/models/updatetestendpointteststestuuidputop.ts

@@ -16,7 +16,6 @@ import { TestUpdate, TestUpdate$zodSchema } from "./testupdate.js";
 export type UpdateTestEndpointTestsTestUuidPutRequest = {
   test_uuid: string;
   xAPIKey?: string | null | undefined;
-  xOrgUUID?: string | null | undefined;
   body: TestUpdate;
 };
 
@@ -26,7 +25,6 @@ export const UpdateTestEndpointTestsTestUuidPutRequest$zodSchema: z.ZodType<
   body: TestUpdate$zodSchema,
   test_uuid: z.string().describe("Test to update"),
   xAPIKey: z.string().nullable().optional(),
-  xOrgUUID: z.string().nullable().optional(),
 });
 
 export type UpdateTestEndpointTestsTestUuidPutResponse =

package/src/models/variablespec.ts

@@ -0,0 +1,23 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type VariableSpec = {
+  name: string;
+  description?: string | null | undefined;
+  default?: string | null | undefined;
+};
+
+export const VariableSpec$zodSchema: z.ZodType<VariableSpec> = z.object({
+  default: z.string().nullable().optional().describe(
+    "Default value used when you omit this variable. Omit for no default",
+  ),
+  description: z.string().nullable().optional().describe(
+    "What the variable is for. Omit if self-evident",
+  ),
+  name: z.string().describe(
+    "Name of a `{{\"{{\"}}placeholder}}` used in the system prompt",
+  ),
+});

package/src/models/verifyagentconnectionagentsagentuuidverifyconnectionpostop.ts

@@ -0,0 +1,42 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  AgentVerifyRequest,
+  AgentVerifyRequest$zodSchema,
+} from "./agentverifyrequest.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import {
+  VerifyConnectionResponse,
+  VerifyConnectionResponse$zodSchema,
+} from "./verifyconnectionresponse.js";
+
+export type VerifyAgentConnectionAgentsAgentUuidVerifyConnectionPostRequest = {
+  agent_uuid: string;
+  xAPIKey?: string | null | undefined;
+  body: AgentVerifyRequest;
+};
+
+export const VerifyAgentConnectionAgentsAgentUuidVerifyConnectionPostRequest$zodSchema:
+  z.ZodType<VerifyAgentConnectionAgentsAgentUuidVerifyConnectionPostRequest> = z
+    .object({
+      agent_uuid: z.string().describe("The agent whose connection to verify"),
+      body: AgentVerifyRequest$zodSchema,
+      xAPIKey: z.string().nullable().optional(),
+    });
+
+export type VerifyAgentConnectionAgentsAgentUuidVerifyConnectionPostResponse =
+  | VerifyConnectionResponse
+  | HTTPValidationError;
+
+export const VerifyAgentConnectionAgentsAgentUuidVerifyConnectionPostResponse$zodSchema:
+  z.ZodType<VerifyAgentConnectionAgentsAgentUuidVerifyConnectionPostResponse> =
+    z.union([
+      VerifyConnectionResponse$zodSchema,
+      HTTPValidationError$zodSchema,
+    ]);

package/src/models/verifyconnectionresponse.ts

@@ -0,0 +1,25 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type VerifyConnectionResponse = {
+  success: boolean;
+  error?: string | null | undefined;
+  sample_response?: { [k: string]: any } | null | undefined;
+};
+
+export const VerifyConnectionResponse$zodSchema: z.ZodType<
+  VerifyConnectionResponse
+> = z.object({
+  error: z.string().nullable().optional().describe(
+    "Reason the verification failed",
+  ),
+  sample_response: z.record(z.string(), z.any()).nullable().optional().describe(
+    "Sample output returned by the agent during verification",
+  ),
+  success: z.boolean().describe(
+    "True if the agent responded successfully to the verification probe",
+  ),
+});

package/src/models/versioncreateresponse.ts

@@ -0,0 +1,18 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type VersionCreateResponse = {
+  version_uuid: string;
+  version_number: number;
+};
+
+export const VersionCreateResponse$zodSchema: z.ZodType<VersionCreateResponse> =
+  z.object({
+    version_number: z.int().describe(
+      "The version's number. The first version is 1, and it goes up by one for each new version of the evaluator",
+    ),
+    version_uuid: z.string().describe("ID of the newly created version"),
+  });

package/src/tool-names.ts

@@ -1,5 +1,9 @@
 // Auto-generated at build time
 export const toolNames: Array<{ name: string; description: string }>= [
+  {
+    "name": "verify-agent-connection",
+    "description": "Verify that a connection-type agent responds. Optionally pass a model to verify it before benchmarking. The endpoint and headers come from the agent's stored config; a successful check is recorded on the agent.\n"
+  },
   {
     "name": "resolve-agent-names",
     "description": "Map human-friendly agent names to UUIDs within the caller's org. Each\nname resolves to at most one agent; unknown names are returned under\nnot_found. Use this before run-agent-tests when you have names instead\nof UUIDs. Does not create or modify agents.\n"
@@ -40,6 +44,18 @@ export const toolNames: Array<{ name: string; description: string }>= [
     "name": "update-test",
     "description": "Update an existing test by UUID within the caller's org. Supply the\nfields to change (name, config). Returns the updated test. Fails with\n404 if the test does not exist or belongs to another org.\n"
   },
+  {
+    "name": "link-tests-to-agent",
+    "description": "Link one or more existing tests to an agent so they run together. Tests already linked are skipped. The agent and every test must be in your workspace.\n"
+  },
+  {
+    "name": "list-agent-tests",
+    "description": "List the tests linked to an agent.\n"
+  },
+  {
+    "name": "list-agent-test-runs",
+    "description": "List an agent's test and benchmark runs, most recently created first, with their status and results.\n"
+  },
   {
     "name": "run-agent-tests",
     "description": "Start a background LLM unit-test job for one agent. Runs all tests\nlinked to the agent unless test_uuids is provided. Returns a task_id\nto poll with get-agent-test-run. The agent must belong to the caller's\norg and have a verified connection; otherwise the call fails with 400\nor 404.\n"
@@ -51,5 +67,69 @@ export const toolNames: Array<{ name: string; description: string }>= [
   {
     "name": "get-agent-test-run",
     "description": "Poll the status and results of an agent test run by task_id. Returns\nqueued, in_progress, completed, failed, or aborted status. When\ncompleted, includes per-test-case pass/fail results and evaluator\njudgments. Use the task_id from run-agent-tests or\nrun-agent-tests-batch.\n"
+  },
+  {
+    "name": "run-agent-benchmark",
+    "description": "Start a multi-model benchmark over an agent's linked tests as a background job. Optionally restrict to a subset of linked tests. Returns a task_id to poll with get-agent-benchmark.\n"
+  },
+  {
+    "name": "get-agent-benchmark",
+    "description": "Get the results of a benchmark run, including per-model scores and the leaderboard.\n"
+  },
+  {
+    "name": "create-evaluator",
+    "description": "Create an evaluator and its first version, which is set live. An evaluator judges an output against a prompt and rubric.\n"
+  },
+  {
+    "name": "list-evaluators",
+    "description": "List your evaluators. Optionally filter by what they judge or their modality, and include the built-in defaults.\n"
+  },
+  {
+    "name": "get-evaluator",
+    "description": "Get one evaluator with its full version history.\n"
+  },
+  {
+    "name": "create-evaluator-version",
+    "description": "Add a new version to an evaluator you created, optionally making it live. The variable names must match the previous version.\n"
+  },
+  {
+    "name": "create-annotation-task",
+    "description": "Create an annotation task for labelling items against evaluators. Optionally link evaluators in order.\n"
+  },
+  {
+    "name": "list-annotation-tasks",
+    "description": "List your annotation tasks with their linked evaluators.\n"
+  },
+  {
+    "name": "get-annotation-task",
+    "description": "Get one annotation task with its evaluators, items, and labelling jobs.\n"
+  },
+  {
+    "name": "link-evaluator-to-task",
+    "description": "Link an evaluator to an annotation task, appending it to the display order.\n"
+  },
+  {
+    "name": "add-annotation-items",
+    "description": "Bulk-create items in an annotation task, optionally seeding human annotations.\n"
+  },
+  {
+    "name": "update-annotation-items",
+    "description": "Bulk-update item payloads in an annotation task.\n"
+  },
+  {
+    "name": "run-annotation-evaluators",
+    "description": "Run one or more of a task's evaluators over its items as a background job. Returns a job_uuid to poll with get-annotation-evaluator-run.\n"
+  },
+  {
+    "name": "get-annotation-evaluator-run",
+    "description": "Get one evaluator-run job with its results and human-agreement summary.\n"
+  },
+  {
+    "name": "get-annotation-agreement",
+    "description": "Get human-vs-human and human-vs-evaluator agreement metrics for a task, with a trend series.\n"
+  },
+  {
+    "name": "get-annotation-summary",
+    "description": "Get a paginated summary table for a task, with each item's evaluator values and human labels.\n"
   }
 ];