@dalmia/calibrate-mcp 0.0.13

package/src/models/getagenttestrunstatusagenttestsruntaskidgetop.ts

@@ -0,0 +1,36 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import {
+  TestRunStatusResponse,
+  TestRunStatusResponse$zodSchema,
+} from "./testrunstatusresponse.js";
+
+export type GetAgentTestRunStatusAgentTestsRunTaskIdGetRequest = {
+  task_id: string;
+  xAPIKey?: string | null | undefined;
+  xOrgUUID?: string | null | undefined;
+};
+
+export const GetAgentTestRunStatusAgentTestsRunTaskIdGetRequest$zodSchema:
+  z.ZodType<GetAgentTestRunStatusAgentTestsRunTaskIdGetRequest> = z.object({
+    task_id: z.string().describe("Test run to poll for status and results"),
+    xAPIKey: z.string().nullable().optional(),
+    xOrgUUID: z.string().nullable().optional(),
+  });
+
+export type GetAgentTestRunStatusAgentTestsRunTaskIdGetResponse =
+  | TestRunStatusResponse
+  | HTTPValidationError;
+
+export const GetAgentTestRunStatusAgentTestsRunTaskIdGetResponse$zodSchema:
+  z.ZodType<GetAgentTestRunStatusAgentTestsRunTaskIdGetResponse> = z.union([
+    TestRunStatusResponse$zodSchema,
+    HTTPValidationError$zodSchema,
+  ]);

package/src/models/gettestendpointteststestuuidgetop.ts

@@ -0,0 +1,38 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import {
+  RoutersTestsTestResponse,
+  RoutersTestsTestResponse$zodSchema,
+} from "./routersteststestresponse.js";
+
+export type GetTestEndpointTestsTestUuidGetRequest = {
+  test_uuid: string;
+  xAPIKey?: string | null | undefined;
+  xOrgUUID?: string | null | undefined;
+};
+
+export const GetTestEndpointTestsTestUuidGetRequest$zodSchema: z.ZodType<
+  GetTestEndpointTestsTestUuidGetRequest
+> = z.object({
+  test_uuid: z.string().describe("Test to retrieve"),
+  xAPIKey: z.string().nullable().optional(),
+  xOrgUUID: z.string().nullable().optional(),
+});
+
+export type GetTestEndpointTestsTestUuidGetResponse =
+  | RoutersTestsTestResponse
+  | HTTPValidationError;
+
+export const GetTestEndpointTestsTestUuidGetResponse$zodSchema: z.ZodType<
+  GetTestEndpointTestsTestUuidGetResponse
+> = z.union([
+  RoutersTestsTestResponse$zodSchema,
+  HTTPValidationError$zodSchema,
+]);

package/src/models/httpvalidationerror.ts

@@ -0,0 +1,18 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  ValidationError,
+  ValidationError$zodSchema,
+} from "./validationerror.js";
+
+export type HTTPValidationError = {
+  detail?: Array<ValidationError> | undefined;
+};
+
+export const HTTPValidationError$zodSchema: z.ZodType<HTTPValidationError> = z
+  .object({
+    detail: z.array(ValidationError$zodSchema).optional(),
+  });

package/src/models/judgeresult.ts

@@ -0,0 +1,35 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type JudgeResult = {
+  evaluator_uuid?: string | null | undefined;
+  reasoning?: string | null | undefined;
+  match?: boolean | null | undefined;
+  score?: number | null | undefined;
+  value_name?: string | null | undefined;
+  variable_values?: { [k: string]: any } | null | undefined;
+};
+
+export const JudgeResult$zodSchema: z.ZodType<JudgeResult> = z.object({
+  evaluator_uuid: z.string().nullable().optional().describe(
+    "ID of the evaluator that produced this verdict",
+  ),
+  match: z.boolean().nullable().optional().describe(
+    "Pass/fail verdict, for binary evaluators",
+  ),
+  reasoning: z.string().nullable().optional().describe(
+    "Judge's rationale for this verdict",
+  ),
+  score: z.number().nullable().optional().describe(
+    "Numeric score, for rating evaluators",
+  ),
+  value_name: z.string().nullable().optional().describe(
+    "Readable label for the verdict, from the run's rubric",
+  ),
+  variable_values: z.record(z.string(), z.any()).nullable().optional().describe(
+    "`{{\"{{\"}}var}}` substitutions used for this evaluator on this test case",
+  ),
+});

package/src/models/listagentsagentsgetop.ts

@@ -0,0 +1,36 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import {
+  RoutersAgentsAgentResponse,
+  RoutersAgentsAgentResponse$zodSchema,
+} from "./routersagentsagentresponse.js";
+
+export type ListAgentsAgentsGetRequest = {
+  xAPIKey?: string | null | undefined;
+  xOrgUUID?: string | null | undefined;
+};
+
+export const ListAgentsAgentsGetRequest$zodSchema: z.ZodType<
+  ListAgentsAgentsGetRequest
+> = z.object({
+  xAPIKey: z.string().nullable().optional(),
+  xOrgUUID: z.string().nullable().optional(),
+});
+
+export type ListAgentsAgentsGetResponse =
+  | Array<RoutersAgentsAgentResponse>
+  | HTTPValidationError;
+
+export const ListAgentsAgentsGetResponse$zodSchema: z.ZodType<
+  ListAgentsAgentsGetResponse
+> = z.union([
+  z.array(RoutersAgentsAgentResponse$zodSchema).describe("Successful Response"),
+  HTTPValidationError$zodSchema,
+]);

package/src/models/listteststestsgetop.ts

@@ -0,0 +1,36 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import {
+  RoutersTestsTestResponse,
+  RoutersTestsTestResponse$zodSchema,
+} from "./routersteststestresponse.js";
+
+export type ListTestsTestsGetRequest = {
+  xAPIKey?: string | null | undefined;
+  xOrgUUID?: string | null | undefined;
+};
+
+export const ListTestsTestsGetRequest$zodSchema: z.ZodType<
+  ListTestsTestsGetRequest
+> = z.object({
+  xAPIKey: z.string().nullable().optional(),
+  xOrgUUID: z.string().nullable().optional(),
+});
+
+export type ListTestsTestsGetResponse =
+  | Array<RoutersTestsTestResponse>
+  | HTTPValidationError;
+
+export const ListTestsTestsGetResponse$zodSchema: z.ZodType<
+  ListTestsTestsGetResponse
+> = z.union([
+  z.array(RoutersTestsTestResponse$zodSchema).describe("Successful Response"),
+  HTTPValidationError$zodSchema,
+]);

package/src/models/resolveagentnamesagentsresolvepostop.ts

@@ -0,0 +1,42 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import {
+  ResolveAgentNamesRequest,
+  ResolveAgentNamesRequest$zodSchema,
+} from "./resolveagentnamesrequest.js";
+import {
+  ResolveAgentNamesResponse,
+  ResolveAgentNamesResponse$zodSchema,
+} from "./resolveagentnamesresponse.js";
+
+export type ResolveAgentNamesAgentsResolvePostRequest = {
+  xAPIKey?: string | null | undefined;
+  xOrgUUID?: string | null | undefined;
+  body: ResolveAgentNamesRequest;
+};
+
+export const ResolveAgentNamesAgentsResolvePostRequest$zodSchema: z.ZodType<
+  ResolveAgentNamesAgentsResolvePostRequest
+> = z.object({
+  body: ResolveAgentNamesRequest$zodSchema,
+  xAPIKey: z.string().nullable().optional(),
+  xOrgUUID: z.string().nullable().optional(),
+});
+
+export type ResolveAgentNamesAgentsResolvePostResponse =
+  | ResolveAgentNamesResponse
+  | HTTPValidationError;
+
+export const ResolveAgentNamesAgentsResolvePostResponse$zodSchema: z.ZodType<
+  ResolveAgentNamesAgentsResolvePostResponse
+> = z.union([
+  ResolveAgentNamesResponse$zodSchema,
+  HTTPValidationError$zodSchema,
+]);

package/src/models/resolveagentnamesrequest.ts

@@ -0,0 +1,13 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type ResolveAgentNamesRequest = { names: Array<string> };
+
+export const ResolveAgentNamesRequest$zodSchema: z.ZodType<
+  ResolveAgentNamesRequest
+> = z.object({
+  names: z.array(z.string()).describe("Agent names to resolve to IDs"),
+});

package/src/models/resolveagentnamesresponse.ts

@@ -0,0 +1,21 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type ResolveAgentNamesResponse = {
+  resolved: { [k: string]: string };
+  not_found: Array<string>;
+};
+
+export const ResolveAgentNamesResponse$zodSchema: z.ZodType<
+  ResolveAgentNamesResponse
+> = z.object({
+  not_found: z.array(z.string()).describe(
+    "Names with no matching agent in your workspace",
+  ),
+  resolved: z.record(z.string(), z.string()).describe(
+    "Map of name to agent ID for each name that matched",
+  ),
+});

package/src/models/routersagentsagentresponse.ts

@@ -0,0 +1,59 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+/**
+ * - `agent`: built inside Calibrate
+ *
+ * @remarks
+ * - `connection`: your existing agent connected to Calibrate
+ */
+export const RoutersAgentsAgentResponseType = {
+  Agent: "agent",
+  Connection: "connection",
+} as const;
+/**
+ * - `agent`: built inside Calibrate
+ *
+ * @remarks
+ * - `connection`: your existing agent connected to Calibrate
+ */
+export type RoutersAgentsAgentResponseType = ClosedEnum<
+  typeof RoutersAgentsAgentResponseType
+>;
+
+export const RoutersAgentsAgentResponseType$zodSchema = z.enum([
+  "agent",
+  "connection",
+]).describe(
+  "- `agent`: built inside Calibrate\n- `connection`: your existing agent connected to Calibrate",
+);
+
+export type RoutersAgentsAgentResponse = {
+  uuid: string;
+  name: string;
+  type: RoutersAgentsAgentResponseType;
+  config?: { [k: string]: any } | null | undefined;
+  created_at: string;
+  updated_at: string;
+};
+
+export const RoutersAgentsAgentResponse$zodSchema: z.ZodType<
+  RoutersAgentsAgentResponse
+> = z.object({
+  config: z.record(z.string(), z.any()).nullable().optional().describe(
+    "Agent configuration",
+  ),
+  created_at: z.string().describe("When the agent was created (ISO 8601 UTC)"),
+  name: z.string().describe("Name of the agent"),
+  type: RoutersAgentsAgentResponseType$zodSchema.describe(
+    "- `agent`: built inside Calibrate\n- `connection`: your existing agent connected to Calibrate",
+  ),
+  updated_at: z.string().describe(
+    "When the agent was last updated (ISO 8601 UTC)",
+  ),
+  uuid: z.string().describe("ID of the agent"),
+});

package/src/models/routerstestsevaluatorref.ts

@@ -0,0 +1,19 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type RoutersTestsEvaluatorRef = {
+  evaluator_uuid: string;
+  variable_values?: { [k: string]: any } | null | undefined;
+};
+
+export const RoutersTestsEvaluatorRef$zodSchema: z.ZodType<
+  RoutersTestsEvaluatorRef
+> = z.object({
+  evaluator_uuid: z.string().describe("Evaluator to attach to the test"),
+  variable_values: z.record(z.string(), z.any()).nullable().optional().describe(
+    "Values for the evaluator's `{{\"{{\"}}placeholder}}` variables, pinned per test. Omit to inherit the evaluator version's defaults",
+  ),
+});

package/src/models/routersteststestresponse.ts

@@ -0,0 +1,73 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+/**
+ * What the test judges:
+ *
+ * @remarks
+ *
+ * - `response`: judges the generated reply
+ * - `tool_call`: diffs the generated tool calls
+ * - `conversation`: judges the full conversation
+ */
+export const RoutersTestsTestResponseType = {
+  Response: "response",
+  ToolCall: "tool_call",
+  Conversation: "conversation",
+} as const;
+/**
+ * What the test judges:
+ *
+ * @remarks
+ *
+ * - `response`: judges the generated reply
+ * - `tool_call`: diffs the generated tool calls
+ * - `conversation`: judges the full conversation
+ */
+export type RoutersTestsTestResponseType = ClosedEnum<
+  typeof RoutersTestsTestResponseType
+>;
+
+export const RoutersTestsTestResponseType$zodSchema = z.enum([
+  "response",
+  "tool_call",
+  "conversation",
+]).describe(
+  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation",
+);
+
+export type RoutersTestsTestResponse = {
+  uuid: string;
+  name: string;
+  type: RoutersTestsTestResponseType;
+  config?: { [k: string]: any } | null | undefined;
+  created_at: string;
+  updated_at: string;
+  evaluators?: Array<{ [k: string]: any }> | undefined;
+};
+
+export const RoutersTestsTestResponse$zodSchema: z.ZodType<
+  RoutersTestsTestResponse
+> = z.object({
+  config: z.record(z.string(), z.any()).nullable().optional().describe(
+    "Config for the test (`history`, `evaluation`, optional `settings`)",
+  ),
+  created_at: z.string().describe(
+    "Timestamp when the test was created (ISO 8601 UTC)",
+  ),
+  evaluators: z.array(z.record(z.string(), z.any())).optional().describe(
+    "Linked evaluators, resolved to their current live version at read time",
+  ),
+  name: z.string().describe("Name of the test"),
+  type: RoutersTestsTestResponseType$zodSchema.describe(
+    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation",
+  ),
+  updated_at: z.string().describe(
+    "Timestamp when the test was last updated (ISO 8601 UTC)",
+  ),
+  uuid: z.string().describe("Unique ID for the test"),
+});

package/src/models/runagenttestagenttestsagentagentuuidrunpostop.ts

@@ -0,0 +1,39 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  AgentTestRunCreateResponse,
+  AgentTestRunCreateResponse$zodSchema,
+} from "./agenttestruncreateresponse.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import { RunTestRequest, RunTestRequest$zodSchema } from "./runtestrequest.js";
+
+export type RunAgentTestAgentTestsAgentAgentUuidRunPostRequest = {
+  agent_uuid: string;
+  xAPIKey?: string | null | undefined;
+  xOrgUUID?: string | null | undefined;
+  body: RunTestRequest;
+};
+
+export const RunAgentTestAgentTestsAgentAgentUuidRunPostRequest$zodSchema:
+  z.ZodType<RunAgentTestAgentTestsAgentAgentUuidRunPostRequest> = z.object({
+    agent_uuid: z.string().describe("The agent to test."),
+    body: RunTestRequest$zodSchema,
+    xAPIKey: z.string().nullable().optional(),
+    xOrgUUID: z.string().nullable().optional(),
+  });
+
+export type RunAgentTestAgentTestsAgentAgentUuidRunPostResponse =
+  | AgentTestRunCreateResponse
+  | HTTPValidationError;
+
+export const RunAgentTestAgentTestsAgentAgentUuidRunPostResponse$zodSchema:
+  z.ZodType<RunAgentTestAgentTestsAgentAgentUuidRunPostResponse> = z.union([
+    AgentTestRunCreateResponse$zodSchema,
+    HTTPValidationError$zodSchema,
+  ]);

package/src/models/runtestrequest.ts

@@ -0,0 +1,13 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type RunTestRequest = { test_uuids?: Array<string> | null | undefined };
+
+export const RunTestRequest$zodSchema: z.ZodType<RunTestRequest> = z.object({
+  test_uuids: z.array(z.string()).nullable().optional().describe(
+    "Tests to run. Omit to run all tests linked to the agent",
+  ),
+});

package/src/models/runtestsbatchagenttestsrunpostop.ts

@@ -0,0 +1,42 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  BatchRunRequest,
+  BatchRunRequest$zodSchema,
+} from "./batchrunrequest.js";
+import {
+  BatchTestRunResponse,
+  BatchTestRunResponse$zodSchema,
+} from "./batchtestrunresponse.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+
+export type RunTestsBatchAgentTestsRunPostRequest = {
+  xAPIKey?: string | null | undefined;
+  xOrgUUID?: string | null | undefined;
+  body?: BatchRunRequest | null | undefined;
+};
+
+export const RunTestsBatchAgentTestsRunPostRequest$zodSchema: z.ZodType<
+  RunTestsBatchAgentTestsRunPostRequest
+> = z.object({
+  body: BatchRunRequest$zodSchema.nullable().optional(),
+  xAPIKey: z.string().nullable().optional(),
+  xOrgUUID: z.string().nullable().optional(),
+});
+
+export type RunTestsBatchAgentTestsRunPostResponse =
+  | BatchTestRunResponse
+  | HTTPValidationError;
+
+export const RunTestsBatchAgentTestsRunPostResponse$zodSchema: z.ZodType<
+  RunTestsBatchAgentTestsRunPostResponse
+> = z.union([
+  BatchTestRunResponse$zodSchema,
+  HTTPValidationError$zodSchema,
+]);

package/src/models/security.ts

@@ -0,0 +1,13 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type Security = { ApiKeyAuth: string };
+
+export const Security$zodSchema: z.ZodType<Security> = z.object({
+  ApiKeyAuth: z.string().describe(
+    "Workspace API key. Create one under Workspace settings → API keys.",
+  ),
+});

package/src/models/taskstatus.ts

@@ -0,0 +1,23 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+export const TaskStatus = {
+  Queued: "queued",
+  InProgress: "in_progress",
+  Cancelled: "cancelled",
+  Done: "done",
+  Failed: "failed",
+} as const;
+export type TaskStatus = ClosedEnum<typeof TaskStatus>;
+
+export const TaskStatus$zodSchema = z.enum([
+  "queued",
+  "in_progress",
+  "cancelled",
+  "done",
+  "failed",
+]);

package/src/models/testcaseresult.ts

@@ -0,0 +1,45 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { JudgeResult, JudgeResult$zodSchema } from "./judgeresult.js";
+import { TestOutput, TestOutput$zodSchema } from "./testoutput.js";
+
+export type TestCaseResult = {
+  test_case_id?: string | null | undefined;
+  name?: string | null | undefined;
+  passed?: boolean | null | undefined;
+  reasoning?: string | null | undefined;
+  output?: TestOutput | null | undefined;
+  test_case?: { [k: string]: any } | null | undefined;
+  judge_results?: Array<JudgeResult> | null | undefined;
+  latency_ms?: number | null | undefined;
+  cost?: number | null | undefined;
+};
+
+export const TestCaseResult$zodSchema: z.ZodType<TestCaseResult> = z.object({
+  cost: z.number().nullable().optional().describe("Cost of this case in USD"),
+  judge_results: z.array(JudgeResult$zodSchema).nullable().optional().describe(
+    "One verdict per evaluator, for response and conversation tests. Null for tool-call tests or rows still running",
+  ),
+  latency_ms: z.number().nullable().optional().describe(
+    "Response latency in milliseconds for the agent under test",
+  ),
+  name: z.string().nullable().optional().describe("Name of the test"),
+  output: TestOutput$zodSchema.nullable().optional().describe(
+    "The agent's output for this case. Present only when done",
+  ),
+  passed: z.boolean().nullable().optional().describe(
+    "Whether the case passed. Present only when done",
+  ),
+  reasoning: z.string().nullable().optional().describe(
+    "Judge reasoning, or the tool-call diff for tool-call tests",
+  ),
+  test_case: z.record(z.string(), z.any()).nullable().optional().describe(
+    "The input test case definition. Present only when done",
+  ),
+  test_case_id: z.string().nullable().optional().describe(
+    "ID of the test case within the run",
+  ),
+});