@dalmia/calibrate-mcp 0.0.14 → 0.0.15

package/src/models/bulkcreateitemsannotationtaskstaskuuiditemspostop.ts

@@ -0,0 +1,40 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  BulkCreateItemsResponse,
+  BulkCreateItemsResponse$zodSchema,
+} from "./bulkcreateitemsresponse.js";
+import {
+  BulkItemsRequest,
+  BulkItemsRequest$zodSchema,
+} from "./bulkitemsrequest.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+
+export type BulkCreateItemsAnnotationTasksTaskUuidItemsPostRequest = {
+  task_uuid: string;
+  xAPIKey?: string | null | undefined;
+  body: BulkItemsRequest;
+};
+
+export const BulkCreateItemsAnnotationTasksTaskUuidItemsPostRequest$zodSchema:
+  z.ZodType<BulkCreateItemsAnnotationTasksTaskUuidItemsPostRequest> = z.object({
+    body: BulkItemsRequest$zodSchema,
+    task_uuid: z.string().describe("Annotation task to act on"),
+    xAPIKey: z.string().nullable().optional(),
+  });
+
+export type BulkCreateItemsAnnotationTasksTaskUuidItemsPostResponse =
+  | BulkCreateItemsResponse
+  | HTTPValidationError;
+
+export const BulkCreateItemsAnnotationTasksTaskUuidItemsPostResponse$zodSchema:
+  z.ZodType<BulkCreateItemsAnnotationTasksTaskUuidItemsPostResponse> = z.union([
+    BulkCreateItemsResponse$zodSchema,
+    HTTPValidationError$zodSchema,
+  ]);

package/src/models/bulkcreateitemsresponse.ts

@@ -0,0 +1,31 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type BulkCreateItemsResponse = {
+  item_ids: Array<string>;
+  count: number;
+  new_item_ids?: Array<string> | null | undefined;
+  existing_item_ids?: Array<string> | null | undefined;
+  annotation_job_id?: string | null | undefined;
+};
+
+export const BulkCreateItemsResponse$zodSchema: z.ZodType<
+  BulkCreateItemsResponse
+> = z.object({
+  annotation_job_id: z.string().nullable().optional().describe(
+    "ID of the labelling job that holds the annotations you provided, present only when you send annotations with the items",
+  ),
+  count: z.int().describe("How many items the request resolved to"),
+  existing_item_ids: z.array(z.string()).nullable().optional().describe(
+    "IDs of items that already existed by name and were reused",
+  ),
+  item_ids: z.array(z.string()).describe(
+    "IDs of every item the request resolved to, in request order",
+  ),
+  new_item_ids: z.array(z.string()).nullable().optional().describe(
+    "IDs of the items that were newly created",
+  ),
+});

package/src/models/bulkitemsrequest.ts

@@ -0,0 +1,25 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  AnnotationItemPayload,
+  AnnotationItemPayload$zodSchema,
+} from "./annotationitempayload.js";
+
+export type BulkItemsRequest = {
+  items: Array<AnnotationItemPayload>;
+  annotator_id?: string | null | undefined;
+};
+
+export const BulkItemsRequest$zodSchema: z.ZodType<BulkItemsRequest> = z.object(
+  {
+    annotator_id: z.string().nullable().optional().describe(
+      "Annotator these initial annotations belong to. Required when any item carries annotations",
+    ),
+    items: z.array(AnnotationItemPayload$zodSchema).describe(
+      "Items to insert. Insertion order is preserved",
+    ),
+  },
+);

package/src/models/bulktestitem.ts

@@ -22,7 +22,7 @@ export type BulkTestItem = {
 
 export const BulkTestItem$zodSchema: z.ZodType<BulkTestItem> = z.object({
   conversation_history: z.array(ChatMessage$zodSchema).describe(
-    "Ordered messages ending at the user turn the agent should answer (must be non-empty)",
+    "Ordered messages ending at the user turn the agent should answer",
   ),
   evaluators: z.array(RoutersTestsEvaluatorRef$zodSchema).nullable().optional()
     .describe(

package/src/models/bulktestupload.ts

@@ -15,7 +15,7 @@ import { BulkTestItem, BulkTestItem$zodSchema } from "./bulktestitem.js";
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
  *
- * Applied to every test in the batch.
+ * Applied to every test in the batch
  */
 export const BulkTestUploadType = {
   Response: "response",
@@ -31,7 +31,7 @@ export const BulkTestUploadType = {
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
  *
- * Applied to every test in the batch.
+ * Applied to every test in the batch
  */
 export type BulkTestUploadType = ClosedEnum<typeof BulkTestUploadType>;
 
@@ -40,7 +40,7 @@ export const BulkTestUploadType$zodSchema = z.enum([
   "tool_call",
   "conversation",
 ]).describe(
-  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\nApplied to every test in the batch.",
+  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\n\nApplied to every test in the batch",
 );
 
 export type BulkTestUpload = {
@@ -52,15 +52,15 @@ export type BulkTestUpload = {
 
 export const BulkTestUpload$zodSchema: z.ZodType<BulkTestUpload> = z.object({
   agent_uuids: z.array(z.string()).nullable().optional().describe(
-    "Agents (IDs) to link every created test to. Omit to link none",
+    "IDs of agents to link every created test to. Omit to link none",
   ),
   language: z.string().nullable().optional().describe(
     "Language written to each test's `config.settings.language`. Omit to leave unset",
   ),
   tests: z.array(BulkTestItem$zodSchema).describe(
-    "Test items to create (non-empty, max 500 per request, names unique within the batch)",
+    "Test items to create, at most 500 per request, with names unique within the batch",
   ),
   type: BulkTestUploadType$zodSchema.describe(
-    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\nApplied to every test in the batch.",
+    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\n\nApplied to every test in the batch",
   ),
 });

package/src/models/bulktestuploadresponse.ts

@@ -20,6 +20,6 @@ export const BulkTestUploadResponse$zodSchema: z.ZodType<
     "IDs of the created tests, in request order",
   ),
   warnings: z.array(z.string()).nullable().optional().describe(
-    "Non-fatal issues (e.g. agents some tests couldn't link to). Null when there were none",
+    "Non-fatal issues, such as agents some tests could not be linked to",
   ),
 });

package/src/models/bulkupdateitemsannotationtaskstaskuuiditemsputop.ts

@@ -0,0 +1,40 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  BulkUpdateItemsRequest,
+  BulkUpdateItemsRequest$zodSchema,
+} from "./bulkupdateitemsrequest.js";
+import {
+  BulkUpdateItemsResponse,
+  BulkUpdateItemsResponse$zodSchema,
+} from "./bulkupdateitemsresponse.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+
+export type BulkUpdateItemsAnnotationTasksTaskUuidItemsPutRequest = {
+  task_uuid: string;
+  xAPIKey?: string | null | undefined;
+  body: BulkUpdateItemsRequest;
+};
+
+export const BulkUpdateItemsAnnotationTasksTaskUuidItemsPutRequest$zodSchema:
+  z.ZodType<BulkUpdateItemsAnnotationTasksTaskUuidItemsPutRequest> = z.object({
+    body: BulkUpdateItemsRequest$zodSchema,
+    task_uuid: z.string().describe("Annotation task to act on"),
+    xAPIKey: z.string().nullable().optional(),
+  });
+
+export type BulkUpdateItemsAnnotationTasksTaskUuidItemsPutResponse =
+  | BulkUpdateItemsResponse
+  | HTTPValidationError;
+
+export const BulkUpdateItemsAnnotationTasksTaskUuidItemsPutResponse$zodSchema:
+  z.ZodType<BulkUpdateItemsAnnotationTasksTaskUuidItemsPutResponse> = z.union([
+    BulkUpdateItemsResponse$zodSchema,
+    HTTPValidationError$zodSchema,
+  ]);

package/src/models/bulkupdateitemsrequest.ts

@@ -0,0 +1,19 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  ItemUpdatePayload,
+  ItemUpdatePayload$zodSchema,
+} from "./itemupdatepayload.js";
+
+export type BulkUpdateItemsRequest = { updates: Array<ItemUpdatePayload> };
+
+export const BulkUpdateItemsRequest$zodSchema: z.ZodType<
+  BulkUpdateItemsRequest
+> = z.object({
+  updates: z.array(ItemUpdatePayload$zodSchema).describe(
+    "The new payload for each item you're updating. Entries not in this task, or referencing deleted items, are skipped",
+  ),
+});

package/src/models/bulkupdateitemsresponse.ts

@@ -0,0 +1,13 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type BulkUpdateItemsResponse = { updated_count: number };
+
+export const BulkUpdateItemsResponse$zodSchema: z.ZodType<
+  BulkUpdateItemsResponse
+> = z.object({
+  updated_count: z.int().describe("How many items were updated"),
+});

package/src/models/bulkuploadteststestsbulkpostop.ts

@@ -15,7 +15,6 @@ import {
 
 export type BulkUploadTestsTestsBulkPostRequest = {
   xAPIKey?: string | null | undefined;
-  xOrgUUID?: string | null | undefined;
   body: BulkTestUpload;
 };
 
@@ -24,7 +23,6 @@ export const BulkUploadTestsTestsBulkPostRequest$zodSchema: z.ZodType<
 > = z.object({
   body: BulkTestUpload$zodSchema,
   xAPIKey: z.string().nullable().optional(),
-  xOrgUUID: z.string().nullable().optional(),
 });
 
 export type BulkUploadTestsTestsBulkPostResponse =

package/src/models/createagentendpointagentspostop.ts

@@ -15,7 +15,6 @@ import {
 
 export type CreateAgentEndpointAgentsPostRequest = {
   xAPIKey?: string | null | undefined;
-  xOrgUUID?: string | null | undefined;
   body: AgentCreate;
 };
 
@@ -24,7 +23,6 @@ export const CreateAgentEndpointAgentsPostRequest$zodSchema: z.ZodType<
 > = z.object({
   body: AgentCreate$zodSchema,
   xAPIKey: z.string().nullable().optional(),
-  xOrgUUID: z.string().nullable().optional(),
 });
 
 export type CreateAgentEndpointAgentsPostResponse =

package/src/models/createagenttestlinksagenttestspostop.ts

@@ -0,0 +1,40 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  AgentTestsCreate,
+  AgentTestsCreate$zodSchema,
+} from "./agenttestscreate.js";
+import {
+  AgentTestsCreateResponse,
+  AgentTestsCreateResponse$zodSchema,
+} from "./agenttestscreateresponse.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+
+export type CreateAgentTestLinksAgentTestsPostRequest = {
+  xAPIKey?: string | null | undefined;
+  body: AgentTestsCreate;
+};
+
+export const CreateAgentTestLinksAgentTestsPostRequest$zodSchema: z.ZodType<
+  CreateAgentTestLinksAgentTestsPostRequest
+> = z.object({
+  body: AgentTestsCreate$zodSchema,
+  xAPIKey: z.string().nullable().optional(),
+});
+
+export type CreateAgentTestLinksAgentTestsPostResponse =
+  | AgentTestsCreateResponse
+  | HTTPValidationError;
+
+export const CreateAgentTestLinksAgentTestsPostResponse$zodSchema: z.ZodType<
+  CreateAgentTestLinksAgentTestsPostResponse
+> = z.union([
+  AgentTestsCreateResponse$zodSchema,
+  HTTPValidationError$zodSchema,
+]);

package/src/models/createannotationtaskendpointannotationtaskspostop.ts

@@ -0,0 +1,38 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  AnnotationTaskCreate,
+  AnnotationTaskCreate$zodSchema,
+} from "./annotationtaskcreate.js";
+import {
+  AnnotationTaskCreateResponse,
+  AnnotationTaskCreateResponse$zodSchema,
+} from "./annotationtaskcreateresponse.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+
+export type CreateAnnotationTaskEndpointAnnotationTasksPostRequest = {
+  xAPIKey?: string | null | undefined;
+  body: AnnotationTaskCreate;
+};
+
+export const CreateAnnotationTaskEndpointAnnotationTasksPostRequest$zodSchema:
+  z.ZodType<CreateAnnotationTaskEndpointAnnotationTasksPostRequest> = z.object({
+    body: AnnotationTaskCreate$zodSchema,
+    xAPIKey: z.string().nullable().optional(),
+  });
+
+export type CreateAnnotationTaskEndpointAnnotationTasksPostResponse =
+  | AnnotationTaskCreateResponse
+  | HTTPValidationError;
+
+export const CreateAnnotationTaskEndpointAnnotationTasksPostResponse$zodSchema:
+  z.ZodType<CreateAnnotationTaskEndpointAnnotationTasksPostResponse> = z.union([
+    AnnotationTaskCreateResponse$zodSchema,
+    HTTPValidationError$zodSchema,
+  ]);

package/src/models/createevaluatorendpointevaluatorspostop.ts

@@ -0,0 +1,40 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  EvaluatorCreate,
+  EvaluatorCreate$zodSchema,
+} from "./evaluatorcreate.js";
+import {
+  EvaluatorCreateResponse,
+  EvaluatorCreateResponse$zodSchema,
+} from "./evaluatorcreateresponse.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+
+export type CreateEvaluatorEndpointEvaluatorsPostRequest = {
+  xAPIKey?: string | null | undefined;
+  body: EvaluatorCreate;
+};
+
+export const CreateEvaluatorEndpointEvaluatorsPostRequest$zodSchema: z.ZodType<
+  CreateEvaluatorEndpointEvaluatorsPostRequest
+> = z.object({
+  body: EvaluatorCreate$zodSchema,
+  xAPIKey: z.string().nullable().optional(),
+});
+
+export type CreateEvaluatorEndpointEvaluatorsPostResponse =
+  | EvaluatorCreateResponse
+  | HTTPValidationError;
+
+export const CreateEvaluatorEndpointEvaluatorsPostResponse$zodSchema: z.ZodType<
+  CreateEvaluatorEndpointEvaluatorsPostResponse
+> = z.union([
+  EvaluatorCreateResponse$zodSchema,
+  HTTPValidationError$zodSchema,
+]);

package/src/models/createtestendpointtestspostop.ts

@@ -15,7 +15,6 @@ import {
 
 export type CreateTestEndpointTestsPostRequest = {
   xAPIKey?: string | null | undefined;
-  xOrgUUID?: string | null | undefined;
   body: TestCreate;
 };
 
@@ -24,7 +23,6 @@ export const CreateTestEndpointTestsPostRequest$zodSchema: z.ZodType<
 > = z.object({
   body: TestCreate$zodSchema,
   xAPIKey: z.string().nullable().optional(),
-  xOrgUUID: z.string().nullable().optional(),
 });
 
 export type CreateTestEndpointTestsPostResponse =

package/src/models/createversionevaluatorsevaluatoruuidversionspostop.ts

@@ -0,0 +1,44 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  EvaluatorVersionCreateRequest,
+  EvaluatorVersionCreateRequest$zodSchema,
+} from "./evaluatorversioncreaterequest.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import {
+  VersionCreateResponse,
+  VersionCreateResponse$zodSchema,
+} from "./versioncreateresponse.js";
+
+export type CreateVersionEvaluatorsEvaluatorUuidVersionsPostRequest = {
+  evaluator_uuid: string;
+  xAPIKey?: string | null | undefined;
+  body: EvaluatorVersionCreateRequest;
+};
+
+export const CreateVersionEvaluatorsEvaluatorUuidVersionsPostRequest$zodSchema:
+  z.ZodType<CreateVersionEvaluatorsEvaluatorUuidVersionsPostRequest> = z.object(
+    {
+      body: EvaluatorVersionCreateRequest$zodSchema,
+      evaluator_uuid: z.string().describe("Evaluator to add a version to"),
+      xAPIKey: z.string().nullable().optional(),
+    },
+  );
+
+export type CreateVersionEvaluatorsEvaluatorUuidVersionsPostResponse =
+  | VersionCreateResponse
+  | HTTPValidationError;
+
+export const CreateVersionEvaluatorsEvaluatorUuidVersionsPostResponse$zodSchema:
+  z.ZodType<CreateVersionEvaluatorsEvaluatorUuidVersionsPostResponse> = z.union(
+    [
+      VersionCreateResponse$zodSchema,
+      HTTPValidationError$zodSchema,
+    ],
+  );

package/src/models/evaluatorcreate.ts

@@ -0,0 +1,142 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+import {
+  EvaluatorVersionCreate,
+  EvaluatorVersionCreate$zodSchema,
+} from "./evaluatorversioncreate.js";
+
+/**
+ * What the evaluator judges:
+ *
+ * @remarks
+ *
+ * - `tts`: TTS audio
+ * - `stt`: one transcript
+ * - `llm`: a reply with its conversation history
+ * - `llm-general`: a standalone input and output pair
+ * - `conversation`: a full conversation
+ */
+export const EvaluatorCreateEvaluatorType = {
+  Tts: "tts",
+  Stt: "stt",
+  Llm: "llm",
+  LlmGeneral: "llm-general",
+  Conversation: "conversation",
+} as const;
+/**
+ * What the evaluator judges:
+ *
+ * @remarks
+ *
+ * - `tts`: TTS audio
+ * - `stt`: one transcript
+ * - `llm`: a reply with its conversation history
+ * - `llm-general`: a standalone input and output pair
+ * - `conversation`: a full conversation
+ */
+export type EvaluatorCreateEvaluatorType = ClosedEnum<
+  typeof EvaluatorCreateEvaluatorType
+>;
+
+export const EvaluatorCreateEvaluatorType$zodSchema = z.enum([
+  "tts",
+  "stt",
+  "llm",
+  "llm-general",
+  "conversation",
+]).describe(
+  "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n",
+);
+
+/**
+ * The modality the judge reads:
+ *
+ * @remarks
+ *
+ * - `text`
+ * - `audio`
+ */
+export const EvaluatorCreateDataType = {
+  Text: "text",
+  Audio: "audio",
+} as const;
+/**
+ * The modality the judge reads:
+ *
+ * @remarks
+ *
+ * - `text`
+ * - `audio`
+ */
+export type EvaluatorCreateDataType = ClosedEnum<
+  typeof EvaluatorCreateDataType
+>;
+
+export const EvaluatorCreateDataType$zodSchema = z.enum([
+  "text",
+  "audio",
+]).describe("The modality the judge reads:\n\n- `text`\n- `audio`\n");
+
+/**
+ * How the evaluator scores:
+ *
+ * @remarks
+ *
+ * - `binary`: pass or fail
+ * - `rating`: a numeric score, using the scale in `output_config`
+ */
+export const EvaluatorCreateOutputType = {
+  Binary: "binary",
+  Rating: "rating",
+} as const;
+/**
+ * How the evaluator scores:
+ *
+ * @remarks
+ *
+ * - `binary`: pass or fail
+ * - `rating`: a numeric score, using the scale in `output_config`
+ */
+export type EvaluatorCreateOutputType = ClosedEnum<
+  typeof EvaluatorCreateOutputType
+>;
+
+export const EvaluatorCreateOutputType$zodSchema = z.enum([
+  "binary",
+  "rating",
+]).describe(
+  "How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n",
+);
+
+export type EvaluatorCreate = {
+  name: string;
+  description?: string | null | undefined;
+  evaluator_type?: EvaluatorCreateEvaluatorType | undefined;
+  data_type?: EvaluatorCreateDataType | undefined;
+  output_type?: EvaluatorCreateOutputType | undefined;
+  version: EvaluatorVersionCreate;
+};
+
+export const EvaluatorCreate$zodSchema: z.ZodType<EvaluatorCreate> = z.object({
+  data_type: EvaluatorCreateDataType$zodSchema.default("text").describe(
+    "The modality the judge reads:\n\n- `text`\n- `audio`\n",
+  ),
+  description: z.string().nullable().optional().describe(
+    "Description. Omit to leave blank",
+  ),
+  evaluator_type: EvaluatorCreateEvaluatorType$zodSchema.default("llm")
+    .describe(
+      "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n",
+    ),
+  name: z.string().describe("Evaluator name, unique within your workspace"),
+  output_type: EvaluatorCreateOutputType$zodSchema.default("binary").describe(
+    "How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n",
+  ),
+  version: EvaluatorVersionCreate$zodSchema.describe(
+    "One version of an evaluator: its judge prompt, model, variables, and rubric.",
+  ),
+});

package/src/models/evaluatorcreateresponse.ts

@@ -0,0 +1,14 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+export type EvaluatorCreateResponse = { uuid: string; version_uuid: string };
+
+export const EvaluatorCreateResponse$zodSchema: z.ZodType<
+  EvaluatorCreateResponse
+> = z.object({
+  uuid: z.string().describe("ID of the created evaluator"),
+  version_uuid: z.string().describe("ID of its initial version"),
+});