@dalmia/calibrate-mcp 0.0.14 → 0.0.16

package/src/models/runagentbenchmarkagenttestsagentagentuuidbenchmarkpostop.ts

@@ -0,0 +1,42 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  AgentTestRunCreateResponse,
+  AgentTestRunCreateResponse$zodSchema,
+} from "./agenttestruncreateresponse.js";
+import {
+  BenchmarkRequest,
+  BenchmarkRequest$zodSchema,
+} from "./benchmarkrequest.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+
+export type RunAgentBenchmarkAgentTestsAgentAgentUuidBenchmarkPostRequest = {
+  agent_uuid: string;
+  xAPIKey?: string | null | undefined;
+  body: BenchmarkRequest;
+};
+
+export const RunAgentBenchmarkAgentTestsAgentAgentUuidBenchmarkPostRequest$zodSchema:
+  z.ZodType<RunAgentBenchmarkAgentTestsAgentAgentUuidBenchmarkPostRequest> = z
+    .object({
+      agent_uuid: z.string().describe("Agent to benchmark"),
+      body: BenchmarkRequest$zodSchema,
+      xAPIKey: z.string().nullable().optional(),
+    });
+
+export type RunAgentBenchmarkAgentTestsAgentAgentUuidBenchmarkPostResponse =
+  | AgentTestRunCreateResponse
+  | HTTPValidationError;
+
+export const RunAgentBenchmarkAgentTestsAgentAgentUuidBenchmarkPostResponse$zodSchema:
+  z.ZodType<RunAgentBenchmarkAgentTestsAgentAgentUuidBenchmarkPostResponse> = z
+    .union([
+      AgentTestRunCreateResponse$zodSchema,
+      HTTPValidationError$zodSchema,
+    ]);

package/src/models/runagenttestagenttestsagentagentuuidrunpostop.ts

@@ -16,16 +16,14 @@ import { RunTestRequest, RunTestRequest$zodSchema } from "./runtestrequest.js";
 export type RunAgentTestAgentTestsAgentAgentUuidRunPostRequest = {
   agent_uuid: string;
   xAPIKey?: string | null | undefined;
-  xOrgUUID?: string | null | undefined;
   body: RunTestRequest;
 };
 
 export const RunAgentTestAgentTestsAgentAgentUuidRunPostRequest$zodSchema:
   z.ZodType<RunAgentTestAgentTestsAgentAgentUuidRunPostRequest> = z.object({
-    agent_uuid: z.string().describe("The agent to test."),
+    agent_uuid: z.string().describe("Agent to test"),
     body: RunTestRequest$zodSchema,
     xAPIKey: z.string().nullable().optional(),
-    xOrgUUID: z.string().nullable().optional(),
   });
 
 export type RunAgentTestAgentTestsAgentAgentUuidRunPostResponse =

package/src/models/runtestsbatchagenttestsrunpostop.ts

@@ -18,7 +18,6 @@ import {
 
 export type RunTestsBatchAgentTestsRunPostRequest = {
   xAPIKey?: string | null | undefined;
-  xOrgUUID?: string | null | undefined;
   body?: BatchRunRequest | null | undefined;
 };
 
@@ -27,7 +26,6 @@ export const RunTestsBatchAgentTestsRunPostRequest$zodSchema: z.ZodType<
 > = z.object({
   body: BatchRunRequest$zodSchema.nullable().optional(),
   xAPIKey: z.string().nullable().optional(),
-  xOrgUUID: z.string().nullable().optional(),
 });
 
 export type RunTestsBatchAgentTestsRunPostResponse =

package/src/models/startevaluatorrunannotationtaskstaskuuidevaluatorrunspostop.ts

@@ -0,0 +1,42 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  EvaluatorRunLaunchResponse,
+  EvaluatorRunLaunchResponse$zodSchema,
+} from "./evaluatorrunlaunchresponse.js";
+import {
+  EvaluatorRunStartRequest,
+  EvaluatorRunStartRequest$zodSchema,
+} from "./evaluatorrunstartrequest.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+
+export type StartEvaluatorRunAnnotationTasksTaskUuidEvaluatorRunsPostRequest = {
+  task_uuid: string;
+  xAPIKey?: string | null | undefined;
+  body: EvaluatorRunStartRequest;
+};
+
+export const StartEvaluatorRunAnnotationTasksTaskUuidEvaluatorRunsPostRequest$zodSchema:
+  z.ZodType<StartEvaluatorRunAnnotationTasksTaskUuidEvaluatorRunsPostRequest> =
+    z.object({
+      body: EvaluatorRunStartRequest$zodSchema,
+      task_uuid: z.string().describe("Annotation task to act on"),
+      xAPIKey: z.string().nullable().optional(),
+    });
+
+export type StartEvaluatorRunAnnotationTasksTaskUuidEvaluatorRunsPostResponse =
+  | EvaluatorRunLaunchResponse
+  | HTTPValidationError;
+
+export const StartEvaluatorRunAnnotationTasksTaskUuidEvaluatorRunsPostResponse$zodSchema:
+  z.ZodType<StartEvaluatorRunAnnotationTasksTaskUuidEvaluatorRunsPostResponse> =
+    z.union([
+      EvaluatorRunLaunchResponse$zodSchema,
+      HTTPValidationError$zodSchema,
+    ]);

package/src/models/taskagreementannotationtaskstaskuuidagreementgetop.ts

@@ -0,0 +1,66 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import {
+  TaskAgreementResponse,
+  TaskAgreementResponse$zodSchema,
+} from "./taskagreementresponse.js";
+
+/**
+ * How to bucket points in the trend series
+ */
+export const Bucket = {
+  Week: "week",
+  Month: "month",
+  Year: "year",
+} as const;
+/**
+ * How to bucket points in the trend series
+ */
+export type Bucket = ClosedEnum<typeof Bucket>;
+
+export const Bucket$zodSchema = z.enum([
+  "week",
+  "month",
+  "year",
+]).describe("How to bucket points in the trend series");
+
+export type TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest = {
+  task_uuid: string;
+  bucket?: Bucket | undefined;
+  days?: number | undefined;
+  xAPIKey?: string | null | undefined;
+};
+
+export const TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest$zodSchema:
+  z.ZodType<TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest> = z.object(
+    {
+      bucket: Bucket$zodSchema.default("week").describe(
+        "How to bucket points in the trend series",
+      ),
+      days: z.int().default(90).describe(
+        "Trailing window in days for the trend series",
+      ),
+      task_uuid: z.string().describe("Annotation task to act on"),
+      xAPIKey: z.string().nullable().optional(),
+    },
+  );
+
+export type TaskAgreementAnnotationTasksTaskUuidAgreementGetResponse =
+  | TaskAgreementResponse
+  | HTTPValidationError;
+
+export const TaskAgreementAnnotationTasksTaskUuidAgreementGetResponse$zodSchema:
+  z.ZodType<TaskAgreementAnnotationTasksTaskUuidAgreementGetResponse> = z.union(
+    [
+      TaskAgreementResponse$zodSchema,
+      HTTPValidationError$zodSchema,
+    ],
+  );

package/src/models/taskagreementresponse.ts

@@ -0,0 +1,52 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { collectExtraKeys as collectExtraKeys$ } from "../lib/schemas.js";
+import {
+  HumanAgreementBlock,
+  HumanAgreementBlock$zodSchema,
+  HumanAgreementBlock$zodSchemaOutbound,
+} from "./humanagreementblock.js";
+
+export type TaskAgreementResponse = {
+  task_id: string;
+  bucket: string;
+  days: number;
+  human_human: HumanAgreementBlock;
+  evaluators: Array<{ [k: string]: any }>;
+  AdditionalProperties?: { [k: string]: any };
+};
+
+export const TaskAgreementResponse$zodSchema: z.ZodType<TaskAgreementResponse> =
+  collectExtraKeys$(
+    z.object({
+      bucket: z.string().describe("Time bucket used for the trend series"),
+      days: z.int().describe("Length of the trailing window in days"),
+      evaluators: z.array(z.record(z.string(), z.any())).describe(
+        "Agreement between each evaluator and the annotators, one entry per linked evaluator",
+      ),
+      human_human: HumanAgreementBlock$zodSchema,
+      task_id: z.string().describe("ID of the task"),
+    }).catchall(z.any()),
+    "AdditionalProperties",
+    true,
+  );
+
+export const TaskAgreementResponse$zodSchemaOutbound: z.ZodType<any> = z.object(
+  {
+    AdditionalProperties: z.record(z.string(), z.any()).optional(),
+    bucket: z.string(),
+    days: z.int(),
+    evaluators: z.array(z.record(z.string(), z.any())),
+    human_human: HumanAgreementBlock$zodSchemaOutbound,
+    task_id: z.string(),
+  },
+).transform((v) => {
+  const { AdditionalProperties, ...rest } = v;
+  return {
+    ...rest,
+    ...AdditionalProperties,
+  };
+});

package/src/models/tasksummaryannotationtaskstaskuuidsummarygetop.ts

@@ -0,0 +1,76 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import {
+  TaskSummaryResponse,
+  TaskSummaryResponse$zodSchema,
+} from "./tasksummaryresponse.js";
+
+/**
+ * Sort direction
+ */
+export const Order = {
+  Asc: "asc",
+  Desc: "desc",
+} as const;
+/**
+ * Sort direction
+ */
+export type Order = ClosedEnum<typeof Order>;
+
+export const Order$zodSchema = z.enum([
+  "asc",
+  "desc",
+]).describe("Sort direction");
+
+export type TaskSummaryAnnotationTasksTaskUuidSummaryGetRequest = {
+  task_uuid: string;
+  item_id?: string | null | undefined;
+  live_only?: boolean | undefined;
+  q?: string | null | undefined;
+  sort_by?: string | undefined;
+  order?: Order | undefined;
+  limit?: number | undefined;
+  offset?: number | undefined;
+  xAPIKey?: string | null | undefined;
+};
+
+export const TaskSummaryAnnotationTasksTaskUuidSummaryGetRequest$zodSchema:
+  z.ZodType<TaskSummaryAnnotationTasksTaskUuidSummaryGetRequest> = z.object({
+    item_id: z.string().describe(
+      "Filter rows to a single item. The full task-wide annotator union is still returned in `annotators`",
+    ).nullable().optional(),
+    limit: z.int().default(50).describe("Maximum number of items to return"),
+    live_only: z.boolean().default(false).describe(
+      "When true, emit only one row for each (item, evaluator) pair using the evaluator's live version. Versions other than the live one that have runs are excluded",
+    ),
+    offset: z.int().default(0).describe(
+      "Number of items to skip before returning results",
+    ),
+    order: Order$zodSchema.default("desc").describe("Sort direction"),
+    q: z.string().describe(
+      "Case-insensitive substring search on `payload.name`. Blank is a no-op",
+    ).nullable().optional(),
+    sort_by: z.string().default("created_at").describe(
+      "Sort key for the results",
+    ),
+    task_uuid: z.string().describe("Annotation task to act on"),
+    xAPIKey: z.string().nullable().optional(),
+  });
+
+export type TaskSummaryAnnotationTasksTaskUuidSummaryGetResponse =
+  | TaskSummaryResponse
+  | HTTPValidationError;
+
+export const TaskSummaryAnnotationTasksTaskUuidSummaryGetResponse$zodSchema:
+  z.ZodType<TaskSummaryAnnotationTasksTaskUuidSummaryGetResponse> = z.union([
+    TaskSummaryResponse$zodSchema,
+    HTTPValidationError$zodSchema,
+  ]);

package/src/models/tasksummaryresponse.ts

@@ -0,0 +1,80 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { collectExtraKeys as collectExtraKeys$ } from "../lib/schemas.js";
+import { ClosedEnum } from "../types/enums.js";
+import { PaginationMeta, PaginationMeta$zodSchema } from "./paginationmeta.js";
+
+/**
+ * Type of the task
+ */
+export const TaskType = {
+  Stt: "stt",
+  Llm: "llm",
+  LlmGeneral: "llm-general",
+  Conversation: "conversation",
+} as const;
+/**
+ * Type of the task
+ */
+export type TaskType = ClosedEnum<typeof TaskType>;
+
+export const TaskType$zodSchema = z.enum([
+  "stt",
+  "llm",
+  "llm-general",
+  "conversation",
+]).describe("Type of the task");
+
+export type TaskSummaryResponse = {
+  task_id: string;
+  task_type: TaskType;
+  evaluators: Array<{ [k: string]: any }>;
+  annotators: Array<{ [k: string]: any }>;
+  rows: Array<{ [k: string]: any }>;
+  item_comments: { [k: string]: any };
+  pagination: PaginationMeta;
+  AdditionalProperties?: { [k: string]: any };
+};
+
+export const TaskSummaryResponse$zodSchema: z.ZodType<TaskSummaryResponse> =
+  collectExtraKeys$(
+    z.object({
+      annotators: z.array(z.record(z.string(), z.any())).describe(
+        "The annotators who labelled or commented on items in the task, stable across pages",
+      ),
+      evaluators: z.array(z.record(z.string(), z.any())).describe(
+        "The task's evaluators, stable across pages, each with its live rubric",
+      ),
+      item_comments: z.record(z.string(), z.any()).describe(
+        "Annotator comments, keyed by item ID",
+      ),
+      pagination: PaginationMeta$zodSchema,
+      rows: z.array(z.record(z.string(), z.any())).describe(
+        "One row per item, evaluator, and version on this page, with the evaluator's value and each annotator's label",
+      ),
+      task_id: z.string().describe("ID of the task"),
+      task_type: TaskType$zodSchema.describe("Type of the task"),
+    }).catchall(z.any()),
+    "AdditionalProperties",
+    true,
+  );
+
+export const TaskSummaryResponse$zodSchemaOutbound: z.ZodType<any> = z.object({
+  AdditionalProperties: z.record(z.string(), z.any()).optional(),
+  annotators: z.array(z.record(z.string(), z.any())),
+  evaluators: z.array(z.record(z.string(), z.any())),
+  item_comments: z.record(z.string(), z.any()),
+  pagination: PaginationMeta$zodSchema,
+  rows: z.array(z.record(z.string(), z.any())),
+  task_id: z.string(),
+  task_type: TaskType$zodSchema,
+}).transform((v) => {
+  const { AdditionalProperties, ...rest } = v;
+  return {
+    ...rest,
+    ...AdditionalProperties,
+  };
+});

package/src/models/testcaseresult.ts

@@ -19,25 +19,23 @@ export type TestCaseResult = {
 };
 
 export const TestCaseResult$zodSchema: z.ZodType<TestCaseResult> = z.object({
-  cost: z.number().nullable().optional().describe("Cost of this case in USD"),
+  cost: z.number().nullable().optional().describe("Cost of this case (USD)"),
   judge_results: z.array(JudgeResult$zodSchema).nullable().optional().describe(
-    "One verdict per evaluator, for response and conversation tests. Null for tool-call tests or rows still running",
+    "One verdict for each evaluator",
   ),
   latency_ms: z.number().nullable().optional().describe(
-    "Response latency in milliseconds for the agent under test",
+    "How long the agent took to respond, in milliseconds",
   ),
   name: z.string().nullable().optional().describe("Name of the test"),
   output: TestOutput$zodSchema.nullable().optional().describe(
-    "The agent's output for this case. Present only when done",
-  ),
-  passed: z.boolean().nullable().optional().describe(
-    "Whether the case passed. Present only when done",
+    "The agent's output for this case",
   ),
+  passed: z.boolean().nullable().optional().describe("Whether the case passed"),
   reasoning: z.string().nullable().optional().describe(
-    "Judge reasoning, or the tool-call diff for tool-call tests",
+    "The judge's reasoning, or the tool-call diff for a tool-call test",
   ),
   test_case: z.record(z.string(), z.any()).nullable().optional().describe(
-    "The input test case definition. Present only when done",
+    "The test case definition that was run",
   ),
   test_case_id: z.string().nullable().optional().describe(
     "ID of the test case within the run",

package/src/models/testcreate.ts

@@ -39,7 +39,7 @@ export const TestCreateType$zodSchema = z.enum([
   "tool_call",
   "conversation",
 ]).describe(
-  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation",
+  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
 );
 
 export type TestCreate = {
@@ -51,7 +51,7 @@ export type TestCreate = {
 
 export const TestCreate$zodSchema: z.ZodType<TestCreate> = z.object({
   config: z.record(z.string(), z.any()).nullable().optional().describe(
-    "The calibrate test config. Three top-level keys.\n\n- `history` (array, required): the conversation up to the agent's turn. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `evaluation` (object, required): `{type, ...}`, where `type` matches the test's `type` (below).\n- `settings` (object, optional): e.g. `{\"language\": \"en\"}`.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nOmit to create the test with no config and fill it in later via update.",
+    "The calibrate test config. Three top-level keys.\n\n- `history`: the required conversation up to the agent's turn. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `evaluation`: the required `{type, ...}`, where `type` matches the test's `type` below.\n- `settings`: an optional object, e.g. `{\"language\": \"en\"}`.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nOmit to create the test with no config and fill it in later via update",
   ),
   evaluators: z.array(RoutersTestsEvaluatorRef$zodSchema).nullable().optional()
     .describe(
@@ -59,6 +59,6 @@ export const TestCreate$zodSchema: z.ZodType<TestCreate> = z.object({
     ),
   name: z.string().describe("Name of the test, unique within the workspace"),
   type: TestCreateType$zodSchema.describe(
-    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation",
+    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
   ),
 });

package/src/models/testoutput.ts

@@ -12,9 +12,9 @@ export type TestOutput = {
 
 export const TestOutput$zodSchema: z.ZodType<TestOutput> = z.object({
   response: z.string().nullable().optional().describe(
-    "The agent's generated reply. Null for tool-call-only cases",
+    "The reply the agent generated",
   ),
   tool_calls: z.array(ToolCallOutput$zodSchema).nullable().optional().describe(
-    "Tool calls the agent generated. Null when it made none",
+    "Tool calls the agent generated",
   ),
 });

package/src/models/testrunevaluator.ts

@@ -5,13 +5,15 @@
 import * as z from "zod";
 import { ClosedEnum } from "../types/enums.js";
 
-export const OutputType = {
+export const TestRunEvaluatorOutputType = {
   Binary: "binary",
   Rating: "rating",
 } as const;
-export type OutputType = ClosedEnum<typeof OutputType>;
+export type TestRunEvaluatorOutputType = ClosedEnum<
+  typeof TestRunEvaluatorOutputType
+>;
 
-export const OutputType$zodSchema = z.enum([
+export const TestRunEvaluatorOutputType$zodSchema = z.enum([
   "binary",
   "rating",
 ]);
@@ -20,7 +22,7 @@ export type TestRunEvaluator = {
   uuid?: string | null | undefined;
   name?: string | null | undefined;
   description?: string | null | undefined;
-  output_type?: OutputType | null | undefined;
+  output_type?: TestRunEvaluatorOutputType | null | undefined;
   output_config?: { [k: string]: any } | null | undefined;
   scale_min?: number | null | undefined;
   scale_max?: number | null | undefined;
@@ -34,20 +36,21 @@ export const TestRunEvaluator$zodSchema: z.ZodType<TestRunEvaluator> = z.object(
     ),
     name: z.string().nullable().optional().describe("Name of the evaluator"),
     output_config: z.record(z.string(), z.any()).nullable().optional().describe(
-      "Rubric: the scale values, labels, and colors a verdict maps to",
-    ),
-    output_type: OutputType$zodSchema.nullable().optional().describe(
-      "Verdict shape: pass/fail or a numeric rating",
+      "The rubric: the scale values, labels, and colors a verdict maps to",
     ),
+    output_type: TestRunEvaluatorOutputType$zodSchema.nullable().optional()
+      .describe(
+        "The shape of the verdict:\n- `binary`: a pass/fail verdict\n- `rating`: a numeric score",
+      ),
     scale_max: z.number().nullable().optional().describe(
-      "Highest rating value. Set for rating evaluators",
+      "Highest value on a rating scale",
     ),
     scale_min: z.number().nullable().optional().describe(
-      "Lowest rating value. Set for rating evaluators",
+      "Lowest value on a rating scale",
     ),
     uuid: z.string().nullable().optional().describe("ID of the evaluator"),
     version_number: z.int().nullable().optional().describe(
-      "Evaluator version this run used",
+      "The evaluator version this run used",
     ),
   },
 );

package/src/models/testrunstatusresponse.ts

@@ -29,7 +29,7 @@ export type TestRunStatusResponse = {
 export const TestRunStatusResponse$zodSchema: z.ZodType<TestRunStatusResponse> =
   z.object({
     cost: z.record(z.string(), z.any()).nullable().optional().describe(
-      "Aggregated cost `{mean, min, max, count}` (USD)",
+      "Aggregated cost as `{mean, min, max, count}` (USD)",
     ),
     error: z.boolean().default(false).describe("True if the run failed"),
     evaluators: z.array(TestRunEvaluator$zodSchema).nullable().optional()
@@ -37,29 +37,29 @@ export const TestRunStatusResponse$zodSchema: z.ZodType<TestRunStatusResponse> =
         "The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`",
       ),
     failed: z.int().nullable().optional().describe(
-      "Number of test cases that failed. Null until done",
+      "Number of test cases that failed",
     ),
     is_public: z.boolean().default(false).describe(
       "Whether the run is shared publicly",
     ),
     latency_ms: z.record(z.string(), z.any()).nullable().optional().describe(
-      "Aggregated response latency in milliseconds: `{p50, p95, p99, count}`",
+      "Aggregated response latency in milliseconds, as `{p50, p95, p99, count}`",
     ),
     passed: z.int().nullable().optional().describe(
-      "Number of test cases that passed. Null until done",
+      "Number of test cases that passed",
     ),
     results: z.array(TestCaseResult$zodSchema).nullable().optional().describe(
-      "Results for each test case. Null until available",
+      "Results for each test case",
     ),
     share_token: z.string().nullable().optional().describe(
-      "Public share token. Null unless the run is public",
+      "Token for building the public share URL",
     ),
     status: TaskStatus$zodSchema,
     task_id: z.string().describe("Test run job ID"),
     total_tests: z.int().nullable().optional().describe(
-      "Total number of test cases. Null until known",
+      "Total number of test cases",
     ),
     total_tokens: z.record(z.string(), z.any()).nullable().optional().describe(
-      "Aggregated token usage `{mean, min, max, count}`",
+      "Aggregated token usage as `{mean, min, max, count}`",
     ),
   });

package/src/models/testupdate.ts

@@ -31,7 +31,7 @@ export type TestUpdate = {
 
 export const TestUpdate$zodSchema: z.ZodType<TestUpdate> = z.object({
   config: z.record(z.string(), z.any()).nullable().optional().describe(
-    "The calibrate test config. Three top-level keys.\n\n- `history` (array, required): the conversation up to the agent's turn. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `evaluation` (object, required): `{type, ...}`, where `type` matches the test's `type` (below).\n- `settings` (object, optional): e.g. `{\"language\": \"en\"}`.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nReplaces the stored config. Omit to leave unchanged.",
+    "The calibrate test config. Three top-level keys.\n\n- `history`: the required conversation up to the agent's turn. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `evaluation`: the required `{type, ...}`, where `type` matches the test's `type` below.\n- `settings`: an optional object, e.g. `{\"language\": \"en\"}`.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nReplaces the stored config. Omit to leave unchanged",
   ),
   evaluators: z.array(RoutersTestsEvaluatorRef$zodSchema).nullable().optional()
     .describe(
@@ -41,6 +41,6 @@ export const TestUpdate$zodSchema: z.ZodType<TestUpdate> = z.object({
     "New test name. Omit to leave unchanged",
   ),
   type: TestUpdateType$zodSchema.nullable().optional().describe(
-    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\nImmutable. Omit, or send the existing value. A different value is rejected (400).",
+    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\n\nImmutable. Omit it, or send the current value",
   ),
 });

package/src/models/toolcalloutput.ts

@@ -12,10 +12,10 @@ export type ToolCallOutput = {
 
 export const ToolCallOutput$zodSchema: z.ZodType<ToolCallOutput> = z.object({
   arguments: z.record(z.string(), z.any()).nullable().optional().describe(
-    "Arguments the agent passed to the tool. Null if none",
+    "Arguments the agent passed to the tool",
   ),
   output: z.any().nullable().optional().describe(
-    "Tool execution result (any JSON value). Present only when the agent runs the tool and returns its result",
+    "Tool execution result, when the agent ran the tool and returned its result",
   ),
   tool: z.string().describe("Name of the tool the agent called"),
 });

package/src/models/updateagentendpointagentsagentuuidputop.ts

@@ -16,17 +16,15 @@ import {
 export type UpdateAgentEndpointAgentsAgentUuidPutRequest = {
   agent_uuid: string;
   xAPIKey?: string | null | undefined;
-  xOrgUUID?: string | null | undefined;
   body: AgentUpdate;
 };
 
 export const UpdateAgentEndpointAgentsAgentUuidPutRequest$zodSchema: z.ZodType<
   UpdateAgentEndpointAgentsAgentUuidPutRequest
 > = z.object({
-  agent_uuid: z.string().describe("The agent to update."),
+  agent_uuid: z.string().describe("The agent to update"),
   body: AgentUpdate$zodSchema,
   xAPIKey: z.string().nullable().optional(),
-  xOrgUUID: z.string().nullable().optional(),
 });
 
 export type UpdateAgentEndpointAgentsAgentUuidPutResponse =

package/src/models/updatetestendpointteststestuuidputop.ts

@@ -16,7 +16,6 @@ import { TestUpdate, TestUpdate$zodSchema } from "./testupdate.js";
 export type UpdateTestEndpointTestsTestUuidPutRequest = {
   test_uuid: string;
   xAPIKey?: string | null | undefined;
-  xOrgUUID?: string | null | undefined;
   body: TestUpdate;
 };
 
@@ -26,7 +25,6 @@ export const UpdateTestEndpointTestsTestUuidPutRequest$zodSchema: z.ZodType<
   body: TestUpdate$zodSchema,
   test_uuid: z.string().describe("Test to update"),
   xAPIKey: z.string().nullable().optional(),
-  xOrgUUID: z.string().nullable().optional(),
 });
 
 export type UpdateTestEndpointTestsTestUuidPutResponse =