@dalmia/calibrate-mcp 0.0.14 → 0.0.15

package/src/models/routersevaluatorsevaluatorresponse.ts

@@ -0,0 +1,167 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+import {
+  EvaluatorVersionResponse,
+  EvaluatorVersionResponse$zodSchema,
+} from "./evaluatorversionresponse.js";
+
+/**
+ * What the evaluator judges:
+ *
+ * @remarks
+ *
+ * - `tts`: TTS audio
+ * - `stt`: one transcript
+ * - `llm`: a reply with its conversation history
+ * - `llm-general`: a standalone input and output pair
+ * - `conversation`: a full conversation
+ */
+export const RoutersEvaluatorsEvaluatorResponseEvaluatorType = {
+  Tts: "tts",
+  Stt: "stt",
+  Llm: "llm",
+  LlmGeneral: "llm-general",
+  Conversation: "conversation",
+} as const;
+/**
+ * What the evaluator judges:
+ *
+ * @remarks
+ *
+ * - `tts`: TTS audio
+ * - `stt`: one transcript
+ * - `llm`: a reply with its conversation history
+ * - `llm-general`: a standalone input and output pair
+ * - `conversation`: a full conversation
+ */
+export type RoutersEvaluatorsEvaluatorResponseEvaluatorType = ClosedEnum<
+  typeof RoutersEvaluatorsEvaluatorResponseEvaluatorType
+>;
+
+export const RoutersEvaluatorsEvaluatorResponseEvaluatorType$zodSchema = z.enum(
+  [
+    "tts",
+    "stt",
+    "llm",
+    "llm-general",
+    "conversation",
+  ],
+).describe(
+  "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n",
+);
+
+/**
+ * The modality the judge reads:
+ *
+ * @remarks
+ *
+ * - `text`
+ * - `audio`
+ */
+export const RoutersEvaluatorsEvaluatorResponseDataType = {
+  Text: "text",
+  Audio: "audio",
+} as const;
+/**
+ * The modality the judge reads:
+ *
+ * @remarks
+ *
+ * - `text`
+ * - `audio`
+ */
+export type RoutersEvaluatorsEvaluatorResponseDataType = ClosedEnum<
+  typeof RoutersEvaluatorsEvaluatorResponseDataType
+>;
+
+export const RoutersEvaluatorsEvaluatorResponseDataType$zodSchema = z.enum([
+  "text",
+  "audio",
+]).describe("The modality the judge reads:\n\n- `text`\n- `audio`\n");
+
+/**
+ * How the evaluator scores:
+ *
+ * @remarks
+ *
+ * - `binary`: pass or fail
+ * - `rating`: a numeric score, using the scale in `output_config`
+ */
+export const RoutersEvaluatorsEvaluatorResponseOutputType = {
+  Binary: "binary",
+  Rating: "rating",
+} as const;
+/**
+ * How the evaluator scores:
+ *
+ * @remarks
+ *
+ * - `binary`: pass or fail
+ * - `rating`: a numeric score, using the scale in `output_config`
+ */
+export type RoutersEvaluatorsEvaluatorResponseOutputType = ClosedEnum<
+  typeof RoutersEvaluatorsEvaluatorResponseOutputType
+>;
+
+export const RoutersEvaluatorsEvaluatorResponseOutputType$zodSchema = z.enum([
+  "binary",
+  "rating",
+]).describe(
+  "How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n",
+);
+
+export type RoutersEvaluatorsEvaluatorResponse = {
+  uuid: string;
+  name: string;
+  description?: string | null | undefined;
+  evaluator_type: RoutersEvaluatorsEvaluatorResponseEvaluatorType;
+  data_type: RoutersEvaluatorsEvaluatorResponseDataType;
+  output_type: RoutersEvaluatorsEvaluatorResponseOutputType;
+  is_default: boolean;
+  slug?: string | null | undefined;
+  live_version_id?: string | null | undefined;
+  created_at: string;
+  updated_at: string;
+  live_version?: EvaluatorVersionResponse | null | undefined;
+};
+
+export const RoutersEvaluatorsEvaluatorResponse$zodSchema: z.ZodType<
+  RoutersEvaluatorsEvaluatorResponse
+> = z.object({
+  created_at: z.string().describe(
+    "When the evaluator was created (ISO 8601 UTC)",
+  ),
+  data_type: RoutersEvaluatorsEvaluatorResponseDataType$zodSchema.describe(
+    "The modality the judge reads:\n\n- `text`\n- `audio`\n",
+  ),
+  description: z.string().nullable().optional().describe(
+    "What the evaluator checks",
+  ),
+  evaluator_type: RoutersEvaluatorsEvaluatorResponseEvaluatorType$zodSchema
+    .describe(
+      "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n",
+    ),
+  is_default: z.boolean().describe(
+    "True for a built-in default evaluator, which you can't edit. False for an evaluator you created, which you can edit and add versions to",
+  ),
+  live_version: EvaluatorVersionResponse$zodSchema.nullable().optional()
+    .describe("The version that is currently live"),
+  live_version_id: z.string().nullable().optional().describe(
+    "ID of the version that is currently live",
+  ),
+  name: z.string().describe("Evaluator name"),
+  output_type: RoutersEvaluatorsEvaluatorResponseOutputType$zodSchema.describe(
+    "How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n",
+  ),
+  slug: z.string().nullable().optional().describe(
+    "Stable slug for a built-in default evaluator",
+  ),
+  updated_at: z.string().describe(
+    "When the evaluator was last updated (ISO 8601 UTC)",
+  ),
+  uuid: z.string().describe("Evaluator ID"),
+});

package/src/models/routerstestsevaluatorref.ts

@@ -14,6 +14,6 @@ export const RoutersTestsEvaluatorRef$zodSchema: z.ZodType<
 > = z.object({
   evaluator_uuid: z.string().describe("Evaluator to attach to the test"),
   variable_values: z.record(z.string(), z.any()).nullable().optional().describe(
-    "Values for the evaluator's `{{\"{{\"}}placeholder}}` variables, pinned per test. Omit to inherit the evaluator version's defaults",
+    "Values for the evaluator's `{{\"{{\"}}placeholder}}` variables, pinned on this test. Omit to inherit the evaluator version's defaults",
   ),
 });

package/src/models/routersteststestresponse.ts

@@ -37,7 +37,7 @@ export const RoutersTestsTestResponseType$zodSchema = z.enum([
   "tool_call",
   "conversation",
 ]).describe(
-  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation",
+  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
 );
 
 export type RoutersTestsTestResponse = {
@@ -54,20 +54,18 @@ export const RoutersTestsTestResponse$zodSchema: z.ZodType<
   RoutersTestsTestResponse
 > = z.object({
   config: z.record(z.string(), z.any()).nullable().optional().describe(
-    "Config for the test (`history`, `evaluation`, optional `settings`)",
-  ),
-  created_at: z.string().describe(
-    "Timestamp when the test was created (ISO 8601 UTC)",
+    "The stored config: `history`, `evaluation`, and an optional `settings`",
   ),
+  created_at: z.string().describe("When the test was created (ISO 8601 UTC)"),
   evaluators: z.array(z.record(z.string(), z.any())).optional().describe(
     "Linked evaluators, resolved to their current live version at read time",
   ),
   name: z.string().describe("Name of the test"),
   type: RoutersTestsTestResponseType$zodSchema.describe(
-    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation",
+    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
   ),
   updated_at: z.string().describe(
-    "Timestamp when the test was last updated (ISO 8601 UTC)",
+    "When the test was last updated (ISO 8601 UTC)",
   ),
   uuid: z.string().describe("Unique ID for the test"),
 });

package/src/models/runagentbenchmarkagenttestsagentagentuuidbenchmarkpostop.ts

@@ -0,0 +1,42 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  AgentTestRunCreateResponse,
+  AgentTestRunCreateResponse$zodSchema,
+} from "./agenttestruncreateresponse.js";
+import {
+  BenchmarkRequest,
+  BenchmarkRequest$zodSchema,
+} from "./benchmarkrequest.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+
+export type RunAgentBenchmarkAgentTestsAgentAgentUuidBenchmarkPostRequest = {
+  agent_uuid: string;
+  xAPIKey?: string | null | undefined;
+  body: BenchmarkRequest;
+};
+
+export const RunAgentBenchmarkAgentTestsAgentAgentUuidBenchmarkPostRequest$zodSchema:
+  z.ZodType<RunAgentBenchmarkAgentTestsAgentAgentUuidBenchmarkPostRequest> = z
+    .object({
+      agent_uuid: z.string().describe("Agent to benchmark"),
+      body: BenchmarkRequest$zodSchema,
+      xAPIKey: z.string().nullable().optional(),
+    });
+
+export type RunAgentBenchmarkAgentTestsAgentAgentUuidBenchmarkPostResponse =
+  | AgentTestRunCreateResponse
+  | HTTPValidationError;
+
+export const RunAgentBenchmarkAgentTestsAgentAgentUuidBenchmarkPostResponse$zodSchema:
+  z.ZodType<RunAgentBenchmarkAgentTestsAgentAgentUuidBenchmarkPostResponse> = z
+    .union([
+      AgentTestRunCreateResponse$zodSchema,
+      HTTPValidationError$zodSchema,
+    ]);

package/src/models/runagenttestagenttestsagentagentuuidrunpostop.ts

@@ -16,16 +16,14 @@ import { RunTestRequest, RunTestRequest$zodSchema } from "./runtestrequest.js";
 export type RunAgentTestAgentTestsAgentAgentUuidRunPostRequest = {
   agent_uuid: string;
   xAPIKey?: string | null | undefined;
-  xOrgUUID?: string | null | undefined;
   body: RunTestRequest;
 };
 
 export const RunAgentTestAgentTestsAgentAgentUuidRunPostRequest$zodSchema:
   z.ZodType<RunAgentTestAgentTestsAgentAgentUuidRunPostRequest> = z.object({
-    agent_uuid: z.string().describe("The agent to test."),
+    agent_uuid: z.string().describe("Agent to test"),
     body: RunTestRequest$zodSchema,
     xAPIKey: z.string().nullable().optional(),
-    xOrgUUID: z.string().nullable().optional(),
   });
 
 export type RunAgentTestAgentTestsAgentAgentUuidRunPostResponse =

package/src/models/runtestsbatchagenttestsrunpostop.ts

@@ -18,7 +18,6 @@ import {
 
 export type RunTestsBatchAgentTestsRunPostRequest = {
   xAPIKey?: string | null | undefined;
-  xOrgUUID?: string | null | undefined;
   body?: BatchRunRequest | null | undefined;
 };
 
@@ -27,7 +26,6 @@ export const RunTestsBatchAgentTestsRunPostRequest$zodSchema: z.ZodType<
 > = z.object({
   body: BatchRunRequest$zodSchema.nullable().optional(),
   xAPIKey: z.string().nullable().optional(),
-  xOrgUUID: z.string().nullable().optional(),
 });
 
 export type RunTestsBatchAgentTestsRunPostResponse =

package/src/models/startevaluatorrunannotationtaskstaskuuidevaluatorrunspostop.ts

@@ -0,0 +1,42 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  EvaluatorRunLaunchResponse,
+  EvaluatorRunLaunchResponse$zodSchema,
+} from "./evaluatorrunlaunchresponse.js";
+import {
+  EvaluatorRunStartRequest,
+  EvaluatorRunStartRequest$zodSchema,
+} from "./evaluatorrunstartrequest.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+
+export type StartEvaluatorRunAnnotationTasksTaskUuidEvaluatorRunsPostRequest = {
+  task_uuid: string;
+  xAPIKey?: string | null | undefined;
+  body: EvaluatorRunStartRequest;
+};
+
+export const StartEvaluatorRunAnnotationTasksTaskUuidEvaluatorRunsPostRequest$zodSchema:
+  z.ZodType<StartEvaluatorRunAnnotationTasksTaskUuidEvaluatorRunsPostRequest> =
+    z.object({
+      body: EvaluatorRunStartRequest$zodSchema,
+      task_uuid: z.string().describe("Annotation task to act on"),
+      xAPIKey: z.string().nullable().optional(),
+    });
+
+export type StartEvaluatorRunAnnotationTasksTaskUuidEvaluatorRunsPostResponse =
+  | EvaluatorRunLaunchResponse
+  | HTTPValidationError;
+
+export const StartEvaluatorRunAnnotationTasksTaskUuidEvaluatorRunsPostResponse$zodSchema:
+  z.ZodType<StartEvaluatorRunAnnotationTasksTaskUuidEvaluatorRunsPostResponse> =
+    z.union([
+      EvaluatorRunLaunchResponse$zodSchema,
+      HTTPValidationError$zodSchema,
+    ]);

package/src/models/taskagreementannotationtaskstaskuuidagreementgetop.ts

@@ -0,0 +1,66 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import {
+  TaskAgreementResponse,
+  TaskAgreementResponse$zodSchema,
+} from "./taskagreementresponse.js";
+
+/**
+ * How to bucket points in the trend series
+ */
+export const Bucket = {
+  Week: "week",
+  Month: "month",
+  Year: "year",
+} as const;
+/**
+ * How to bucket points in the trend series
+ */
+export type Bucket = ClosedEnum<typeof Bucket>;
+
+export const Bucket$zodSchema = z.enum([
+  "week",
+  "month",
+  "year",
+]).describe("How to bucket points in the trend series");
+
+export type TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest = {
+  task_uuid: string;
+  bucket?: Bucket | undefined;
+  days?: number | undefined;
+  xAPIKey?: string | null | undefined;
+};
+
+export const TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest$zodSchema:
+  z.ZodType<TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest> = z.object(
+    {
+      bucket: Bucket$zodSchema.default("week").describe(
+        "How to bucket points in the trend series",
+      ),
+      days: z.int().default(90).describe(
+        "Trailing window in days for the trend series",
+      ),
+      task_uuid: z.string().describe("Annotation task to act on"),
+      xAPIKey: z.string().nullable().optional(),
+    },
+  );
+
+export type TaskAgreementAnnotationTasksTaskUuidAgreementGetResponse =
+  | TaskAgreementResponse
+  | HTTPValidationError;
+
+export const TaskAgreementAnnotationTasksTaskUuidAgreementGetResponse$zodSchema:
+  z.ZodType<TaskAgreementAnnotationTasksTaskUuidAgreementGetResponse> = z.union(
+    [
+      TaskAgreementResponse$zodSchema,
+      HTTPValidationError$zodSchema,
+    ],
+  );

package/src/models/taskagreementresponse.ts

@@ -0,0 +1,52 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { collectExtraKeys as collectExtraKeys$ } from "../lib/schemas.js";
+import {
+  HumanAgreementBlock,
+  HumanAgreementBlock$zodSchema,
+  HumanAgreementBlock$zodSchemaOutbound,
+} from "./humanagreementblock.js";
+
+export type TaskAgreementResponse = {
+  task_id: string;
+  bucket: string;
+  days: number;
+  human_human: HumanAgreementBlock;
+  evaluators: Array<{ [k: string]: any }>;
+  AdditionalProperties?: { [k: string]: any };
+};
+
+export const TaskAgreementResponse$zodSchema: z.ZodType<TaskAgreementResponse> =
+  collectExtraKeys$(
+    z.object({
+      bucket: z.string().describe("Time bucket used for the trend series"),
+      days: z.int().describe("Length of the trailing window in days"),
+      evaluators: z.array(z.record(z.string(), z.any())).describe(
+        "Agreement between each evaluator and the annotators, one entry per linked evaluator",
+      ),
+      human_human: HumanAgreementBlock$zodSchema,
+      task_id: z.string().describe("ID of the task"),
+    }).catchall(z.any()),
+    "AdditionalProperties",
+    true,
+  );
+
+export const TaskAgreementResponse$zodSchemaOutbound: z.ZodType<any> = z.object(
+  {
+    AdditionalProperties: z.record(z.string(), z.any()).optional(),
+    bucket: z.string(),
+    days: z.int(),
+    evaluators: z.array(z.record(z.string(), z.any())),
+    human_human: HumanAgreementBlock$zodSchemaOutbound,
+    task_id: z.string(),
+  },
+).transform((v) => {
+  const { AdditionalProperties, ...rest } = v;
+  return {
+    ...rest,
+    ...AdditionalProperties,
+  };
+});

package/src/models/tasksummaryannotationtaskstaskuuidsummarygetop.ts

@@ -0,0 +1,76 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import {
+  TaskSummaryResponse,
+  TaskSummaryResponse$zodSchema,
+} from "./tasksummaryresponse.js";
+
+/**
+ * Sort direction
+ */
+export const Order = {
+  Asc: "asc",
+  Desc: "desc",
+} as const;
+/**
+ * Sort direction
+ */
+export type Order = ClosedEnum<typeof Order>;
+
+export const Order$zodSchema = z.enum([
+  "asc",
+  "desc",
+]).describe("Sort direction");
+
+export type TaskSummaryAnnotationTasksTaskUuidSummaryGetRequest = {
+  task_uuid: string;
+  item_id?: string | null | undefined;
+  live_only?: boolean | undefined;
+  q?: string | null | undefined;
+  sort_by?: string | undefined;
+  order?: Order | undefined;
+  limit?: number | undefined;
+  offset?: number | undefined;
+  xAPIKey?: string | null | undefined;
+};
+
+export const TaskSummaryAnnotationTasksTaskUuidSummaryGetRequest$zodSchema:
+  z.ZodType<TaskSummaryAnnotationTasksTaskUuidSummaryGetRequest> = z.object({
+    item_id: z.string().describe(
+      "Filter rows to a single item. The full task-wide annotator union is still returned in `annotators`",
+    ).nullable().optional(),
+    limit: z.int().default(50).describe("Maximum number of items to return"),
+    live_only: z.boolean().default(false).describe(
+      "When true, emit only one row for each (item, evaluator) pair using the evaluator's live version. Versions other than the live one that have runs are excluded",
+    ),
+    offset: z.int().default(0).describe(
+      "Number of items to skip before returning results",
+    ),
+    order: Order$zodSchema.default("desc").describe("Sort direction"),
+    q: z.string().describe(
+      "Case-insensitive substring search on `payload.name`. Blank is a no-op",
+    ).nullable().optional(),
+    sort_by: z.string().default("created_at").describe(
+      "Sort key for the results",
+    ),
+    task_uuid: z.string().describe("Annotation task to act on"),
+    xAPIKey: z.string().nullable().optional(),
+  });
+
+export type TaskSummaryAnnotationTasksTaskUuidSummaryGetResponse =
+  | TaskSummaryResponse
+  | HTTPValidationError;
+
+export const TaskSummaryAnnotationTasksTaskUuidSummaryGetResponse$zodSchema:
+  z.ZodType<TaskSummaryAnnotationTasksTaskUuidSummaryGetResponse> = z.union([
+    TaskSummaryResponse$zodSchema,
+    HTTPValidationError$zodSchema,
+  ]);

package/src/models/tasksummaryresponse.ts

@@ -0,0 +1,82 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { collectExtraKeys as collectExtraKeys$ } from "../lib/schemas.js";
+import { ClosedEnum } from "../types/enums.js";
+import { PaginationMeta, PaginationMeta$zodSchema } from "./paginationmeta.js";
+
+/**
+ * Type of the task
+ */
+export const TaskType = {
+  Stt: "stt",
+  Tts: "tts",
+  Llm: "llm",
+  LlmGeneral: "llm-general",
+  Conversation: "conversation",
+} as const;
+/**
+ * Type of the task
+ */
+export type TaskType = ClosedEnum<typeof TaskType>;
+
+export const TaskType$zodSchema = z.enum([
+  "stt",
+  "tts",
+  "llm",
+  "llm-general",
+  "conversation",
+]).describe("Type of the task");
+
+export type TaskSummaryResponse = {
+  task_id: string;
+  task_type: TaskType;
+  evaluators: Array<{ [k: string]: any }>;
+  annotators: Array<{ [k: string]: any }>;
+  rows: Array<{ [k: string]: any }>;
+  item_comments: { [k: string]: any };
+  pagination: PaginationMeta;
+  AdditionalProperties?: { [k: string]: any };
+};
+
+export const TaskSummaryResponse$zodSchema: z.ZodType<TaskSummaryResponse> =
+  collectExtraKeys$(
+    z.object({
+      annotators: z.array(z.record(z.string(), z.any())).describe(
+        "The annotators who labelled or commented on items in the task, stable across pages",
+      ),
+      evaluators: z.array(z.record(z.string(), z.any())).describe(
+        "The task's evaluators, stable across pages, each with its live rubric",
+      ),
+      item_comments: z.record(z.string(), z.any()).describe(
+        "Annotator comments, keyed by item ID",
+      ),
+      pagination: PaginationMeta$zodSchema,
+      rows: z.array(z.record(z.string(), z.any())).describe(
+        "One row per item, evaluator, and version on this page, with the evaluator's value and each annotator's label",
+      ),
+      task_id: z.string().describe("ID of the task"),
+      task_type: TaskType$zodSchema.describe("Type of the task"),
+    }).catchall(z.any()),
+    "AdditionalProperties",
+    true,
+  );
+
+export const TaskSummaryResponse$zodSchemaOutbound: z.ZodType<any> = z.object({
+  AdditionalProperties: z.record(z.string(), z.any()).optional(),
+  annotators: z.array(z.record(z.string(), z.any())),
+  evaluators: z.array(z.record(z.string(), z.any())),
+  item_comments: z.record(z.string(), z.any()),
+  pagination: PaginationMeta$zodSchema,
+  rows: z.array(z.record(z.string(), z.any())),
+  task_id: z.string(),
+  task_type: TaskType$zodSchema,
+}).transform((v) => {
+  const { AdditionalProperties, ...rest } = v;
+  return {
+    ...rest,
+    ...AdditionalProperties,
+  };
+});

package/src/models/testcaseresult.ts

@@ -19,25 +19,23 @@ export type TestCaseResult = {
 };
 
 export const TestCaseResult$zodSchema: z.ZodType<TestCaseResult> = z.object({
-  cost: z.number().nullable().optional().describe("Cost of this case in USD"),
+  cost: z.number().nullable().optional().describe("Cost of this case (USD)"),
   judge_results: z.array(JudgeResult$zodSchema).nullable().optional().describe(
-    "One verdict per evaluator, for response and conversation tests. Null for tool-call tests or rows still running",
+    "One verdict for each evaluator",
   ),
   latency_ms: z.number().nullable().optional().describe(
-    "Response latency in milliseconds for the agent under test",
+    "How long the agent took to respond, in milliseconds",
   ),
   name: z.string().nullable().optional().describe("Name of the test"),
   output: TestOutput$zodSchema.nullable().optional().describe(
-    "The agent's output for this case. Present only when done",
-  ),
-  passed: z.boolean().nullable().optional().describe(
-    "Whether the case passed. Present only when done",
+    "The agent's output for this case",
   ),
+  passed: z.boolean().nullable().optional().describe("Whether the case passed"),
   reasoning: z.string().nullable().optional().describe(
-    "Judge reasoning, or the tool-call diff for tool-call tests",
+    "The judge's reasoning, or the tool-call diff for a tool-call test",
   ),
   test_case: z.record(z.string(), z.any()).nullable().optional().describe(
-    "The input test case definition. Present only when done",
+    "The test case definition that was run",
   ),
   test_case_id: z.string().nullable().optional().describe(
     "ID of the test case within the run",

package/src/models/testcreate.ts

@@ -39,7 +39,7 @@ export const TestCreateType$zodSchema = z.enum([
   "tool_call",
   "conversation",
 ]).describe(
-  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation",
+  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
 );
 
 export type TestCreate = {
@@ -51,7 +51,7 @@ export type TestCreate = {
 
 export const TestCreate$zodSchema: z.ZodType<TestCreate> = z.object({
   config: z.record(z.string(), z.any()).nullable().optional().describe(
-    "The calibrate test config. Three top-level keys.\n\n- `history` (array, required): the conversation up to the agent's turn. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `evaluation` (object, required): `{type, ...}`, where `type` matches the test's `type` (below).\n- `settings` (object, optional): e.g. `{\"language\": \"en\"}`.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nOmit to create the test with no config and fill it in later via update.",
+    "The calibrate test config. Three top-level keys.\n\n- `history`: the required conversation up to the agent's turn. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `evaluation`: the required `{type, ...}`, where `type` matches the test's `type` below.\n- `settings`: an optional object, e.g. `{\"language\": \"en\"}`.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nOmit to create the test with no config and fill it in later via update",
   ),
   evaluators: z.array(RoutersTestsEvaluatorRef$zodSchema).nullable().optional()
     .describe(
@@ -59,6 +59,6 @@ export const TestCreate$zodSchema: z.ZodType<TestCreate> = z.object({
     ),
   name: z.string().describe("Name of the test, unique within the workspace"),
   type: TestCreateType$zodSchema.describe(
-    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation",
+    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
   ),
 });