@dalmia/calibrate-mcp 0.0.52 → 0.0.54

package/src/models/testcaseresult.ts

@@ -3,12 +3,30 @@
  */
 
 import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
 import { JudgeResult, JudgeResult$zodSchema } from "./judgeresult.js";
 import { TestOutput, TestOutput$zodSchema } from "./testoutput.js";
 
+export const TestType = {
+  Response: "response",
+  ToolCall: "tool_call",
+  Conversation: "conversation",
+  General: "general",
+} as const;
+export type TestType = ClosedEnum<typeof TestType>;
+
+export const TestType$zodSchema = z.enum([
+  "response",
+  "tool_call",
+  "conversation",
+  "general",
+]);
+
 export type TestCaseResult = {
   test_case_id?: string | null | undefined;
   name?: string | null | undefined;
+  test_uuid?: string | null | undefined;
+  test_type?: TestType | null | undefined;
   passed?: boolean | null | undefined;
   reasoning?: string | null | undefined;
   output?: TestOutput | null | undefined;
@@ -49,6 +67,12 @@ export const TestCaseResult$zodSchema: z.ZodType<TestCaseResult> = z.object({
   test_case_id: z.string().nullable().optional().describe(
     "ID of the test case within the run",
   ),
+  test_type: TestType$zodSchema.nullable().optional().describe(
+    "What the test asks of the agent, which decides how a reader draws the case",
+  ),
+  test_uuid: z.string().nullable().optional().describe(
+    "ID of the test this case ran, which is what you pass to read the case on its own",
+  ),
   unanswered: z.boolean().default(false).describe(
     "Whether this case produced no answer because the agent or the judge could not be reached, in which case `reasoning` carries the error and `passed` is not a verdict on the agent",
   ),

package/src/models/testrunstatusresponse.ts

@@ -12,6 +12,7 @@ import {
 
 export type TestRunStatusResponse = {
   task_id: string;
+  name: string;
   status: TaskStatus;
   test_uuids?: Array<string> | null | undefined;
   total_tests?: number | null | undefined;
@@ -21,6 +22,7 @@ export type TestRunStatusResponse = {
   cost?: { [k: string]: any } | null | undefined;
   total_tokens?: { [k: string]: any } | null | undefined;
   evaluators?: Array<TestRunEvaluator> | null | undefined;
+  evaluator_summary?: Array<{ [k: string]: any }> | null | undefined;
   results?: Array<TestCaseResult> | null | undefined;
   unanswered_tests?: number | null | undefined;
   stopped_early?: boolean | undefined;
@@ -39,6 +41,10 @@ export const TestRunStatusResponse$zodSchema: z.ZodType<TestRunStatusResponse> =
       "Aggregated cost as `{mean, min, max, count}` (USD)",
     ),
     error: z.boolean().default(false).describe("True if the run failed"),
+    evaluator_summary: z.array(z.record(z.string(), z.any())).nullable()
+      .optional().describe(
+        "Totals for each evaluator over the whole run, matching the shape a benchmark reports for each model. Only evaluators that returned a verdict appear",
+      ),
     evaluators: z.array(TestRunEvaluator$zodSchema).nullable().optional()
       .describe(
         "The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`",
@@ -52,6 +58,9 @@ export const TestRunStatusResponse$zodSchema: z.ZodType<TestRunStatusResponse> =
     latency_ms: z.record(z.string(), z.any()).nullable().optional().describe(
       "Aggregated response latency in milliseconds, as `{p50, p95, p99, count}`",
     ),
+    name: z.string().describe(
+      "Name of the run. A run nobody has renamed shows its number instead, such as `Run 1` for a test run or `Benchmark 1` for a benchmark",
+    ),
     passed: z.int().nullable().optional().describe(
       "Number of test cases that passed",
     ),

package/src/tool-names.ts

@@ -80,6 +80,10 @@ export const toolNames: Array<{ name: string; description: string }>= [
     "name": "get-agent-test-run",
     "description": "Poll the status and results of an agent test run by task_id. Returns\nqueued, in_progress, completed, failed, or aborted status. When\ncompleted, includes per-test-case pass/fail results and evaluator\njudgments. Use the task_id from run-agent-tests or\nrun-agent-tests-batch.\n"
   },
+  {
+    "name": "get-agent-test-case-result",
+    "description": "Fetch the full result of one test case in a run, including the\nconversation, the agent's output and each evaluator's reasoning. Pass\nthe model query param for a benchmark, which runs every test once per\nmodel. Use with mode=summary on get-agent-test-run, which returns\nlight rows without this detail.\n"
+  },
   {
     "name": "run-agent-benchmark",
     "description": "Start a multi-model benchmark over an agent's linked tests as a background job. Optionally restrict to a subset of linked tests. Returns a task_id to poll with get-agent-benchmark.\n"