@dalmia/calibrate-mcp 0.0.53 → 0.0.55

package/esm/models/testrunstatusresponse.d.ts

@@ -20,6 +20,9 @@ export type TestRunStatusResponse = {
         [k: string]: any;
     } | null | undefined;
     evaluators?: Array<TestRunEvaluator> | null | undefined;
+    evaluator_summary?: Array<{
+        [k: string]: any;
+    }> | null | undefined;
     results?: Array<TestCaseResult> | null | undefined;
     unanswered_tests?: number | null | undefined;
     stopped_early?: boolean | undefined;

package/esm/models/testrunstatusresponse.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"testrunstatusresponse.d.ts","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,cAAc,EAA4B,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,OAAO,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnD,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7C,aAAa,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACpC,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAmDzE,CAAC"}
+{"version":3,"file":"testrunstatusresponse.d.ts","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,cAAc,EAA4B,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,iBAAiB,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnE,OAAO,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnD,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7C,aAAa,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACpC,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAuDzE,CAAC"}

package/esm/models/testrunstatusresponse.js

@@ -9,6 +9,8 @@ export const TestRunStatusResponse$zodSchema = z.object({
     aborted: z.boolean().default(false).describe("Whether a user stopped this run before it finished. The results collected up to that point are kept, and test cases that never ran are counted neither as passed nor as failed"),
     cost: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated cost as `{mean, min, max, count}` (USD)"),
     error: z.boolean().default(false).describe("True if the run failed"),
+    evaluator_summary: z.array(z.record(z.string(), z.any())).nullable()
+        .optional().describe("Totals for each evaluator over the whole run, matching the shape a benchmark reports for each model. Only evaluators that returned a verdict appear"),
     evaluators: z.array(TestRunEvaluator$zodSchema).nullable().optional()
         .describe("The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`"),
     failed: z.int().nullable().optional().describe("Number of test cases that failed"),

package/esm/models/testrunstatusresponse.js.map

@@ -1 +1 @@
-{"version":3,"file":"testrunstatusresponse.js","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAuB/B,MAAM,CAAC,MAAM,+BAA+B,GAC1C,CAAC,CAAC,MAAM,CAAC;IACP,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC1C,gLAAgL,CACjL;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,0EAA0E,CAC3E;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,qIAAqI,CACtI;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,4BAA4B,CAC7B;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,yFAAyF,CAC1F;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,kDAAkD,CACnD;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;CACF,CAAC,CAAC"}
+{"version":3,"file":"testrunstatusresponse.js","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAwB/B,MAAM,CAAC,MAAM,+BAA+B,GAC1C,CAAC,CAAC,MAAM,CAAC;IACP,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC1C,gLAAgL,CACjL;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;SACjE,QAAQ,EAAE,CAAC,QAAQ,CAClB,qJAAqJ,CACtJ;IACH,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,0EAA0E,CAC3E;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,qIAAqI,CACtI;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,4BAA4B,CAC7B;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,yFAAyF,CAC1F;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,kDAAkD,CACnD;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;CACF,CAAC,CAAC"}

package/esm/tool-names.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tool-names.d.ts","sourceRoot":"","sources":["../src/tool-names.ts"],"names":[],"mappings":"AACA,eAAO,MAAM,SAAS,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAqKlE,CAAC"}
+{"version":3,"file":"tool-names.d.ts","sourceRoot":"","sources":["../src/tool-names.ts"],"names":[],"mappings":"AACA,eAAO,MAAM,SAAS,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAyKlE,CAAC"}

package/esm/tool-names.js

@@ -80,6 +80,10 @@ export const toolNames = [
         "name": "get-agent-test-run",
         "description": "Poll the status and results of an agent test run by task_id. Returns\nqueued, in_progress, completed, failed, or aborted status. When\ncompleted, includes per-test-case pass/fail results and evaluator\njudgments. Use the task_id from run-agent-tests or\nrun-agent-tests-batch.\n"
     },
+    {
+        "name": "get-agent-test-case-result",
+        "description": "Fetch the full result of one test case in a run, including the\nconversation, the agent's output and each evaluator's reasoning. Pass\nthe model query param for a benchmark, which runs every test once per\nmodel. Use with mode=summary on get-agent-test-run, which returns\nlight rows without this detail.\n"
+    },
     {
         "name": "run-agent-benchmark",
         "description": "Start a multi-model benchmark over an agent's linked tests as a background job. Optionally restrict to a subset of linked tests. Returns a task_id to poll with get-agent-benchmark.\n"

package/esm/tool-names.js.map

@@ -1 +1 @@
-{"version":3,"file":"tool-names.js","sourceRoot":"","sources":["../src/tool-names.ts"],"names":[],"mappings":"AAAA,+BAA+B;AAC/B,MAAM,CAAC,MAAM,SAAS,GAAgD;IACpE;QACE,MAAM,EAAE,yBAAyB;QACjC,aAAa,EAAE,sNAAsN;KACtO;IACD;QACE,MAAM,EAAE,qBAAqB;QAC7B,aAAa,EAAE,oQAAoQ;KACpR;IACD;QACE,MAAM,EAAE,cAAc;QACtB,aAAa,EAAE,kMAAkM;KAClN;IACD;QACE,MAAM,EAAE,aAAa;QACrB,aAAa,EAAE,iPAAiP;KACjQ;IACD;QACE,MAAM,EAAE,WAAW;QACnB,aAAa,EAAE,6KAA6K;KAC7L;IACD;QACE,MAAM,EAAE,cAAc;QACtB,aAAa,EAAE,kNAAkN;KAClO;IACD;QACE,MAAM,EAAE,uBAAuB;QAC/B,aAAa,EAAE,6IAA6I;KAC7J;IACD;QACE,MAAM,EAAE,0BAA0B;QAClC,aAAa,EAAE,kLAAkL;KAClM;IACD;QACE,MAAM,EAAE,mBAAmB;QAC3B,aAAa,EAAE,+KAA+K;KAC/L;IACD;QACE,MAAM,EAAE,aAAa;QACrB,aAAa,EAAE,+LAA+L;KAC/M;IACD;QACE,MAAM,EAAE,YAAY;QACpB,aAAa,EAAE,kKAAkK;KAClL;IACD;QACE,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,0KAA0K;KAC1L;IACD;QACE,MAAM,EAAE,aAAa;QACrB,aAAa,EAAE,yMAAyM;KACzN;IACD;QACE,MAAM,EAAE,qBAAqB;QAC7B,aAAa,EAAE,2JAA2J;KAC3K;IACD;QACE,MAAM,EAAE,kBAAkB;QAC1B,aAAa,EAAE,sCAAsC;KACtD;IACD;QACE,MAAM,EAAE,sBAAsB;QAC9B,aAAa,EAAE,wGAAwG;KACxH;IACD;QACE,MAAM,EAAE,yBAAyB;QACjC,aAAa,EAAE,kWAAkW;KAClX;IACD;QACE,MAAM,EAAE,iBAAiB;QACzB,aAAa,EAAE,oSAAoS;KACpT;IACD;QACE,MAAM,EAAE,uBAAuB;QAC/B,aAAa,EAAE,qUAAqU;KACrV;IACD;QACE,MAAM,EAAE,oBAAoB;QAC5B,aAAa,EAAE,wRAAwR;KACxS;IACD;QACE,MAAM,EAAE,qBAAqB;QAC7B,aAAa,EAAE,wLAAwL;KACxM;IACD;QACE,MAAM,EAAE,qBAAqB;QAC7B,aAAa,EAAE,uFAAuF;KACvG;IACD;QACE,MAAM,EAAE,kBAAkB;QAC1B,aAAa,EAAE,4HAA4H;KAC5I;IACD;QACE,MAAM,EAAE,iBAAiB;QACzB,aAAa,EAAE,oHAAoH;KACpI;IACD;QACE,MAAM,EAAE,eAAe;QACvB,aAAa,EAAE,oDAAoD;KACpE;IACD;QACE,MAAM,EAAE,0BAA0B;QAClC,aAAa,EAAE,iIAAiI;KACjJ;IACD;QACE,MAAM,EAAE,wBAAwB;QAChC,aAAa,EAAE,0GAA0G;KAC1H;IACD;QACE,MAAM,EAAE,uBAAuB;QAC/B,aAAa,EAAE,4DAA4D;KAC5E;IACD;QACE,MAAM,EAAE,qBAAqB;QAC7B,aAAa,EAAE,2EAA2E;KAC3F;IACD;QACE,MAAM,EAAE,qBAAqB;QAC7B,aAAa,EAAE,sTAAsT;KACtU;IACD;QACE,MAAM,EAAE,sBAAsB;QAC9B,aAAa,EAAE,kFAAkF;KAClG;IACD;QACE,MAAM,EAAE,yBAAyB;QACjC,aAAa,EAAE,oDAAoD;KACpE;IACD;QACE,MAAM,EAAE,uBAAuB;QAC/B,aAAa,EAAE,qJAAqJ;KACrK;IACD;QACE,MAAM,EAAE,2BAA2B;QACnC,aAAa,EAAE,4IAA4I;KAC5J;IACD;QACE,MAAM,EAAE,8BAA8B;QACtC,aAAa,EAAE,2EAA2E;KAC3F;IACD;QACE,MAAM,EAAE,0BAA0B;QAClC,aAAa,EAAE,gGAAgG;KAChH;IACD;QACE,MAAM,EAAE,wBAAwB;QAChC,aAAa,EAAE,iGAAiG;KACjH;IACD;QACE,MAAM,EAAE,kBAAkB;QAC1B,aAAa,EAAE,+EAA+E;KAC/F;IACD;QACE,MAAM,EAAE,iBAAiB;QACzB,aAAa,EAAE,oEAAoE;KACpF;IACD;QACE,MAAM,EAAE,kBAAkB;QAC1B,aAAa,EAAE,0CAA0C;KAC1D;IACD;QACE,MAAM,EAAE,cAAc;QACtB,aAAa,EAAE,+SAA+S;KAC/T;CACF,CAAC"}
+{"version":3,"file":"tool-names.js","sourceRoot":"","sources":["../src/tool-names.ts"],"names":[],"mappings":"AAAA,+BAA+B;AAC/B,MAAM,CAAC,MAAM,SAAS,GAAgD;IACpE;QACE,MAAM,EAAE,yBAAyB;QACjC,aAAa,EAAE,sNAAsN;KACtO;IACD;QACE,MAAM,EAAE,qBAAqB;QAC7B,aAAa,EAAE,oQAAoQ;KACpR;IACD;QACE,MAAM,EAAE,cAAc;QACtB,aAAa,EAAE,kMAAkM;KAClN;IACD;QACE,MAAM,EAAE,aAAa;QACrB,aAAa,EAAE,iPAAiP;KACjQ;IACD;QACE,MAAM,EAAE,WAAW;QACnB,aAAa,EAAE,6KAA6K;KAC7L;IACD;QACE,MAAM,EAAE,cAAc;QACtB,aAAa,EAAE,kNAAkN;KAClO;IACD;QACE,MAAM,EAAE,uBAAuB;QAC/B,aAAa,EAAE,6IAA6I;KAC7J;IACD;QACE,MAAM,EAAE,0BAA0B;QAClC,aAAa,EAAE,kLAAkL;KAClM;IACD;QACE,MAAM,EAAE,mBAAmB;QAC3B,aAAa,EAAE,+KAA+K;KAC/L;IACD;QACE,MAAM,EAAE,aAAa;QACrB,aAAa,EAAE,+LAA+L;KAC/M;IACD;QACE,MAAM,EAAE,YAAY;QACpB,aAAa,EAAE,kKAAkK;KAClL;IACD;QACE,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,0KAA0K;KAC1L;IACD;QACE,MAAM,EAAE,aAAa;QACrB,aAAa,EAAE,yMAAyM;KACzN;IACD;QACE,MAAM,EAAE,qBAAqB;QAC7B,aAAa,EAAE,2JAA2J;KAC3K;IACD;QACE,MAAM,EAAE,kBAAkB;QAC1B,aAAa,EAAE,sCAAsC;KACtD;IACD;QACE,MAAM,EAAE,sBAAsB;QAC9B,aAAa,EAAE,wGAAwG;KACxH;IACD;QACE,MAAM,EAAE,yBAAyB;QACjC,aAAa,EAAE,kWAAkW;KAClX;IACD;QACE,MAAM,EAAE,iBAAiB;QACzB,aAAa,EAAE,oSAAoS;KACpT;IACD;QACE,MAAM,EAAE,uBAAuB;QAC/B,aAAa,EAAE,qUAAqU;KACrV;IACD;QACE,MAAM,EAAE,oBAAoB;QAC5B,aAAa,EAAE,wRAAwR;KACxS;IACD;QACE,MAAM,EAAE,4BAA4B;QACpC,aAAa,EAAE,oTAAoT;KACpU;IACD;QACE,MAAM,EAAE,qBAAqB;QAC7B,aAAa,EAAE,wLAAwL;KACxM;IACD;QACE,MAAM,EAAE,qBAAqB;QAC7B,aAAa,EAAE,uFAAuF;KACvG;IACD;QACE,MAAM,EAAE,kBAAkB;QAC1B,aAAa,EAAE,4HAA4H;KAC5I;IACD;QACE,MAAM,EAAE,iBAAiB;QACzB,aAAa,EAAE,oHAAoH;KACpI;IACD;QACE,MAAM,EAAE,eAAe;QACvB,aAAa,EAAE,oDAAoD;KACpE;IACD;QACE,MAAM,EAAE,0BAA0B;QAClC,aAAa,EAAE,iIAAiI;KACjJ;IACD;QACE,MAAM,EAAE,wBAAwB;QAChC,aAAa,EAAE,0GAA0G;KAC1H;IACD;QACE,MAAM,EAAE,uBAAuB;QAC/B,aAAa,EAAE,4DAA4D;KAC5E;IACD;QACE,MAAM,EAAE,qBAAqB;QAC7B,aAAa,EAAE,2EAA2E;KAC3F;IACD;QACE,MAAM,EAAE,qBAAqB;QAC7B,aAAa,EAAE,sTAAsT;KACtU;IACD;QACE,MAAM,EAAE,sBAAsB;QAC9B,aAAa,EAAE,kFAAkF;KAClG;IACD;QACE,MAAM,EAAE,yBAAyB;QACjC,aAAa,EAAE,oDAAoD;KACpE;IACD;QACE,MAAM,EAAE,uBAAuB;QAC/B,aAAa,EAAE,qJAAqJ;KACrK;IACD;QACE,MAAM,EAAE,2BAA2B;QACnC,aAAa,EAAE,4IAA4I;KAC5J;IACD;QACE,MAAM,EAAE,8BAA8B;QACtC,aAAa,EAAE,2EAA2E;KAC3F;IACD;QACE,MAAM,EAAE,0BAA0B;QAClC,aAAa,EAAE,gGAAgG;KAChH;IACD;QACE,MAAM,EAAE,wBAAwB;QAChC,aAAa,EAAE,iGAAiG;KACjH;IACD;QACE,MAAM,EAAE,kBAAkB;QAC1B,aAAa,EAAE,+EAA+E;KAC/F;IACD;QACE,MAAM,EAAE,iBAAiB;QACzB,aAAa,EAAE,oEAAoE;KACpF;IACD;QACE,MAAM,EAAE,kBAAkB;QAC1B,aAAa,EAAE,0CAA0C;KAC1D;IACD;QACE,MAAM,EAAE,cAAc;QACtB,aAAa,EAAE,+SAA+S;KAC/T;CACF,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dalmia/calibrate-mcp",
-  "version": "0.0.53",
+  "version": "0.0.55",
   "author": "Speakeasy",
   "type": "module",
   "sideEffects": false,

package/src/funcs/agentTestsGetBenchmark.ts

@@ -98,6 +98,7 @@ async function $do(
   );
   const query$ = encodeFormQuery({
     "compact": payload$.compact,
+    "mode": payload$.mode,
     "only_failed": payload$.only_failed,
   });
 

package/src/funcs/agentTestsGetRun.ts

@@ -98,6 +98,7 @@ async function $do(
   );
   const query$ = encodeFormQuery({
     "compact": payload$.compact,
+    "mode": payload$.mode,
     "only_failed": payload$.only_failed,
   });
 

package/src/funcs/agentTestsGetRunCase.ts

@@ -0,0 +1,167 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { CalibrateMcpCore } from "../core.js";
+import { encodeFormQuery, encodeSimple } from "../lib/encodings.js";
+import { compactMap } from "../lib/primitives.js";
+import { safeParse } from "../lib/schemas.js";
+import { RequestOptions } from "../lib/sdks.js";
+import { extractSecurity, resolveGlobalSecurity } from "../lib/security.js";
+import { pathToFunc } from "../lib/url.js";
+import { APIError } from "../models/errors/apierror.js";
+import {
+  ConnectionError,
+  InvalidRequestError,
+  RequestAbortedError,
+  RequestTimeoutError,
+  UnexpectedClientError,
+} from "../models/errors/httpclienterrors.js";
+import { SDKValidationError } from "../models/errors/sdkvalidationerror.js";
+import {
+  GetAgentTestCaseResultAgentTestsRunTaskIdResultsTestUuidGetRequest,
+  GetAgentTestCaseResultAgentTestsRunTaskIdResultsTestUuidGetRequest$zodSchema,
+} from "../models/getagenttestcaseresultagenttestsruntaskidresultstestuuidgetop.js";
+import { APICall, APIPromise } from "../types/async.js";
+import { Result } from "../types/fp.js";
+
+/**
+ * Get test case result
+ *
+ * @remarks
+ * Get the full result of one test case in a run
+ */
+export function agentTestsGetRunCase(
+  client$: CalibrateMcpCore,
+  request: GetAgentTestCaseResultAgentTestsRunTaskIdResultsTestUuidGetRequest,
+  options?: RequestOptions,
+): APIPromise<
+  Result<
+    Response,
+    | APIError
+    | SDKValidationError
+    | UnexpectedClientError
+    | InvalidRequestError
+    | RequestAbortedError
+    | RequestTimeoutError
+    | ConnectionError
+  >
+> {
+  return new APIPromise($do(
+    client$,
+    request,
+    options,
+  ));
+}
+
+async function $do(
+  client$: CalibrateMcpCore,
+  request: GetAgentTestCaseResultAgentTestsRunTaskIdResultsTestUuidGetRequest,
+  options?: RequestOptions,
+): Promise<
+  [
+    Result<
+      Response,
+      | APIError
+      | SDKValidationError
+      | UnexpectedClientError
+      | InvalidRequestError
+      | RequestAbortedError
+      | RequestTimeoutError
+      | ConnectionError
+    >,
+    APICall,
+  ]
+> {
+  const parsed$ = safeParse(
+    request,
+    (value$) =>
+      GetAgentTestCaseResultAgentTestsRunTaskIdResultsTestUuidGetRequest$zodSchema
+        .parse(value$),
+    "Input validation failed",
+  );
+  if (!parsed$.ok) {
+    return [parsed$, { status: "invalid" }];
+  }
+  const payload$ = parsed$.value;
+  const body$ = null;
+
+  const pathParams$ = {
+    task_id: encodeSimple("task_id", payload$.task_id, {
+      explode: false,
+      charEncoding: "percent",
+    }),
+    test_uuid: encodeSimple("test_uuid", payload$.test_uuid, {
+      explode: false,
+      charEncoding: "percent",
+    }),
+  };
+  const path$ = pathToFunc("/agent-tests/run/{task_id}/results/{test_uuid}")(
+    pathParams$,
+  );
+  const query$ = encodeFormQuery({
+    "model": payload$.model,
+  });
+
+  const headers$ = new Headers(compactMap({
+    Accept: "application/json",
+    "X-API-Key": encodeSimple("X-API-Key", payload$.xAPIKey, {
+      explode: false,
+      charEncoding: "none",
+    }),
+  }));
+  const securityInput = await extractSecurity(client$._options.security);
+  const requestSecurity = resolveGlobalSecurity(securityInput);
+
+  const context = {
+    options: client$._options,
+    baseURL: options?.serverURL ?? client$._baseURL ?? "",
+    operationID:
+      "get_agent_test_case_result_agent_tests_run__task_id__results__test_uuid__get",
+    oAuth2Scopes: null,
+    resolvedSecurity: requestSecurity,
+    securitySource: client$._options.security,
+    retryConfig: options?.retries
+      || client$._options.retryConfig
+      || { strategy: "none" },
+    retryCodes: options?.retryCodes || [
+      "429",
+      "500",
+      "502",
+      "503",
+      "504",
+    ],
+  };
+
+  const requestRes = client$._createRequest(context, {
+    security: requestSecurity,
+    method: "GET",
+    baseURL: options?.serverURL,
+    path: path$,
+    headers: headers$,
+    query: query$,
+    body: body$,
+    userAgent: client$._options.userAgent,
+    timeoutMs: options?.timeoutMs || client$._options.timeoutMs
+      || -1,
+  }, options);
+  if (!requestRes.ok) {
+    return [requestRes, { status: "invalid" }];
+  }
+  const req$ = requestRes.value;
+
+  const doResult = await client$._do(req$, {
+    context,
+    errorCodes: [],
+    retryConfig: context.retryConfig,
+    retryCodes: context.retryCodes,
+  });
+  if (!doResult.ok) {
+    return [doResult, { status: "request-error", request: req$ }];
+  }
+  return [doResult, {
+    status: "complete",
+    "request": req$,
+    response: doResult.value,
+  }];
+}

package/src/landing-page.ts

@@ -930,7 +930,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
         <h1>Instructions</h1>
         <p>One-click installation for Claude Desktop users</p>
         <div class="instruction-item">
-          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.53/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
+          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.55/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
             📥 Download MCP Bundle
           </a>
         </div>

package/src/lib/config.ts

@@ -65,8 +65,8 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
 export const SDK_METADATA = {
   language: "typescript",
   openapiDocVersion: "0.1.0",
-  sdkVersion: "0.0.53",
+  sdkVersion: "0.0.55",
   genVersion: "2.915.1",
   userAgent:
-    "speakeasy-sdk/mcp-typescript 0.0.53 2.915.1 0.1.0 @dalmia/calibrate-mcp",
+    "speakeasy-sdk/mcp-typescript 0.0.55 2.915.1 0.1.0 @dalmia/calibrate-mcp",
 } as const;

package/src/mcp-server/mcp-server.ts

@@ -21,7 +21,7 @@ const routes = buildRouteMap({
 export const app = buildApplication(routes, {
   name: "mcp",
   versionInfo: {
-    currentVersion: "0.0.53",
+    currentVersion: "0.0.55",
   },
 });
 

package/src/mcp-server/server.ts

@@ -29,6 +29,7 @@ import { tool$agentTestsBenchmark } from "./tools/agentTestsBenchmark.js";
 import { tool$agentTestsBulkUnlink } from "./tools/agentTestsBulkUnlink.js";
 import { tool$agentTestsGetBenchmark } from "./tools/agentTestsGetBenchmark.js";
 import { tool$agentTestsGetRun } from "./tools/agentTestsGetRun.js";
+import { tool$agentTestsGetRunCase } from "./tools/agentTestsGetRunCase.js";
 import { tool$agentTestsLink } from "./tools/agentTestsLink.js";
 import { tool$agentTestsListForAgent } from "./tools/agentTestsListForAgent.js";
 import { tool$agentTestsListRunsForAgent } from "./tools/agentTestsListRunsForAgent.js";
@@ -72,7 +73,7 @@ export function createMCPServer(deps: {
 }) {
   const server = new McpServer({
     name: "CalibrateMcp",
-    version: "0.0.53",
+    version: "0.0.55",
   });
 
   const getClient = deps.getSDK || (() =>
@@ -137,6 +138,7 @@ export function createMCPServer(deps: {
   tool(tool$agentTestsRun);
   tool(tool$agentTestsRunBatch);
   tool(tool$agentTestsGetRun);
+  tool(tool$agentTestsGetRunCase);
   tool(tool$agentTestsBenchmark);
   tool(tool$agentTestsGetBenchmark);
   tool(tool$evaluatorsCreate);

package/src/mcp-server/tools/agentTestsGetRunCase.ts

@@ -0,0 +1,47 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import { agentTestsGetRunCase } from "../../funcs/agentTestsGetRunCase.js";
+import { GetAgentTestCaseResultAgentTestsRunTaskIdResultsTestUuidGetRequest$zodSchema } from "../../models/getagenttestcaseresultagenttestsruntaskidresultstestuuidgetop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request:
+    GetAgentTestCaseResultAgentTestsRunTaskIdResultsTestUuidGetRequest$zodSchema,
+};
+
+export const tool$agentTestsGetRunCase: ToolDefinition<typeof args> = {
+  name: "get-agent-test-case-result",
+  description: `Fetch the full result of one test case in a run, including the
+conversation, the agent's output and each evaluator's reasoning. Pass
+the model query param for a benchmark, which runs every test once per
+model. Use with mode=summary on get-agent-test-run, which returns
+light rows without this detail.
+`,
+  scopes: ["read"],
+  annotations: {
+    "title": "Get agent test case result",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": true,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await agentTestsGetRunCase(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/models/getagenttestcaseresultagenttestsruntaskidresultstestuuidgetop.ts

@@ -0,0 +1,44 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import { TestCaseResult, TestCaseResult$zodSchema } from "./testcaseresult.js";
+
+export type GetAgentTestCaseResultAgentTestsRunTaskIdResultsTestUuidGetRequest =
+  {
+    task_id: string;
+    test_uuid: string;
+    model?: string | null | undefined;
+    xAPIKey?: string | null | undefined;
+  };
+
+export const GetAgentTestCaseResultAgentTestsRunTaskIdResultsTestUuidGetRequest$zodSchema:
+  z.ZodType<
+    GetAgentTestCaseResultAgentTestsRunTaskIdResultsTestUuidGetRequest
+  > = z.object({
+    model: z.string().describe(
+      "Which model's answer to read. Required for a benchmark, which runs every test once per model",
+    ).nullable().optional(),
+    task_id: z.string().describe("Test run or benchmark the case was run in"),
+    test_uuid: z.string().describe(
+      "The test whose result to read, as `test_uuid` on the case",
+    ),
+    xAPIKey: z.string().nullable().optional(),
+  });
+
+export type GetAgentTestCaseResultAgentTestsRunTaskIdResultsTestUuidGetResponse =
+  | TestCaseResult
+  | HTTPValidationError;
+
+export const GetAgentTestCaseResultAgentTestsRunTaskIdResultsTestUuidGetResponse$zodSchema:
+  z.ZodType<
+    GetAgentTestCaseResultAgentTestsRunTaskIdResultsTestUuidGetResponse
+  > = z.union([
+    TestCaseResult$zodSchema,
+    HTTPValidationError$zodSchema,
+  ]);

package/src/models/getagenttestrunstatusagenttestsruntaskidgetop.ts

@@ -3,6 +3,7 @@
  */
 
 import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
 import {
   HTTPValidationError,
   HTTPValidationError$zodSchema,
@@ -12,9 +13,33 @@ import {
   TestRunStatusResponse$zodSchema,
 } from "./testrunstatusresponse.js";
 
+/**
+ * How much of each test case to return. `full` returns every field of every case. `summary` returns one light row per case, with its ID, name, verdict and short reason, leaving out the conversation, the agent's output and the evaluator verdicts. Read those one case at a time from `GET /agent-tests/run/{task_id}/results/{test_uuid}`
+ */
+export const GetAgentTestRunStatusAgentTestsRunTaskIdGetMode = {
+  Full: "full",
+  Summary: "summary",
+} as const;
+/**
+ * How much of each test case to return. `full` returns every field of every case. `summary` returns one light row per case, with its ID, name, verdict and short reason, leaving out the conversation, the agent's output and the evaluator verdicts. Read those one case at a time from `GET /agent-tests/run/{task_id}/results/{test_uuid}`
+ */
+export type GetAgentTestRunStatusAgentTestsRunTaskIdGetMode = ClosedEnum<
+  typeof GetAgentTestRunStatusAgentTestsRunTaskIdGetMode
+>;
+
+export const GetAgentTestRunStatusAgentTestsRunTaskIdGetMode$zodSchema = z.enum(
+  [
+    "full",
+    "summary",
+  ],
+).describe(
+  "How much of each test case to return. `full` returns every field of every case. `summary` returns one light row per case, with its ID, name, verdict and short reason, leaving out the conversation, the agent's output and the evaluator verdicts. Read those one case at a time from `GET /agent-tests/run/{task_id}/results/{test_uuid}`",
+);
+
 export type GetAgentTestRunStatusAgentTestsRunTaskIdGetRequest = {
   task_id: string;
   only_failed?: boolean | undefined;
+  mode?: GetAgentTestRunStatusAgentTestsRunTaskIdGetMode | undefined;
   compact?: boolean | undefined;
   xAPIKey?: string | null | undefined;
 };
@@ -24,6 +49,11 @@ export const GetAgentTestRunStatusAgentTestsRunTaskIdGetRequest$zodSchema:
     compact: z.boolean().default(false).describe(
       "Return a compact response that omits heavy detail fields (`results.output`, `results.test_case`, `results.judge_results`, `results.reasoning`, `evaluators.output_config`), keeping only the lightweight decision fields. Omit for full detail",
     ),
+    mode: GetAgentTestRunStatusAgentTestsRunTaskIdGetMode$zodSchema.default(
+      "full",
+    ).describe(
+      "How much of each test case to return. `full` returns every field of every case. `summary` returns one light row per case, with its ID, name, verdict and short reason, leaving out the conversation, the agent's output and the evaluator verdicts. Read those one case at a time from `GET /agent-tests/run/{task_id}/results/{test_uuid}`",
+    ),
     only_failed: z.boolean().default(false).describe(
       "Return only failing test cases. Omit to return every case",
     ),

package/src/models/getbenchmarkstatusagenttestsbenchmarktaskidgetop.ts

@@ -3,6 +3,7 @@
  */
 
 import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
 import {
   BenchmarkStatusResponse,
   BenchmarkStatusResponse$zodSchema,
@@ -12,9 +13,32 @@ import {
   HTTPValidationError$zodSchema,
 } from "./httpvalidationerror.js";
 
+/**
+ * How much of each test case to return. `full` returns every field of every case. `summary` returns one light row per case, with its ID, name, verdict and short reason, leaving out the conversation, the agent's output and the evaluator verdicts. Read those one case at a time from `GET /agent-tests/run/{task_id}/results/{test_uuid}`
+ */
+export const GetBenchmarkStatusAgentTestsBenchmarkTaskIdGetMode = {
+  Full: "full",
+  Summary: "summary",
+} as const;
+/**
+ * How much of each test case to return. `full` returns every field of every case. `summary` returns one light row per case, with its ID, name, verdict and short reason, leaving out the conversation, the agent's output and the evaluator verdicts. Read those one case at a time from `GET /agent-tests/run/{task_id}/results/{test_uuid}`
+ */
+export type GetBenchmarkStatusAgentTestsBenchmarkTaskIdGetMode = ClosedEnum<
+  typeof GetBenchmarkStatusAgentTestsBenchmarkTaskIdGetMode
+>;
+
+export const GetBenchmarkStatusAgentTestsBenchmarkTaskIdGetMode$zodSchema = z
+  .enum([
+    "full",
+    "summary",
+  ]).describe(
+    "How much of each test case to return. `full` returns every field of every case. `summary` returns one light row per case, with its ID, name, verdict and short reason, leaving out the conversation, the agent's output and the evaluator verdicts. Read those one case at a time from `GET /agent-tests/run/{task_id}/results/{test_uuid}`",
+  );
+
 export type GetBenchmarkStatusAgentTestsBenchmarkTaskIdGetRequest = {
   task_id: string;
   only_failed?: boolean | undefined;
+  mode?: GetBenchmarkStatusAgentTestsBenchmarkTaskIdGetMode | undefined;
   compact?: boolean | undefined;
   xAPIKey?: string | null | undefined;
 };
@@ -24,6 +48,11 @@ export const GetBenchmarkStatusAgentTestsBenchmarkTaskIdGetRequest$zodSchema:
     compact: z.boolean().default(false).describe(
       "Return a compact response that omits heavy detail fields (`model_results.test_results`, `evaluators.output_config`), keeping only the lightweight decision fields. Omit for full detail",
     ),
+    mode: GetBenchmarkStatusAgentTestsBenchmarkTaskIdGetMode$zodSchema.default(
+      "full",
+    ).describe(
+      "How much of each test case to return. `full` returns every field of every case. `summary` returns one light row per case, with its ID, name, verdict and short reason, leaving out the conversation, the agent's output and the evaluator verdicts. Read those one case at a time from `GET /agent-tests/run/{task_id}/results/{test_uuid}`",
+    ),
     only_failed: z.boolean().default(false).describe(
       "Return only failing test cases for each model. Omit to return every case",
     ),

package/src/models/testcaseresult.ts

@@ -3,12 +3,30 @@
  */
 
 import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
 import { JudgeResult, JudgeResult$zodSchema } from "./judgeresult.js";
 import { TestOutput, TestOutput$zodSchema } from "./testoutput.js";
 
+export const TestType = {
+  Response: "response",
+  ToolCall: "tool_call",
+  Conversation: "conversation",
+  General: "general",
+} as const;
+export type TestType = ClosedEnum<typeof TestType>;
+
+export const TestType$zodSchema = z.enum([
+  "response",
+  "tool_call",
+  "conversation",
+  "general",
+]);
+
 export type TestCaseResult = {
   test_case_id?: string | null | undefined;
   name?: string | null | undefined;
+  test_uuid?: string | null | undefined;
+  test_type?: TestType | null | undefined;
   passed?: boolean | null | undefined;
   reasoning?: string | null | undefined;
   output?: TestOutput | null | undefined;
@@ -49,6 +67,12 @@ export const TestCaseResult$zodSchema: z.ZodType<TestCaseResult> = z.object({
   test_case_id: z.string().nullable().optional().describe(
     "ID of the test case within the run",
   ),
+  test_type: TestType$zodSchema.nullable().optional().describe(
+    "What the test asks of the agent, which decides how a reader draws the case",
+  ),
+  test_uuid: z.string().nullable().optional().describe(
+    "ID of the test this case ran, which is what you pass to read the case on its own",
+  ),
   unanswered: z.boolean().default(false).describe(
     "Whether this case produced no answer because the agent or the judge could not be reached, in which case `reasoning` carries the error and `passed` is not a verdict on the agent",
   ),

package/src/models/testrunstatusresponse.ts

@@ -22,6 +22,7 @@ export type TestRunStatusResponse = {
   cost?: { [k: string]: any } | null | undefined;
   total_tokens?: { [k: string]: any } | null | undefined;
   evaluators?: Array<TestRunEvaluator> | null | undefined;
+  evaluator_summary?: Array<{ [k: string]: any }> | null | undefined;
   results?: Array<TestCaseResult> | null | undefined;
   unanswered_tests?: number | null | undefined;
   stopped_early?: boolean | undefined;
@@ -40,6 +41,10 @@ export const TestRunStatusResponse$zodSchema: z.ZodType<TestRunStatusResponse> =
       "Aggregated cost as `{mean, min, max, count}` (USD)",
     ),
     error: z.boolean().default(false).describe("True if the run failed"),
+    evaluator_summary: z.array(z.record(z.string(), z.any())).nullable()
+      .optional().describe(
+        "Totals for each evaluator over the whole run, matching the shape a benchmark reports for each model. Only evaluators that returned a verdict appear",
+      ),
     evaluators: z.array(TestRunEvaluator$zodSchema).nullable().optional()
       .describe(
         "The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`",

package/src/tool-names.ts

@@ -80,6 +80,10 @@ export const toolNames: Array<{ name: string; description: string }>= [
     "name": "get-agent-test-run",
     "description": "Poll the status and results of an agent test run by task_id. Returns\nqueued, in_progress, completed, failed, or aborted status. When\ncompleted, includes per-test-case pass/fail results and evaluator\njudgments. Use the task_id from run-agent-tests or\nrun-agent-tests-batch.\n"
   },
+  {
+    "name": "get-agent-test-case-result",
+    "description": "Fetch the full result of one test case in a run, including the\nconversation, the agent's output and each evaluator's reasoning. Pass\nthe model query param for a benchmark, which runs every test once per\nmodel. Use with mode=summary on get-agent-test-run, which returns\nlight rows without this detail.\n"
+  },
   {
     "name": "run-agent-benchmark",
     "description": "Start a multi-model benchmark over an agent's linked tests as a background job. Optionally restrict to a subset of linked tests. Returns a task_id to poll with get-agent-benchmark.\n"