@dalmia/calibrate-mcp 0.0.58 → 0.0.60

package/bin/mcp-server.js

@@ -52761,9 +52761,9 @@ var init_config = __esm(() => {
   SDK_METADATA = {
     language: "typescript",
     openapiDocVersion: "0.1.0",
-    sdkVersion: "0.0.58",
+    sdkVersion: "0.0.60",
     genVersion: "2.915.1",
-    userAgent: "speakeasy-sdk/mcp-typescript 0.0.58 2.915.1 0.1.0 @dalmia/calibrate-mcp"
+    userAgent: "speakeasy-sdk/mcp-typescript 0.0.60 2.915.1 0.1.0 @dalmia/calibrate-mcp"
   };
 });
 
@@ -55613,7 +55613,7 @@ var init_benchmarkrequest = __esm(() => {
   init_zod();
   BenchmarkRequest$zodSchema = object({
     models: array(string2()).describe("Model names to benchmark"),
-    parallel_models: boolean2().default(true).describe("Whether to run the models at the same time. Set false to run them one after another"),
+    parallel_models: boolean2().default(true).describe("How to run the models. `true` runs several at a time, `false` runs each one only after the one before it has finished. Use `false` to keep the load on your own agent down"),
     test_uuids: array(string2()).nullable().optional().describe("A subset of the agent's linked tests to benchmark. Each ID must be linked to the agent. Omit to run all linked tests")
   });
 });
@@ -55987,10 +55987,12 @@ var init_modelresult = __esm(() => {
     message: string2().describe("Status or result message for this model"),
     model: string2().describe("Model name these results are for"),
     passed: int().nullable().optional().describe("Number of test cases that passed"),
+    stopped_early: boolean2().default(false).describe("Whether this model's run stopped before starting every test case, after too many failed in a row"),
     success: boolean2().nullable().optional().describe("Whether this model's run succeeded"),
     test_results: array(TestCaseResult$zodSchema).nullable().optional().describe("Results for each test case for this model"),
     total_tests: int().nullable().optional().describe("Total test cases for this model"),
-    total_tokens: record(string2(), any()).nullable().optional().describe("Aggregated token usage as `{mean, min, max, count}`")
+    total_tokens: record(string2(), any()).nullable().optional().describe("Aggregated token usage as `{mean, min, max, count}`"),
+    unanswered_tests: int().nullable().optional().describe("Number of test cases that produced no answer because the agent or the judge could not be reached, which makes the pass rate an unfair measure of the agent")
   });
 });
 
@@ -56036,14 +56038,16 @@ var init_benchmarkstatusresponse = __esm(() => {
   init_testrunevaluator();
   BenchmarkStatusResponse$zodSchema = object({
     aborted: boolean2().default(false).describe("Whether a user stopped this run before it finished. The results collected up to that point are kept, and test cases that never ran are counted neither as passed nor as failed"),
-    error: boolean2().default(false).describe("True if the run failed"),
+    error: string2().nullable().optional().describe("Why the run could not be carried out, when it failed before producing any result"),
     evaluators: array(TestRunEvaluator$zodSchema).nullable().optional().describe("The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`"),
     is_public: boolean2().default(false).describe("Whether the run is shared publicly"),
     leaderboard_summary: array(record(string2(), any())).nullable().optional().describe("Leaderboard comparing the models, one row per model. Columns vary by benchmark: a `model` column plus pass/fail counts, latency, cost, and one score column per evaluator, keyed by evaluator name"),
     model_results: array(ModelResult$zodSchema).nullable().optional().describe("Results for each model"),
     name: string2().describe("Name of the run. A run nobody has renamed shows its number instead, such as `Run 1` for a test run or `Benchmark 1` for a benchmark"),
+    parallel_models: boolean2().nullable().optional().describe("How the models were run. `true` means several ran at a time, `false` means each one ran only after the one before it had finished"),
     share_token: string2().nullable().optional().describe("Token for building the public share URL"),
     status: TaskStatus$zodSchema,
+    stopped_early: boolean2().default(false).describe("Whether any model's run stopped before starting every test case, after too many failed in a row"),
     task_id: string2().describe("Benchmark run job ID"),
     test_uuids: array(string2()).nullable().optional().describe("IDs of the tests this benchmark executed, in run order")
   });
@@ -56205,7 +56209,7 @@ var init_testrunstatusresponse = __esm(() => {
   TestRunStatusResponse$zodSchema = object({
     aborted: boolean2().default(false).describe("Whether a user stopped this run before it finished. The results collected up to that point are kept, and test cases that never ran are counted neither as passed nor as failed"),
     cost: record(string2(), any()).nullable().optional().describe("Aggregated cost as `{mean, min, max, count}` (USD)"),
-    error: boolean2().default(false).describe("True if the run failed"),
+    error: string2().nullable().optional().describe("Why the run could not be carried out, when it failed before producing any result"),
     evaluator_summary: array(record(string2(), any())).nullable().optional().describe("Totals for each evaluator over the whole run, matching the shape a benchmark reports for each model. Only evaluators that returned a verdict appear"),
     evaluators: array(TestRunEvaluator$zodSchema).nullable().optional().describe("The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`"),
     failed: int().nullable().optional().describe("Number of test cases that failed"),
@@ -56931,6 +56935,7 @@ var init_agenttestrunlistitem = __esm(() => {
     results: array(TestRunCaseSummary$zodSchema).nullable().optional().describe("Flat pass/fail summary for each test case (fetch the run detail for full results)"),
     share_token: string2().nullable().optional().describe("Token for building the public share URL"),
     status: TaskStatus$zodSchema,
+    stopped_early: boolean2().default(false).describe("Whether the run stopped before starting every test case, after too many failed in a row"),
     total_tests: int().nullable().optional().describe("Total number of test cases"),
     total_tokens: record(string2(), any()).nullable().optional().describe("Aggregated token usage as `{mean, min, max, count}`"),
     type: AgentTestRunListItemType$zodSchema.describe("What kind of run this is:\n- `llm-unit-test`: a single run of the agent's tests\n- `llm-benchmark`: a multi-model comparison"),
@@ -61559,7 +61564,7 @@ hits its trace limit.
 function createMCPServer(deps) {
   const server = new McpServer({
     name: "CalibrateMcp",
-    version: "0.0.58"
+    version: "0.0.60"
   });
   const getClient = deps.getSDK || (() => new CalibrateMcpCore({
     security: deps.security,
@@ -62848,7 +62853,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
         <h1>Instructions</h1>
         <p>One-click installation for Claude Desktop users</p>
         <div class="instruction-item">
-          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.58/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
+          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.60/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
             \uD83D\uDCE5 Download MCP Bundle
           </a>
         </div>
@@ -65732,7 +65737,7 @@ var routes = buildRouteMap({
 var app = buildApplication(routes, {
   name: "mcp",
   versionInfo: {
-    currentVersion: "0.0.58"
+    currentVersion: "0.0.60"
   }
 });
 run(app, process3.argv.slice(2), buildContext(process3));
@@ -65740,5 +65745,5 @@ export {
   app
 };
 
-//# debugId=208D24C92691569964756E2164756E21
+//# debugId=BD138050FC63ADA764756E2164756E21
 //# sourceMappingURL=mcp-server.js.map