@avallon-labs/mcp 50.4.0 → 51.0.0-staging.1135

package/dist/index.js

@@ -46,5 +46,5 @@ function setupFetch() {
 
 // src/index.ts
 setupFetch();
-await import("./server-NVU3VZZX.js");
+await import("./server-XM676NBD.js");
 //# sourceMappingURL=index.js.map

package/dist/{server-NVU3VZZX.js → server-XM676NBD.js}

@@ -1453,6 +1453,44 @@ var getEvalRun = async (evalRunId, options) => {
   const data = body ? JSON.parse(body) : {};
   return { data, status: res.status, headers: res.headers };
 };
+var getCreateEvalBenchmarkUrl = (evalId) => {
+  return `/v1/evals/${evalId}/benchmarks`;
+};
+var createEvalBenchmark = async (evalId, createEvalBenchmarkBody, options) => {
+  const getHeaders = (h) => {
+    if (!h) return {};
+    if (h instanceof Headers) return Object.fromEntries(h.entries());
+    if (Array.isArray(h)) return Object.fromEntries(h);
+    return h;
+  };
+  const res = await fetch(
+    getCreateEvalBenchmarkUrl(evalId),
+    {
+      ...options,
+      method: "POST",
+      headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) },
+      body: JSON.stringify(createEvalBenchmarkBody)
+    }
+  );
+  const body = [204, 205, 304].includes(res.status) ? null : await res.text();
+  const data = body ? JSON.parse(body) : {};
+  return { data, status: res.status, headers: res.headers };
+};
+var getGetBenchmarkUrl = (benchmarkId) => {
+  return `/v1/benchmarks/${benchmarkId}`;
+};
+var getBenchmark = async (benchmarkId, options) => {
+  const res = await fetch(
+    getGetBenchmarkUrl(benchmarkId),
+    {
+      ...options,
+      method: "GET"
+    }
+  );
+  const body = [204, 205, 304].includes(res.status) ? null : await res.text();
+  const data = body ? JSON.parse(body) : {};
+  return { data, status: res.status, headers: res.headers };
+};
 var getCreateExtractorUrl = () => {
   return `/v1/extractors`;
 };
@@ -4975,6 +5013,52 @@ var getEvalRunHandler = async (args, options) => {
     structuredContent: res.data
   };
 };
+var createEvalBenchmarkHandler = async (args, options) => {
+  const res = await createEvalBenchmark(args.pathParams.evalId, args.bodyParams, options);
+  if (res.status >= 400) {
+    return {
+      content: [
+        {
+          type: "text",
+          text: JSON.stringify(res.data ?? null)
+        }
+      ],
+      isError: true
+    };
+  }
+  return {
+    content: [
+      {
+        type: "text",
+        text: JSON.stringify(res.data ?? null)
+      }
+    ],
+    structuredContent: res.data
+  };
+};
+var getBenchmarkHandler = async (args, options) => {
+  const res = await getBenchmark(args.pathParams.benchmarkId, options);
+  if (res.status >= 400) {
+    return {
+      content: [
+        {
+          type: "text",
+          text: JSON.stringify(res.data ?? null)
+        }
+      ],
+      isError: true
+    };
+  }
+  return {
+    content: [
+      {
+        type: "text",
+        text: JSON.stringify(res.data ?? null)
+      }
+    ],
+    structuredContent: res.data
+  };
+};
 var createExtractorHandler = async (args, options) => {
   const res = await createExtractor(args.bodyParams, options);
   if (res.status >= 400) {
@@ -8815,13 +8899,18 @@ var CreateWorkerEvalFromRunParams = zod.object({
   "workerRunId": zod.string().uuid()
 });
 var createWorkerEvalFromRunBodyNameMax = 200;
-var createWorkerEvalFromRunBodyGraderCriterionMax = 1e4;
+var createWorkerEvalFromRunBodyGraderOneAssertionsItemMax = 2e3;
+var createWorkerEvalFromRunBodyGraderOneAssertionsMax = 50;
+var createWorkerEvalFromRunBodyGraderTwoCriterionMax = 1e4;
 var CreateWorkerEvalFromRunBody = zod.object({
   "name": zod.string().min(1).max(createWorkerEvalFromRunBodyNameMax),
-  "grader": zod.object({
+  "grader": zod.union([zod.object({
     "type": zod.literal("prompt"),
-    "criterion": zod.string().min(1).max(createWorkerEvalFromRunBodyGraderCriterionMax)
-  })
+    "assertions": zod.array(zod.string().min(1).max(createWorkerEvalFromRunBodyGraderOneAssertionsItemMax)).min(1).max(createWorkerEvalFromRunBodyGraderOneAssertionsMax)
+  }), zod.object({
+    "type": zod.literal("prompt"),
+    "criterion": zod.string().min(1).max(createWorkerEvalFromRunBodyGraderTwoCriterionMax)
+  })])
 });
 var CreateWorkerEvalFromRunResponse = zod.object({
   "data": zod.object({
@@ -8855,7 +8944,8 @@ var CreateWorkerEvalFromRunResponse = zod.object({
     })),
     "grader": zod.object({
       "type": zod.literal("prompt"),
-      "criterion": zod.string()
+      "assertions": zod.array(zod.string()),
+      "criterion": zod.string().describe("Deprecated: assertions joined by newlines. Use assertions.")
     }),
     "created_at": zod.string().datetime({ "offset": true })
   }),
@@ -8870,6 +8960,26 @@ var RunEvalBody = zod.object({
 });
 var runEvalResponseDataTargetVersionMin = -9007199254740991;
 var runEvalResponseDataTargetVersionMax = 9007199254740991;
+var runEvalResponseDataGradingOneSummaryTotalMin = -9007199254740991;
+var runEvalResponseDataGradingOneSummaryTotalMax = 9007199254740991;
+var runEvalResponseDataGradingOneSummaryPassedMin = -9007199254740991;
+var runEvalResponseDataGradingOneSummaryPassedMax = 9007199254740991;
+var runEvalResponseDataGradingOneSummaryFailedMin = -9007199254740991;
+var runEvalResponseDataGradingOneSummaryFailedMax = 9007199254740991;
+var runEvalResponseDataMetricsOneToolCallsMin = -9007199254740991;
+var runEvalResponseDataMetricsOneToolCallsMax = 9007199254740991;
+var runEvalResponseDataMetricsOneFixtureHealthExactMin = -9007199254740991;
+var runEvalResponseDataMetricsOneFixtureHealthExactMax = 9007199254740991;
+var runEvalResponseDataMetricsOneFixtureHealthPartialMin = -9007199254740991;
+var runEvalResponseDataMetricsOneFixtureHealthPartialMax = 9007199254740991;
+var runEvalResponseDataMetricsOneFixtureHealthFifoMin = -9007199254740991;
+var runEvalResponseDataMetricsOneFixtureHealthFifoMax = 9007199254740991;
+var runEvalResponseDataMetricsOneFixtureHealthDefaultMin = -9007199254740991;
+var runEvalResponseDataMetricsOneFixtureHealthDefaultMax = 9007199254740991;
+var runEvalResponseDataMetricsOneFixtureHealthUnavailableMin = -9007199254740991;
+var runEvalResponseDataMetricsOneFixtureHealthUnavailableMax = 9007199254740991;
+var runEvalResponseDataRunNumberOneMin = -9007199254740991;
+var runEvalResponseDataRunNumberOneMax = 9007199254740991;
 var RunEvalResponse = zod.object({
   "data": zod.object({
     "id": zod.string().uuid(),
@@ -8882,6 +8992,34 @@ var RunEvalResponse = zod.object({
     "judge_model": zod.union([zod.string(), zod.null()]),
     "judge_prompt_version": zod.union([zod.string(), zod.null()]),
     "tool_trace": zod.union([zod.unknown(), zod.null()]),
+    "grading": zod.union([zod.object({
+      "expectations": zod.array(zod.object({
+        "text": zod.string(),
+        "passed": zod.boolean(),
+        "evidence": zod.string()
+      })),
+      "summary": zod.object({
+        "total": zod.number().int().min(runEvalResponseDataGradingOneSummaryTotalMin).max(runEvalResponseDataGradingOneSummaryTotalMax),
+        "passed": zod.number().int().min(runEvalResponseDataGradingOneSummaryPassedMin).max(runEvalResponseDataGradingOneSummaryPassedMax),
+        "failed": zod.number().int().min(runEvalResponseDataGradingOneSummaryFailedMin).max(runEvalResponseDataGradingOneSummaryFailedMax),
+        "pass_rate": zod.number()
+      })
+    }), zod.null()]),
+    "metrics": zod.union([zod.object({
+      "duration_seconds": zod.union([zod.number(), zod.null()]),
+      "tool_calls": zod.number().int().min(runEvalResponseDataMetricsOneToolCallsMin).max(runEvalResponseDataMetricsOneToolCallsMax),
+      "fixture_health": zod.object({
+        "exact": zod.number().int().min(runEvalResponseDataMetricsOneFixtureHealthExactMin).max(runEvalResponseDataMetricsOneFixtureHealthExactMax),
+        "partial": zod.number().int().min(runEvalResponseDataMetricsOneFixtureHealthPartialMin).max(runEvalResponseDataMetricsOneFixtureHealthPartialMax),
+        "fifo": zod.number().int().min(runEvalResponseDataMetricsOneFixtureHealthFifoMin).max(runEvalResponseDataMetricsOneFixtureHealthFifoMax),
+        "default": zod.number().int().min(runEvalResponseDataMetricsOneFixtureHealthDefaultMin).max(runEvalResponseDataMetricsOneFixtureHealthDefaultMax),
+        "unavailable": zod.number().int().min(runEvalResponseDataMetricsOneFixtureHealthUnavailableMin).max(runEvalResponseDataMetricsOneFixtureHealthUnavailableMax),
+        "matched_rate": zod.union([zod.number(), zod.null()])
+      })
+    }), zod.null()]),
+    "benchmark_id": zod.union([zod.string().uuid(), zod.null()]),
+    "configuration": zod.union([zod.enum(["candidate", "baseline"]), zod.null()]),
+    "run_number": zod.union([zod.number().int().min(runEvalResponseDataRunNumberOneMin).max(runEvalResponseDataRunNumberOneMax), zod.null()]),
     "created_at": zod.string().datetime({ "offset": true }),
     "started_at": zod.union([zod.string().datetime({ "offset": true }), zod.null()]),
     "completed_at": zod.union([zod.string().datetime({ "offset": true }), zod.null()])
@@ -8893,6 +9031,26 @@ var GetEvalRunParams = zod.object({
 });
 var getEvalRunResponseDataTargetVersionMin = -9007199254740991;
 var getEvalRunResponseDataTargetVersionMax = 9007199254740991;
+var getEvalRunResponseDataGradingOneSummaryTotalMin = -9007199254740991;
+var getEvalRunResponseDataGradingOneSummaryTotalMax = 9007199254740991;
+var getEvalRunResponseDataGradingOneSummaryPassedMin = -9007199254740991;
+var getEvalRunResponseDataGradingOneSummaryPassedMax = 9007199254740991;
+var getEvalRunResponseDataGradingOneSummaryFailedMin = -9007199254740991;
+var getEvalRunResponseDataGradingOneSummaryFailedMax = 9007199254740991;
+var getEvalRunResponseDataMetricsOneToolCallsMin = -9007199254740991;
+var getEvalRunResponseDataMetricsOneToolCallsMax = 9007199254740991;
+var getEvalRunResponseDataMetricsOneFixtureHealthExactMin = -9007199254740991;
+var getEvalRunResponseDataMetricsOneFixtureHealthExactMax = 9007199254740991;
+var getEvalRunResponseDataMetricsOneFixtureHealthPartialMin = -9007199254740991;
+var getEvalRunResponseDataMetricsOneFixtureHealthPartialMax = 9007199254740991;
+var getEvalRunResponseDataMetricsOneFixtureHealthFifoMin = -9007199254740991;
+var getEvalRunResponseDataMetricsOneFixtureHealthFifoMax = 9007199254740991;
+var getEvalRunResponseDataMetricsOneFixtureHealthDefaultMin = -9007199254740991;
+var getEvalRunResponseDataMetricsOneFixtureHealthDefaultMax = 9007199254740991;
+var getEvalRunResponseDataMetricsOneFixtureHealthUnavailableMin = -9007199254740991;
+var getEvalRunResponseDataMetricsOneFixtureHealthUnavailableMax = 9007199254740991;
+var getEvalRunResponseDataRunNumberOneMin = -9007199254740991;
+var getEvalRunResponseDataRunNumberOneMax = 9007199254740991;
 var GetEvalRunResponse = zod.object({
   "data": zod.object({
     "id": zod.string().uuid(),
@@ -8905,12 +9063,341 @@ var GetEvalRunResponse = zod.object({
     "judge_model": zod.union([zod.string(), zod.null()]),
     "judge_prompt_version": zod.union([zod.string(), zod.null()]),
     "tool_trace": zod.union([zod.unknown(), zod.null()]),
+    "grading": zod.union([zod.object({
+      "expectations": zod.array(zod.object({
+        "text": zod.string(),
+        "passed": zod.boolean(),
+        "evidence": zod.string()
+      })),
+      "summary": zod.object({
+        "total": zod.number().int().min(getEvalRunResponseDataGradingOneSummaryTotalMin).max(getEvalRunResponseDataGradingOneSummaryTotalMax),
+        "passed": zod.number().int().min(getEvalRunResponseDataGradingOneSummaryPassedMin).max(getEvalRunResponseDataGradingOneSummaryPassedMax),
+        "failed": zod.number().int().min(getEvalRunResponseDataGradingOneSummaryFailedMin).max(getEvalRunResponseDataGradingOneSummaryFailedMax),
+        "pass_rate": zod.number()
+      })
+    }), zod.null()]),
+    "metrics": zod.union([zod.object({
+      "duration_seconds": zod.union([zod.number(), zod.null()]),
+      "tool_calls": zod.number().int().min(getEvalRunResponseDataMetricsOneToolCallsMin).max(getEvalRunResponseDataMetricsOneToolCallsMax),
+      "fixture_health": zod.object({
+        "exact": zod.number().int().min(getEvalRunResponseDataMetricsOneFixtureHealthExactMin).max(getEvalRunResponseDataMetricsOneFixtureHealthExactMax),
+        "partial": zod.number().int().min(getEvalRunResponseDataMetricsOneFixtureHealthPartialMin).max(getEvalRunResponseDataMetricsOneFixtureHealthPartialMax),
+        "fifo": zod.number().int().min(getEvalRunResponseDataMetricsOneFixtureHealthFifoMin).max(getEvalRunResponseDataMetricsOneFixtureHealthFifoMax),
+        "default": zod.number().int().min(getEvalRunResponseDataMetricsOneFixtureHealthDefaultMin).max(getEvalRunResponseDataMetricsOneFixtureHealthDefaultMax),
+        "unavailable": zod.number().int().min(getEvalRunResponseDataMetricsOneFixtureHealthUnavailableMin).max(getEvalRunResponseDataMetricsOneFixtureHealthUnavailableMax),
+        "matched_rate": zod.union([zod.number(), zod.null()])
+      })
+    }), zod.null()]),
+    "benchmark_id": zod.union([zod.string().uuid(), zod.null()]),
+    "configuration": zod.union([zod.enum(["candidate", "baseline"]), zod.null()]),
+    "run_number": zod.union([zod.number().int().min(getEvalRunResponseDataRunNumberOneMin).max(getEvalRunResponseDataRunNumberOneMax), zod.null()]),
     "created_at": zod.string().datetime({ "offset": true }),
     "started_at": zod.union([zod.string().datetime({ "offset": true }), zod.null()]),
     "completed_at": zod.union([zod.string().datetime({ "offset": true }), zod.null()])
   }),
   "message": zod.string()
 });
+var CreateEvalBenchmarkParams = zod.object({
+  "evalId": zod.string().uuid()
+});
+var createEvalBenchmarkBodyCandidateVersionMax = 9007199254740991;
+var createEvalBenchmarkBodyBaselineVersionMax = 9007199254740991;
+var createEvalBenchmarkBodyRunsPerConfigurationDefault = 3;
+var createEvalBenchmarkBodyRunsPerConfigurationMax = 5;
+var CreateEvalBenchmarkBody = zod.object({
+  "candidate_version": zod.number().int().min(1).max(createEvalBenchmarkBodyCandidateVersionMax),
+  "baseline_version": zod.number().int().min(1).max(createEvalBenchmarkBodyBaselineVersionMax),
+  "runs_per_configuration": zod.number().int().min(1).max(createEvalBenchmarkBodyRunsPerConfigurationMax).default(createEvalBenchmarkBodyRunsPerConfigurationDefault)
+});
+var createEvalBenchmarkResponseDataCandidateVersionMin = -9007199254740991;
+var createEvalBenchmarkResponseDataCandidateVersionMax = 9007199254740991;
+var createEvalBenchmarkResponseDataBaselineVersionMin = -9007199254740991;
+var createEvalBenchmarkResponseDataBaselineVersionMax = 9007199254740991;
+var createEvalBenchmarkResponseDataRunsPerConfigurationMin = -9007199254740991;
+var createEvalBenchmarkResponseDataRunsPerConfigurationMax = 9007199254740991;
+var createEvalBenchmarkResponseDataRunSummaryOneCandidateRunsMin = -9007199254740991;
+var createEvalBenchmarkResponseDataRunSummaryOneCandidateRunsMax = 9007199254740991;
+var createEvalBenchmarkResponseDataRunSummaryOneCandidateGradedRunsMin = -9007199254740991;
+var createEvalBenchmarkResponseDataRunSummaryOneCandidateGradedRunsMax = 9007199254740991;
+var createEvalBenchmarkResponseDataRunSummaryOneBaselineRunsMin = -9007199254740991;
+var createEvalBenchmarkResponseDataRunSummaryOneBaselineRunsMax = 9007199254740991;
+var createEvalBenchmarkResponseDataRunSummaryOneBaselineGradedRunsMin = -9007199254740991;
+var createEvalBenchmarkResponseDataRunSummaryOneBaselineGradedRunsMax = 9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemTargetVersionMin = -9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemTargetVersionMax = 9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemGradingOneSummaryTotalMin = -9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemGradingOneSummaryTotalMax = 9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemGradingOneSummaryPassedMin = -9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemGradingOneSummaryPassedMax = 9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemGradingOneSummaryFailedMin = -9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemGradingOneSummaryFailedMax = 9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemMetricsOneToolCallsMin = -9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemMetricsOneToolCallsMax = 9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthExactMin = -9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthExactMax = 9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthPartialMin = -9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthPartialMax = 9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthFifoMin = -9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthFifoMax = 9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthDefaultMin = -9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthDefaultMax = 9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthUnavailableMin = -9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthUnavailableMax = 9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemRunNumberOneMin = -9007199254740991;
+var createEvalBenchmarkResponseDataRunsItemRunNumberOneMax = 9007199254740991;
+var CreateEvalBenchmarkResponse = zod.object({
+  "data": zod.object({
+    "id": zod.string().uuid(),
+    "eval_id": zod.string().uuid(),
+    "candidate_version": zod.number().int().min(createEvalBenchmarkResponseDataCandidateVersionMin).max(createEvalBenchmarkResponseDataCandidateVersionMax),
+    "baseline_version": zod.number().int().min(createEvalBenchmarkResponseDataBaselineVersionMin).max(createEvalBenchmarkResponseDataBaselineVersionMax),
+    "runs_per_configuration": zod.number().int().min(createEvalBenchmarkResponseDataRunsPerConfigurationMin).max(createEvalBenchmarkResponseDataRunsPerConfigurationMax),
+    "status": zod.enum(["running", "completed", "error"]),
+    "run_summary": zod.union([zod.object({
+      "candidate": zod.object({
+        "runs": zod.number().int().min(createEvalBenchmarkResponseDataRunSummaryOneCandidateRunsMin).max(createEvalBenchmarkResponseDataRunSummaryOneCandidateRunsMax),
+        "graded_runs": zod.number().int().min(createEvalBenchmarkResponseDataRunSummaryOneCandidateGradedRunsMin).max(createEvalBenchmarkResponseDataRunSummaryOneCandidateGradedRunsMax),
+        "pass_rate": zod.union([zod.object({
+          "mean": zod.number(),
+          "stddev": zod.number(),
+          "min": zod.number(),
+          "max": zod.number()
+        }), zod.null()]),
+        "duration_seconds": zod.union([zod.object({
+          "mean": zod.number(),
+          "stddev": zod.number(),
+          "min": zod.number(),
+          "max": zod.number()
+        }), zod.null()]),
+        "fixture_matched_rate": zod.union([zod.object({
+          "mean": zod.number(),
+          "stddev": zod.number(),
+          "min": zod.number(),
+          "max": zod.number()
+        }), zod.null()])
+      }),
+      "baseline": zod.object({
+        "runs": zod.number().int().min(createEvalBenchmarkResponseDataRunSummaryOneBaselineRunsMin).max(createEvalBenchmarkResponseDataRunSummaryOneBaselineRunsMax),
+        "graded_runs": zod.number().int().min(createEvalBenchmarkResponseDataRunSummaryOneBaselineGradedRunsMin).max(createEvalBenchmarkResponseDataRunSummaryOneBaselineGradedRunsMax),
+        "pass_rate": zod.union([zod.object({
+          "mean": zod.number(),
+          "stddev": zod.number(),
+          "min": zod.number(),
+          "max": zod.number()
+        }), zod.null()]),
+        "duration_seconds": zod.union([zod.object({
+          "mean": zod.number(),
+          "stddev": zod.number(),
+          "min": zod.number(),
+          "max": zod.number()
+        }), zod.null()]),
+        "fixture_matched_rate": zod.union([zod.object({
+          "mean": zod.number(),
+          "stddev": zod.number(),
+          "min": zod.number(),
+          "max": zod.number()
+        }), zod.null()])
+      }),
+      "delta": zod.object({
+        "pass_rate": zod.union([zod.number(), zod.null()]),
+        "duration_seconds": zod.union([zod.number(), zod.null()]),
+        "fixture_matched_rate": zod.union([zod.number(), zod.null()])
+      })
+    }), zod.null()]),
+    "notes": zod.union([zod.array(zod.string()), zod.null()]),
+    "runs": zod.array(zod.object({
+      "id": zod.string().uuid(),
+      "eval_id": zod.string().uuid(),
+      "target_version": zod.number().int().min(createEvalBenchmarkResponseDataRunsItemTargetVersionMin).max(createEvalBenchmarkResponseDataRunsItemTargetVersionMax),
+      "status": zod.enum(["queued", "running", "passed", "failed", "error"]),
+      "passed": zod.union([zod.boolean(), zod.null()]),
+      "reason": zod.union([zod.string(), zod.null()]),
+      "error": zod.union([zod.string(), zod.null()]),
+      "judge_model": zod.union([zod.string(), zod.null()]),
+      "judge_prompt_version": zod.union([zod.string(), zod.null()]),
+      "grading": zod.union([zod.object({
+        "expectations": zod.array(zod.object({
+          "text": zod.string(),
+          "passed": zod.boolean(),
+          "evidence": zod.string()
+        })),
+        "summary": zod.object({
+          "total": zod.number().int().min(createEvalBenchmarkResponseDataRunsItemGradingOneSummaryTotalMin).max(createEvalBenchmarkResponseDataRunsItemGradingOneSummaryTotalMax),
+          "passed": zod.number().int().min(createEvalBenchmarkResponseDataRunsItemGradingOneSummaryPassedMin).max(createEvalBenchmarkResponseDataRunsItemGradingOneSummaryPassedMax),
+          "failed": zod.number().int().min(createEvalBenchmarkResponseDataRunsItemGradingOneSummaryFailedMin).max(createEvalBenchmarkResponseDataRunsItemGradingOneSummaryFailedMax),
+          "pass_rate": zod.number()
+        })
+      }), zod.null()]),
+      "metrics": zod.union([zod.object({
+        "duration_seconds": zod.union([zod.number(), zod.null()]),
+        "tool_calls": zod.number().int().min(createEvalBenchmarkResponseDataRunsItemMetricsOneToolCallsMin).max(createEvalBenchmarkResponseDataRunsItemMetricsOneToolCallsMax),
+        "fixture_health": zod.object({
+          "exact": zod.number().int().min(createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthExactMin).max(createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthExactMax),
+          "partial": zod.number().int().min(createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthPartialMin).max(createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthPartialMax),
+          "fifo": zod.number().int().min(createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthFifoMin).max(createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthFifoMax),
+          "default": zod.number().int().min(createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthDefaultMin).max(createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthDefaultMax),
+          "unavailable": zod.number().int().min(createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthUnavailableMin).max(createEvalBenchmarkResponseDataRunsItemMetricsOneFixtureHealthUnavailableMax),
+          "matched_rate": zod.union([zod.number(), zod.null()])
+        })
+      }), zod.null()]),
+      "benchmark_id": zod.union([zod.string().uuid(), zod.null()]),
+      "configuration": zod.union([zod.enum(["candidate", "baseline"]), zod.null()]),
+      "run_number": zod.union([zod.number().int().min(createEvalBenchmarkResponseDataRunsItemRunNumberOneMin).max(createEvalBenchmarkResponseDataRunsItemRunNumberOneMax), zod.null()]),
+      "created_at": zod.string().datetime({ "offset": true }),
+      "started_at": zod.union([zod.string().datetime({ "offset": true }), zod.null()]),
+      "completed_at": zod.union([zod.string().datetime({ "offset": true }), zod.null()])
+    })),
+    "created_at": zod.string().datetime({ "offset": true }),
+    "completed_at": zod.union([zod.string().datetime({ "offset": true }), zod.null()])
+  }),
+  "message": zod.string()
+});
+var GetBenchmarkParams = zod.object({
+  "benchmarkId": zod.string().uuid()
+});
+var getBenchmarkResponseDataCandidateVersionMin = -9007199254740991;
+var getBenchmarkResponseDataCandidateVersionMax = 9007199254740991;
+var getBenchmarkResponseDataBaselineVersionMin = -9007199254740991;
+var getBenchmarkResponseDataBaselineVersionMax = 9007199254740991;
+var getBenchmarkResponseDataRunsPerConfigurationMin = -9007199254740991;
+var getBenchmarkResponseDataRunsPerConfigurationMax = 9007199254740991;
+var getBenchmarkResponseDataRunSummaryOneCandidateRunsMin = -9007199254740991;
+var getBenchmarkResponseDataRunSummaryOneCandidateRunsMax = 9007199254740991;
+var getBenchmarkResponseDataRunSummaryOneCandidateGradedRunsMin = -9007199254740991;
+var getBenchmarkResponseDataRunSummaryOneCandidateGradedRunsMax = 9007199254740991;
+var getBenchmarkResponseDataRunSummaryOneBaselineRunsMin = -9007199254740991;
+var getBenchmarkResponseDataRunSummaryOneBaselineRunsMax = 9007199254740991;
+var getBenchmarkResponseDataRunSummaryOneBaselineGradedRunsMin = -9007199254740991;
+var getBenchmarkResponseDataRunSummaryOneBaselineGradedRunsMax = 9007199254740991;
+var getBenchmarkResponseDataRunsItemTargetVersionMin = -9007199254740991;
+var getBenchmarkResponseDataRunsItemTargetVersionMax = 9007199254740991;
+var getBenchmarkResponseDataRunsItemGradingOneSummaryTotalMin = -9007199254740991;
+var getBenchmarkResponseDataRunsItemGradingOneSummaryTotalMax = 9007199254740991;
+var getBenchmarkResponseDataRunsItemGradingOneSummaryPassedMin = -9007199254740991;
+var getBenchmarkResponseDataRunsItemGradingOneSummaryPassedMax = 9007199254740991;
+var getBenchmarkResponseDataRunsItemGradingOneSummaryFailedMin = -9007199254740991;
+var getBenchmarkResponseDataRunsItemGradingOneSummaryFailedMax = 9007199254740991;
+var getBenchmarkResponseDataRunsItemMetricsOneToolCallsMin = -9007199254740991;
+var getBenchmarkResponseDataRunsItemMetricsOneToolCallsMax = 9007199254740991;
+var getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthExactMin = -9007199254740991;
+var getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthExactMax = 9007199254740991;
+var getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthPartialMin = -9007199254740991;
+var getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthPartialMax = 9007199254740991;
+var getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthFifoMin = -9007199254740991;
+var getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthFifoMax = 9007199254740991;
+var getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthDefaultMin = -9007199254740991;
+var getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthDefaultMax = 9007199254740991;
+var getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthUnavailableMin = -9007199254740991;
+var getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthUnavailableMax = 9007199254740991;
+var getBenchmarkResponseDataRunsItemRunNumberOneMin = -9007199254740991;
+var getBenchmarkResponseDataRunsItemRunNumberOneMax = 9007199254740991;
+var GetBenchmarkResponse = zod.object({
+  "data": zod.object({
+    "id": zod.string().uuid(),
+    "eval_id": zod.string().uuid(),
+    "candidate_version": zod.number().int().min(getBenchmarkResponseDataCandidateVersionMin).max(getBenchmarkResponseDataCandidateVersionMax),
+    "baseline_version": zod.number().int().min(getBenchmarkResponseDataBaselineVersionMin).max(getBenchmarkResponseDataBaselineVersionMax),
+    "runs_per_configuration": zod.number().int().min(getBenchmarkResponseDataRunsPerConfigurationMin).max(getBenchmarkResponseDataRunsPerConfigurationMax),
+    "status": zod.enum(["running", "completed", "error"]),
+    "run_summary": zod.union([zod.object({
+      "candidate": zod.object({
+        "runs": zod.number().int().min(getBenchmarkResponseDataRunSummaryOneCandidateRunsMin).max(getBenchmarkResponseDataRunSummaryOneCandidateRunsMax),
+        "graded_runs": zod.number().int().min(getBenchmarkResponseDataRunSummaryOneCandidateGradedRunsMin).max(getBenchmarkResponseDataRunSummaryOneCandidateGradedRunsMax),
+        "pass_rate": zod.union([zod.object({
+          "mean": zod.number(),
+          "stddev": zod.number(),
+          "min": zod.number(),
+          "max": zod.number()
+        }), zod.null()]),
+        "duration_seconds": zod.union([zod.object({
+          "mean": zod.number(),
+          "stddev": zod.number(),
+          "min": zod.number(),
+          "max": zod.number()
+        }), zod.null()]),
+        "fixture_matched_rate": zod.union([zod.object({
+          "mean": zod.number(),
+          "stddev": zod.number(),
+          "min": zod.number(),
+          "max": zod.number()
+        }), zod.null()])
+      }),
+      "baseline": zod.object({
+        "runs": zod.number().int().min(getBenchmarkResponseDataRunSummaryOneBaselineRunsMin).max(getBenchmarkResponseDataRunSummaryOneBaselineRunsMax),
+        "graded_runs": zod.number().int().min(getBenchmarkResponseDataRunSummaryOneBaselineGradedRunsMin).max(getBenchmarkResponseDataRunSummaryOneBaselineGradedRunsMax),
+        "pass_rate": zod.union([zod.object({
+          "mean": zod.number(),
+          "stddev": zod.number(),
+          "min": zod.number(),
+          "max": zod.number()
+        }), zod.null()]),
+        "duration_seconds": zod.union([zod.object({
+          "mean": zod.number(),
+          "stddev": zod.number(),
+          "min": zod.number(),
+          "max": zod.number()
+        }), zod.null()]),
+        "fixture_matched_rate": zod.union([zod.object({
+          "mean": zod.number(),
+          "stddev": zod.number(),
+          "min": zod.number(),
+          "max": zod.number()
+        }), zod.null()])
+      }),
+      "delta": zod.object({
+        "pass_rate": zod.union([zod.number(), zod.null()]),
+        "duration_seconds": zod.union([zod.number(), zod.null()]),
+        "fixture_matched_rate": zod.union([zod.number(), zod.null()])
+      })
+    }), zod.null()]),
+    "notes": zod.union([zod.array(zod.string()), zod.null()]),
+    "runs": zod.array(zod.object({
+      "id": zod.string().uuid(),
+      "eval_id": zod.string().uuid(),
+      "target_version": zod.number().int().min(getBenchmarkResponseDataRunsItemTargetVersionMin).max(getBenchmarkResponseDataRunsItemTargetVersionMax),
+      "status": zod.enum(["queued", "running", "passed", "failed", "error"]),
+      "passed": zod.union([zod.boolean(), zod.null()]),
+      "reason": zod.union([zod.string(), zod.null()]),
+      "error": zod.union([zod.string(), zod.null()]),
+      "judge_model": zod.union([zod.string(), zod.null()]),
+      "judge_prompt_version": zod.union([zod.string(), zod.null()]),
+      "grading": zod.union([zod.object({
+        "expectations": zod.array(zod.object({
+          "text": zod.string(),
+          "passed": zod.boolean(),
+          "evidence": zod.string()
+        })),
+        "summary": zod.object({
+          "total": zod.number().int().min(getBenchmarkResponseDataRunsItemGradingOneSummaryTotalMin).max(getBenchmarkResponseDataRunsItemGradingOneSummaryTotalMax),
+          "passed": zod.number().int().min(getBenchmarkResponseDataRunsItemGradingOneSummaryPassedMin).max(getBenchmarkResponseDataRunsItemGradingOneSummaryPassedMax),
+          "failed": zod.number().int().min(getBenchmarkResponseDataRunsItemGradingOneSummaryFailedMin).max(getBenchmarkResponseDataRunsItemGradingOneSummaryFailedMax),
+          "pass_rate": zod.number()
+        })
+      }), zod.null()]),
+      "metrics": zod.union([zod.object({
+        "duration_seconds": zod.union([zod.number(), zod.null()]),
+        "tool_calls": zod.number().int().min(getBenchmarkResponseDataRunsItemMetricsOneToolCallsMin).max(getBenchmarkResponseDataRunsItemMetricsOneToolCallsMax),
+        "fixture_health": zod.object({
+          "exact": zod.number().int().min(getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthExactMin).max(getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthExactMax),
+          "partial": zod.number().int().min(getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthPartialMin).max(getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthPartialMax),
+          "fifo": zod.number().int().min(getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthFifoMin).max(getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthFifoMax),
+          "default": zod.number().int().min(getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthDefaultMin).max(getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthDefaultMax),
+          "unavailable": zod.number().int().min(getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthUnavailableMin).max(getBenchmarkResponseDataRunsItemMetricsOneFixtureHealthUnavailableMax),
+          "matched_rate": zod.union([zod.number(), zod.null()])
+        })
+      }), zod.null()]),
+      "benchmark_id": zod.union([zod.string().uuid(), zod.null()]),
+      "configuration": zod.union([zod.enum(["candidate", "baseline"]), zod.null()]),
+      "run_number": zod.union([zod.number().int().min(getBenchmarkResponseDataRunsItemRunNumberOneMin).max(getBenchmarkResponseDataRunsItemRunNumberOneMax), zod.null()]),
+      "created_at": zod.string().datetime({ "offset": true }),
+      "started_at": zod.union([zod.string().datetime({ "offset": true }), zod.null()]),
+      "completed_at": zod.union([zod.string().datetime({ "offset": true }), zod.null()])
+    })),
+    "created_at": zod.string().datetime({ "offset": true }),
+    "completed_at": zod.union([zod.string().datetime({ "offset": true }), zod.null()])
+  }),
+  "message": zod.string()
+});
 var CreateExtractorBody = zod.object({
   "name": zod.string().min(1),
   "schema": zod.record(zod.string(), zod.unknown())
@@ -12304,7 +12791,7 @@ var createMcpServer = (options) => {
     "createWorkerEvalFromRun",
     {
       title: "Create a worker eval from a run",
-      description: "Snapshot a worker run's trigger and resolved artifacts, and record its MCP tool responses from telemetry for replay.",
+      description: "Snapshot a worker run's trigger and resolved artifacts, and record its MCP tool responses from telemetry for replay. Each assertion is graded PASS/FAIL independently on every run.",
       inputSchema: {
         pathParams: CreateWorkerEvalFromRunParams,
         bodyParams: CreateWorkerEvalFromRunBody.optional()
@@ -12350,6 +12837,39 @@ var createMcpServer = (options) => {
       signal: options?.signal ? AbortSignal.any([options.signal, ctx.signal]) : ctx.signal
     })
   );
+  tools.createEvalBenchmark = server2.registerTool(
+    "createEvalBenchmark",
+    {
+      title: "Benchmark two worker versions on an eval",
+      description: "Queue runs_per_configuration trials of the eval against a candidate and a baseline worker version. When all trials finish, the benchmark reports mean \xB1 stddev pass rate per configuration, the candidate-minus-baseline delta, and analyzer notes.",
+      inputSchema: {
+        pathParams: CreateEvalBenchmarkParams,
+        bodyParams: CreateEvalBenchmarkBody.optional()
+      },
+      outputSchema: CreateEvalBenchmarkResponse,
+      annotations: { destructiveHint: true }
+    },
+    (args, ctx) => createEvalBenchmarkHandler(args, {
+      ...options,
+      signal: options?.signal ? AbortSignal.any([options.signal, ctx.signal]) : ctx.signal
+    })
+  );
+  tools.getBenchmark = server2.registerTool(
+    "getBenchmark",
+    {
+      title: "Get a benchmark",
+      description: "Get a benchmark with its per-run gradings and metrics, and once all runs finished, the per-configuration summary, delta, and analyzer notes.",
+      inputSchema: {
+        pathParams: GetBenchmarkParams
+      },
+      outputSchema: GetBenchmarkResponse,
+      annotations: { readOnlyHint: true }
+    },
+    (args, ctx) => getBenchmarkHandler(args, {
+      ...options,
+      signal: options?.signal ? AbortSignal.any([options.signal, ctx.signal]) : ctx.signal
+    })
+  );
   tools.createExtractor = server2.registerTool(
     "createExtractor",
     {
@@ -13934,4 +14454,4 @@ var transport = new StdioServerTransport();
 server.connect(transport).then(() => {
   console.error("MCP server running on stdio");
 }).catch(console.error);
-//# sourceMappingURL=server-NVU3VZZX.js.map
+//# sourceMappingURL=server-XM676NBD.js.map