@dalmia/calibrate-mcp 0.0.23 → 0.0.25

package/bin/mcp-server.js

@@ -51392,9 +51392,9 @@ var init_config = __esm(() => {
   SDK_METADATA = {
     language: "typescript",
     openapiDocVersion: "0.1.0",
-    sdkVersion: "0.0.23",
+    sdkVersion: "0.0.25",
     genVersion: "2.915.1",
-    userAgent: "speakeasy-sdk/mcp-typescript 0.0.23 2.915.1 0.1.0 @dalmia/calibrate-mcp"
+    userAgent: "speakeasy-sdk/mcp-typescript 0.0.25 2.915.1 0.1.0 @dalmia/calibrate-mcp"
   };
 });
 
@@ -54124,6 +54124,8 @@ var init_getbenchmarkstatusagenttestsbenchmarktaskidgetop = __esm(() => {
   init_benchmarkstatusresponse();
   init_httpvalidationerror();
   GetBenchmarkStatusAgentTestsBenchmarkTaskIdGetRequest$zodSchema = object({
+    compact: boolean2().default(false).describe("Return a compact response that omits heavy detail fields (`model_results.test_results`, `evaluators.output_config`), keeping only the lightweight decision fields. Omit for full detail"),
+    only_failed: boolean2().default(false).describe("Return only failing test cases for each model. Omit to return every case"),
     task_id: string2().describe("Benchmark run to poll for status and results"),
     xAPIKey: string2().nullable().optional()
   });
@@ -54151,6 +54153,10 @@ async function $do8(client$, request, options) {
     })
   };
   const path$ = pathToFunc("/agent-tests/benchmark/{task_id}")(pathParams$);
+  const query$ = encodeFormQuery({
+    compact: payload$.compact,
+    only_failed: payload$.only_failed
+  });
   const headers$ = new Headers(compactMap({
     Accept: "application/json",
     "X-API-Key": encodeSimple("X-API-Key", payload$.xAPIKey, {
@@ -54182,6 +54188,7 @@ async function $do8(client$, request, options) {
     baseURL: options?.serverURL,
     path: path$,
     headers: headers$,
+    query: query$,
     body: body$,
     userAgent: client$._options.userAgent,
     timeoutMs: options?.timeoutMs || client$._options.timeoutMs || -1
@@ -54281,6 +54288,8 @@ var init_getagenttestrunstatusagenttestsruntaskidgetop = __esm(() => {
   init_httpvalidationerror();
   init_testrunstatusresponse();
   GetAgentTestRunStatusAgentTestsRunTaskIdGetRequest$zodSchema = object({
+    compact: boolean2().default(false).describe("Return a compact response that omits heavy detail fields (`results.output`, `results.test_case`, `results.judge_results`, `results.reasoning`, `evaluators.output_config`), keeping only the lightweight decision fields. Omit for full detail"),
+    only_failed: boolean2().default(false).describe("Return only failing test cases. Omit to return every case"),
     task_id: string2().describe("Test run to poll for status and results"),
     xAPIKey: string2().nullable().optional()
   });
@@ -54308,6 +54317,10 @@ async function $do9(client$, request, options) {
     })
   };
   const path$ = pathToFunc("/agent-tests/run/{task_id}")(pathParams$);
+  const query$ = encodeFormQuery({
+    compact: payload$.compact,
+    only_failed: payload$.only_failed
+  });
   const headers$ = new Headers(compactMap({
     Accept: "application/json",
     "X-API-Key": encodeSimple("X-API-Key", payload$.xAPIKey, {
@@ -54339,6 +54352,7 @@ async function $do9(client$, request, options) {
     baseURL: options?.serverURL,
     path: path$,
     headers: headers$,
+    query: query$,
     body: body$,
     userAgent: client$._options.userAgent,
     timeoutMs: options?.timeoutMs || client$._options.timeoutMs || -1
@@ -56367,7 +56381,7 @@ var init_tasksummaryresponse = __esm(() => {
   TaskSummaryResponse$zodSchema = collectExtraKeys(object({
     annotators: array(record(string2(), any())).describe("The annotators who labelled or commented on items in the task, stable across pages"),
     evaluators: array(record(string2(), any())).describe("The task's evaluators, stable across pages, each with its live rubric"),
-    item_comments: record(string2(), any()).describe("Annotator comments, keyed by item ID"),
+    item_comments: record(string2(), any()).nullable().optional().describe("Annotator comments, keyed by item ID"),
     pagination: PaginationMeta$zodSchema,
     rows: array(record(string2(), any())).describe("One row per item, evaluator, and version on this page, with the evaluator's value and each annotator's label"),
     task_id: string2().describe("ID of the task"),
@@ -56377,7 +56391,7 @@ var init_tasksummaryresponse = __esm(() => {
     AdditionalProperties: record(string2(), any()).optional(),
     annotators: array(record(string2(), any())),
     evaluators: array(record(string2(), any())),
-    item_comments: record(string2(), any()),
+    item_comments: nullable(record(string2(), any())).optional(),
     pagination: PaginationMeta$zodSchema,
     rows: array(record(string2(), any())),
     task_id: string2(),
@@ -56402,6 +56416,8 @@ var init_tasksummaryannotationtaskstaskuuidsummarygetop = __esm(() => {
     "desc"
   ]).describe("Sort direction");
   TaskSummaryAnnotationTasksTaskUuidSummaryGetRequest$zodSchema = object({
+    compact: boolean2().default(false).describe("Return a compact response that omits heavy detail fields (`rows.payload`, `rows.evaluator_reasoning`, `rows.annotations.reasoning`, `evaluators.versions.system_prompt`, `evaluators.versions.output_config`, `evaluators.versions.variables`, `item_comments`), keeping only the lightweight decision fields. Omit for full detail"),
+    disagreement_only: boolean2().default(false).describe("When true, keep only rows where the evaluator disagreed with at least one annotator"),
     item_id: string2().describe("Filter rows to a single item. The full task-wide annotator union is still returned in `annotators`").nullable().optional(),
     limit: int().default(50).describe("Maximum number of items to return"),
     live_only: boolean2().default(false).describe("When true, emit only one row for each (item, evaluator) pair using the evaluator's live version. Versions other than the live one that have runs are excluded"),
@@ -56437,6 +56453,8 @@ async function $do21(client$, request, options) {
   };
   const path$ = pathToFunc("/annotation-tasks/{task_uuid}/summary")(pathParams$);
   const query$ = encodeFormQuery({
+    compact: payload$.compact,
+    disagreement_only: payload$.disagreement_only,
     item_id: payload$.item_id,
     limit: payload$.limit,
     live_only: payload$.live_only,
@@ -57389,56 +57407,56 @@ var init_evaluatorsCreateVersion2 = __esm(() => {
   };
 });
 
-// src/models/evaluatorversionresponse.ts
-var EvaluatorVersionResponse$zodSchema;
-var init_evaluatorversionresponse = __esm(() => {
+// src/models/evaluatorversioncompact.ts
+var EvaluatorVersionCompact$zodSchema;
+var init_evaluatorversioncompact = __esm(() => {
   init_zod();
   init_outputconfig();
   init_variablespec();
-  EvaluatorVersionResponse$zodSchema = object({
+  EvaluatorVersionCompact$zodSchema = object({
     created_at: string2().describe("When the version was created (ISO 8601 UTC)"),
     judge_model: string2().describe("The model that runs the judge, named the way its provider does, for example `openai/gpt-4.1` or `anthropic/claude-sonnet-4`"),
     output_config: OutputConfig$zodSchema.nullable().optional().describe("The scale points and their labels. Required for a `rating` evaluator. A `binary` evaluator uses the default Correct/Wrong labels unless you set your own"),
-    system_prompt: string2().describe('Judge system prompt, with `{{"{{"}}variable}}` placeholders unrendered'),
+    system_prompt: string2().nullable().optional().describe('Judge system prompt, with `{{"{{"}}variable}}` placeholders unrendered'),
     uuid: string2().describe("Version ID"),
     variables: array(VariableSpec$zodSchema).nullable().optional().describe("Declared prompt variables"),
     version_number: int().describe("The version's number. The first version is 1, and it goes up by one for each new version of the evaluator")
   });
 });
 
-// src/models/evaluatordetailresponse.ts
-var EvaluatorDetailResponseEvaluatorType$zodSchema, EvaluatorDetailResponseDataType$zodSchema, EvaluatorDetailResponseOutputType$zodSchema, EvaluatorDetailResponse$zodSchema;
-var init_evaluatordetailresponse = __esm(() => {
+// src/models/evaluatordetailresponsecompact.ts
+var EvaluatorDetailResponseCompactEvaluatorType$zodSchema, EvaluatorDetailResponseCompactDataType$zodSchema, EvaluatorDetailResponseCompactOutputType$zodSchema, EvaluatorDetailResponseCompact$zodSchema;
+var init_evaluatordetailresponsecompact = __esm(() => {
   init_zod();
-  init_evaluatorversionresponse();
-  EvaluatorDetailResponseEvaluatorType$zodSchema = _enum([
+  init_evaluatorversioncompact();
+  EvaluatorDetailResponseCompactEvaluatorType$zodSchema = _enum([
     "tts",
     "stt",
     "llm",
     "llm-general",
     "conversation"
   ]).describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n");
-  EvaluatorDetailResponseDataType$zodSchema = _enum([
+  EvaluatorDetailResponseCompactDataType$zodSchema = _enum([
     "text",
     "audio"
   ]).describe("The modality the judge reads:\n\n- `text`\n- `audio`\n");
-  EvaluatorDetailResponseOutputType$zodSchema = _enum([
+  EvaluatorDetailResponseCompactOutputType$zodSchema = _enum([
     "binary",
     "rating"
   ]).describe("How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n");
-  EvaluatorDetailResponse$zodSchema = object({
+  EvaluatorDetailResponseCompact$zodSchema = object({
     created_at: string2().describe("When the evaluator was created (ISO 8601 UTC)"),
-    data_type: EvaluatorDetailResponseDataType$zodSchema.describe("The modality the judge reads:\n\n- `text`\n- `audio`\n"),
+    data_type: EvaluatorDetailResponseCompactDataType$zodSchema.describe("The modality the judge reads:\n\n- `text`\n- `audio`\n"),
     description: string2().nullable().optional().describe("What the evaluator checks"),
-    evaluator_type: EvaluatorDetailResponseEvaluatorType$zodSchema.describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n"),
+    evaluator_type: EvaluatorDetailResponseCompactEvaluatorType$zodSchema.describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n"),
     is_default: boolean2().describe("True for a built-in default evaluator, which you can't edit. False for an evaluator you created, which you can edit and add versions to"),
     live_version_id: string2().nullable().optional().describe("ID of the version that is currently live"),
     name: string2().describe("Evaluator name"),
-    output_type: EvaluatorDetailResponseOutputType$zodSchema.describe("How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n"),
+    output_type: EvaluatorDetailResponseCompactOutputType$zodSchema.describe("How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n"),
     slug: string2().nullable().optional().describe("Stable slug for a built-in default evaluator"),
     updated_at: string2().describe("When the evaluator was last updated (ISO 8601 UTC)"),
     uuid: string2().describe("Evaluator ID"),
-    versions: array(EvaluatorVersionResponse$zodSchema).describe("Full version history, oldest first")
+    versions: array(EvaluatorVersionCompact$zodSchema).describe("Full version history, oldest first")
   });
 });
 
@@ -57446,14 +57464,15 @@ var init_evaluatordetailresponse = __esm(() => {
 var GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetRequest$zodSchema, GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetResponse$zodSchema;
 var init_getevaluatorendpointevaluatorsevaluatoruuidgetop = __esm(() => {
   init_zod();
-  init_evaluatordetailresponse();
+  init_evaluatordetailresponsecompact();
   init_httpvalidationerror();
   GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetRequest$zodSchema = object({
+    compact: boolean2().default(false).describe("Return a compact response that omits heavy detail fields (`versions.system_prompt`, `versions.output_config`, `versions.variables`), keeping only the lightweight decision fields. Omit for full detail"),
     evaluator_uuid: string2().describe("Evaluator to retrieve"),
     xAPIKey: string2().nullable().optional()
   });
   GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetResponse$zodSchema = union([
-    EvaluatorDetailResponse$zodSchema,
+    EvaluatorDetailResponseCompact$zodSchema,
     HTTPValidationError$zodSchema
   ]);
 });
@@ -57476,6 +57495,9 @@ async function $do27(client$, request, options) {
     })
   };
   const path$ = pathToFunc("/evaluators/{evaluator_uuid}")(pathParams$);
+  const query$ = encodeFormQuery({
+    compact: payload$.compact
+  });
   const headers$ = new Headers(compactMap({
     Accept: "application/json",
     "X-API-Key": encodeSimple("X-API-Key", payload$.xAPIKey, {
@@ -57507,6 +57529,7 @@ async function $do27(client$, request, options) {
     baseURL: options?.serverURL,
     path: path$,
     headers: headers$,
+    query: query$,
     body: body$,
     userAgent: client$._options.userAgent,
     timeoutMs: options?.timeoutMs || client$._options.timeoutMs || -1
@@ -58636,7 +58659,7 @@ fields to change (name, config). Returns the updated test. Fails with
 function createMCPServer(deps) {
   const server = new McpServer({
     name: "CalibrateMcp",
-    version: "0.0.23"
+    version: "0.0.25"
   });
   const getClient = deps.getSDK || (() => new CalibrateMcpCore({
     security: deps.security,
@@ -59842,7 +59865,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
         <h1>Instructions</h1>
         <p>One-click installation for Claude Desktop users</p>
         <div class="instruction-item">
-          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.23/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
+          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.25/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
             \uD83D\uDCE5 Download MCP Bundle
           </a>
         </div>
@@ -62494,7 +62517,7 @@ var routes = buildRouteMap({
 var app = buildApplication(routes, {
   name: "mcp",
   versionInfo: {
-    currentVersion: "0.0.23"
+    currentVersion: "0.0.25"
   }
 });
 run(app, process4.argv.slice(2), buildContext(process4));
@@ -62502,5 +62525,5 @@ export {
   app
 };
 
-//# debugId=C650F485386E271664756E2164756E21
+//# debugId=4F09830EB4B94F2464756E2164756E21
 //# sourceMappingURL=mcp-server.js.map