@dalmia/calibrate-mcp 0.0.24 → 0.0.26

package/bin/mcp-server.js

@@ -51392,9 +51392,9 @@ var init_config = __esm(() => {
   SDK_METADATA = {
     language: "typescript",
     openapiDocVersion: "0.1.0",
-    sdkVersion: "0.0.24",
+    sdkVersion: "0.0.26",
     genVersion: "2.915.1",
-    userAgent: "speakeasy-sdk/mcp-typescript 0.0.24 2.915.1 0.1.0 @dalmia/calibrate-mcp"
+    userAgent: "speakeasy-sdk/mcp-typescript 0.0.26 2.915.1 0.1.0 @dalmia/calibrate-mcp"
   };
 });
 
@@ -54124,6 +54124,8 @@ var init_getbenchmarkstatusagenttestsbenchmarktaskidgetop = __esm(() => {
   init_benchmarkstatusresponse();
   init_httpvalidationerror();
   GetBenchmarkStatusAgentTestsBenchmarkTaskIdGetRequest$zodSchema = object({
+    compact: boolean2().default(false).describe("Return a compact response that omits heavy detail fields (`model_results.test_results`, `evaluators.output_config`), keeping only the lightweight decision fields. Omit for full detail"),
+    only_failed: boolean2().default(false).describe("Return only failing test cases for each model. Omit to return every case"),
     task_id: string2().describe("Benchmark run to poll for status and results"),
     xAPIKey: string2().nullable().optional()
   });
@@ -54151,6 +54153,10 @@ async function $do8(client$, request, options) {
     })
   };
   const path$ = pathToFunc("/agent-tests/benchmark/{task_id}")(pathParams$);
+  const query$ = encodeFormQuery({
+    compact: payload$.compact,
+    only_failed: payload$.only_failed
+  });
   const headers$ = new Headers(compactMap({
     Accept: "application/json",
     "X-API-Key": encodeSimple("X-API-Key", payload$.xAPIKey, {
@@ -54182,6 +54188,7 @@ async function $do8(client$, request, options) {
     baseURL: options?.serverURL,
     path: path$,
     headers: headers$,
+    query: query$,
     body: body$,
     userAgent: client$._options.userAgent,
     timeoutMs: options?.timeoutMs || client$._options.timeoutMs || -1
@@ -54281,6 +54288,8 @@ var init_getagenttestrunstatusagenttestsruntaskidgetop = __esm(() => {
   init_httpvalidationerror();
   init_testrunstatusresponse();
   GetAgentTestRunStatusAgentTestsRunTaskIdGetRequest$zodSchema = object({
+    compact: boolean2().default(false).describe("Return a compact response that omits heavy detail fields (`results.output`, `results.test_case`, `results.judge_results`, `results.reasoning`, `evaluators.output_config`), keeping only the lightweight decision fields. Omit for full detail"),
+    only_failed: boolean2().default(false).describe("Return only failing test cases. Omit to return every case"),
     task_id: string2().describe("Test run to poll for status and results"),
     xAPIKey: string2().nullable().optional()
   });
@@ -54308,6 +54317,10 @@ async function $do9(client$, request, options) {
     })
   };
   const path$ = pathToFunc("/agent-tests/run/{task_id}")(pathParams$);
+  const query$ = encodeFormQuery({
+    compact: payload$.compact,
+    only_failed: payload$.only_failed
+  });
   const headers$ = new Headers(compactMap({
     Accept: "application/json",
     "X-API-Key": encodeSimple("X-API-Key", payload$.xAPIKey, {
@@ -54339,6 +54352,7 @@ async function $do9(client$, request, options) {
     baseURL: options?.serverURL,
     path: path$,
     headers: headers$,
+    query: query$,
     body: body$,
     userAgent: client$._options.userAgent,
     timeoutMs: options?.timeoutMs || client$._options.timeoutMs || -1
@@ -56367,7 +56381,7 @@ var init_tasksummaryresponse = __esm(() => {
   TaskSummaryResponse$zodSchema = collectExtraKeys(object({
     annotators: array(record(string2(), any())).describe("The annotators who labelled or commented on items in the task, stable across pages"),
     evaluators: array(record(string2(), any())).describe("The task's evaluators, stable across pages, each with its live rubric"),
-    item_comments: record(string2(), any()).describe("Annotator comments, keyed by item ID"),
+    item_comments: record(string2(), any()).nullable().optional().describe("Annotator comments, keyed by item ID"),
     pagination: PaginationMeta$zodSchema,
     rows: array(record(string2(), any())).describe("One row per item, evaluator, and version on this page, with the evaluator's value and each annotator's label"),
     task_id: string2().describe("ID of the task"),
@@ -56377,7 +56391,7 @@ var init_tasksummaryresponse = __esm(() => {
     AdditionalProperties: record(string2(), any()).optional(),
     annotators: array(record(string2(), any())),
     evaluators: array(record(string2(), any())),
-    item_comments: record(string2(), any()),
+    item_comments: nullable(record(string2(), any())).optional(),
     pagination: PaginationMeta$zodSchema,
     rows: array(record(string2(), any())),
     task_id: string2(),
@@ -56402,6 +56416,8 @@ var init_tasksummaryannotationtaskstaskuuidsummarygetop = __esm(() => {
     "desc"
   ]).describe("Sort direction");
   TaskSummaryAnnotationTasksTaskUuidSummaryGetRequest$zodSchema = object({
+    compact: boolean2().default(false).describe("Return a compact response that omits heavy detail fields (`rows.payload`, `rows.evaluator_reasoning`, `rows.annotations.reasoning`, `evaluators.versions.system_prompt`, `evaluators.versions.output_config`, `evaluators.versions.variables`, `item_comments`), keeping only the lightweight decision fields. Omit for full detail"),
+    disagreement_only: boolean2().default(false).describe("When true, keep only rows where the evaluator disagreed with at least one annotator"),
     item_id: string2().describe("Filter rows to a single item. The full task-wide annotator union is still returned in `annotators`").nullable().optional(),
     limit: int().default(50).describe("Maximum number of items to return"),
     live_only: boolean2().default(false).describe("When true, emit only one row for each (item, evaluator) pair using the evaluator's live version. Versions other than the live one that have runs are excluded"),
@@ -56437,6 +56453,8 @@ async function $do21(client$, request, options) {
   };
   const path$ = pathToFunc("/annotation-tasks/{task_uuid}/summary")(pathParams$);
   const query$ = encodeFormQuery({
+    compact: payload$.compact,
+    disagreement_only: payload$.disagreement_only,
     item_id: payload$.item_id,
     limit: payload$.limit,
     live_only: payload$.live_only,
@@ -56545,64 +56563,57 @@ var init_annotationTasksGetSummary2 = __esm(() => {
   };
 });
 
-// src/models/evaluatorlinkrequest.ts
-var EvaluatorLinkRequest$zodSchema;
-var init_evaluatorlinkrequest = __esm(() => {
-  init_zod();
-  EvaluatorLinkRequest$zodSchema = object({
-    evaluator_id: string2().describe("The evaluator to link. Must be one you created or a built-in default")
-  });
-});
-
-// src/models/evaluatorlinkresponse.ts
-var EvaluatorLinkResponse$zodSchema;
-var init_evaluatorlinkresponse = __esm(() => {
+// src/models/paginatedresponseannotationtaskresponse.ts
+var PaginatedResponseAnnotationTaskResponse$zodSchema;
+var init_paginatedresponseannotationtaskresponse = __esm(() => {
   init_zod();
-  EvaluatorLinkResponse$zodSchema = object({
-    message: string2().describe("Confirmation that the evaluator was linked")
+  init_annotationtaskresponse();
+  PaginatedResponseAnnotationTaskResponse$zodSchema = object({
+    items: array(AnnotationTaskResponse$zodSchema).describe("The page of results"),
+    limit: int().nullable().optional().describe("The applied `limit` (null when unbounded)"),
+    offset: int().default(0).describe("The applied `offset`"),
+    total: int().describe("Total number of items matching the query, before pagination")
   });
 });
 
-// src/models/linkevaluatortotaskannotationtaskstaskuuidevaluatorspostop.ts
-var LinkEvaluatorToTaskAnnotationTasksTaskUuidEvaluatorsPostRequest$zodSchema, LinkEvaluatorToTaskAnnotationTasksTaskUuidEvaluatorsPostResponse$zodSchema;
-var init_linkevaluatortotaskannotationtaskstaskuuidevaluatorspostop = __esm(() => {
+// src/models/listannotationtasksannotationtasksgetop.ts
+var ListAnnotationTasksAnnotationTasksGetRequest$zodSchema, ListAnnotationTasksAnnotationTasksGetResponse$zodSchema;
+var init_listannotationtasksannotationtasksgetop = __esm(() => {
   init_zod();
-  init_evaluatorlinkrequest();
-  init_evaluatorlinkresponse();
   init_httpvalidationerror();
-  LinkEvaluatorToTaskAnnotationTasksTaskUuidEvaluatorsPostRequest$zodSchema = object({
-    body: EvaluatorLinkRequest$zodSchema,
-    task_uuid: string2().describe("Annotation task to act on"),
+  init_paginatedresponseannotationtaskresponse();
+  ListAnnotationTasksAnnotationTasksGetRequest$zodSchema = object({
+    limit: int().describe("Maximum number of items to return. Omit for no limit (all items)").nullable().optional(),
+    offset: int().default(0).describe("Number of items to skip before returning results"),
+    q: string2().describe("Case-insensitive substring search on `name`. Blank is a no-op").nullable().optional(),
     xAPIKey: string2().nullable().optional()
   });
-  LinkEvaluatorToTaskAnnotationTasksTaskUuidEvaluatorsPostResponse$zodSchema = union([
-    EvaluatorLinkResponse$zodSchema,
+  ListAnnotationTasksAnnotationTasksGetResponse$zodSchema = union([
+    PaginatedResponseAnnotationTaskResponse$zodSchema,
     HTTPValidationError$zodSchema
   ]);
 });
 
-// src/funcs/annotationTasksLinkEvaluator.ts
-function annotationTasksLinkEvaluator(client$, request, options) {
+// src/funcs/annotationTasksList.ts
+function annotationTasksList(client$, request, options) {
   return new APIPromise($do22(client$, request, options));
 }
 async function $do22(client$, request, options) {
-  const parsed$ = safeParse4(request, (value$) => LinkEvaluatorToTaskAnnotationTasksTaskUuidEvaluatorsPostRequest$zodSchema.parse(value$), "Input validation failed");
+  const parsed$ = safeParse4(request, (value$) => ListAnnotationTasksAnnotationTasksGetRequest$zodSchema.optional().parse(value$), "Input validation failed");
   if (!parsed$.ok) {
     return [parsed$, { status: "invalid" }];
   }
   const payload$ = parsed$.value;
-  const body$ = encodeJSON("body", payload$.body, { explode: true });
-  const pathParams$ = {
-    task_uuid: encodeSimple("task_uuid", payload$.task_uuid, {
-      explode: false,
-      charEncoding: "percent"
-    })
-  };
-  const path$ = pathToFunc("/annotation-tasks/{task_uuid}/evaluators")(pathParams$);
+  const body$ = null;
+  const path$ = pathToFunc("/annotation-tasks")();
+  const query$ = encodeFormQuery({
+    limit: payload$?.limit,
+    offset: payload$?.offset,
+    q: payload$?.q
+  });
   const headers$ = new Headers(compactMap({
-    "Content-Type": "application/json",
     Accept: "application/json",
-    "X-API-Key": encodeSimple("X-API-Key", payload$.xAPIKey, {
+    "X-API-Key": encodeSimple("X-API-Key", payload$?.xAPIKey, {
       explode: false,
       charEncoding: "none"
     })
@@ -56612,7 +56623,7 @@ async function $do22(client$, request, options) {
   const context = {
     options: client$._options,
     baseURL: options?.serverURL ?? client$._baseURL ?? "",
-    operationID: "link_evaluator_to_task_annotation_tasks__task_uuid__evaluators_post",
+    operationID: "list_annotation_tasks_annotation_tasks_get",
     oAuth2Scopes: null,
     resolvedSecurity: requestSecurity,
     securitySource: client$._options.security,
@@ -56627,10 +56638,11 @@ async function $do22(client$, request, options) {
   };
   const requestRes = client$._createRequest(context, {
     security: requestSecurity,
-    method: "POST",
+    method: "GET",
     baseURL: options?.serverURL,
     path: path$,
     headers: headers$,
+    query: query$,
     body: body$,
     userAgent: client$._options.userAgent,
     timeoutMs: options?.timeoutMs || client$._options.timeoutMs || -1
@@ -56654,40 +56666,40 @@ async function $do22(client$, request, options) {
     response: doResult.value
   }];
 }
-var init_annotationTasksLinkEvaluator = __esm(() => {
+var init_annotationTasksList = __esm(() => {
   init_encodings();
   init_primitives();
   init_schemas4();
   init_security();
   init_url();
-  init_linkevaluatortotaskannotationtaskstaskuuidevaluatorspostop();
+  init_listannotationtasksannotationtasksgetop();
   init_async();
 });
 
-// src/mcp-server/tools/annotationTasksLinkEvaluator.ts
-var args22, tool$annotationTasksLinkEvaluator;
-var init_annotationTasksLinkEvaluator2 = __esm(() => {
-  init_annotationTasksLinkEvaluator();
-  init_linkevaluatortotaskannotationtaskstaskuuidevaluatorspostop();
+// src/mcp-server/tools/annotationTasksList.ts
+var args22, tool$annotationTasksList;
+var init_annotationTasksList2 = __esm(() => {
+  init_annotationTasksList();
+  init_listannotationtasksannotationtasksgetop();
   init_tools();
   args22 = {
-    request: LinkEvaluatorToTaskAnnotationTasksTaskUuidEvaluatorsPostRequest$zodSchema
+    request: ListAnnotationTasksAnnotationTasksGetRequest$zodSchema.optional()
   };
-  tool$annotationTasksLinkEvaluator = {
-    name: "link-evaluator-to-task",
-    description: `Link an evaluator to an annotation task, appending it to the display order.
+  tool$annotationTasksList = {
+    name: "list-annotation-tasks",
+    description: `List your annotation tasks with their linked evaluators.
 `,
-    scopes: ["write"],
+    scopes: ["read"],
     annotations: {
-      title: "Link evaluator to task",
+      title: "List annotation tasks",
       destructiveHint: false,
-      idempotentHint: true,
+      idempotentHint: false,
       openWorldHint: true,
-      readOnlyHint: false
+      readOnlyHint: true
     },
     args: args22,
     tool: async (client, args23, ctx) => {
-      const [result] = await annotationTasksLinkEvaluator(client, args23.request, { fetchOptions: { signal: ctx.signal } }).$inspect();
+      const [result] = await annotationTasksList(client, args23.request, { fetchOptions: { signal: ctx.signal } }).$inspect();
       if (!result.ok) {
         return {
           content: [{ type: "text", text: result.error.message }],
@@ -56699,57 +56711,67 @@ var init_annotationTasksLinkEvaluator2 = __esm(() => {
   };
 });
 
-// src/models/paginatedresponseannotationtaskresponse.ts
-var PaginatedResponseAnnotationTaskResponse$zodSchema;
-var init_paginatedresponseannotationtaskresponse = __esm(() => {
+// src/models/evaluatorsetrequest.ts
+var EvaluatorSetRequest$zodSchema;
+var init_evaluatorsetrequest = __esm(() => {
   init_zod();
-  init_annotationtaskresponse();
-  PaginatedResponseAnnotationTaskResponse$zodSchema = object({
-    items: array(AnnotationTaskResponse$zodSchema).describe("The page of results"),
-    limit: int().nullable().optional().describe("The applied `limit` (null when unbounded)"),
-    offset: int().default(0).describe("The applied `offset`"),
-    total: int().describe("Total number of items matching the query, before pagination")
+  EvaluatorSetRequest$zodSchema = object({
+    evaluator_ids: array(string2()).describe("The full ordered set of evaluators the task should end up linked to, in display order. Missing ones are unlinked, new ones are linked, and the order sets their position. Send an empty list to unlink all. Each must be one you created or a built-in default")
   });
 });
 
-// src/models/listannotationtasksannotationtasksgetop.ts
-var ListAnnotationTasksAnnotationTasksGetRequest$zodSchema, ListAnnotationTasksAnnotationTasksGetResponse$zodSchema;
-var init_listannotationtasksannotationtasksgetop = __esm(() => {
+// src/models/evaluatorsetresponse.ts
+var EvaluatorSetResponse$zodSchema;
+var init_evaluatorsetresponse = __esm(() => {
   init_zod();
+  EvaluatorSetResponse$zodSchema = object({
+    evaluator_ids: array(string2()).describe("The task's evaluator IDs after the update, in display order"),
+    linked: array(string2()).describe("Evaluator IDs newly linked by this request"),
+    message: string2().describe("Confirmation that the task's evaluators were updated"),
+    unlinked: array(string2()).describe("Evaluator IDs unlinked by this request")
+  });
+});
+
+// src/models/settaskevaluatorsannotationtaskstaskuuidevaluatorsputop.ts
+var SetTaskEvaluatorsAnnotationTasksTaskUuidEvaluatorsPutRequest$zodSchema, SetTaskEvaluatorsAnnotationTasksTaskUuidEvaluatorsPutResponse$zodSchema;
+var init_settaskevaluatorsannotationtaskstaskuuidevaluatorsputop = __esm(() => {
+  init_zod();
+  init_evaluatorsetrequest();
+  init_evaluatorsetresponse();
   init_httpvalidationerror();
-  init_paginatedresponseannotationtaskresponse();
-  ListAnnotationTasksAnnotationTasksGetRequest$zodSchema = object({
-    limit: int().describe("Maximum number of items to return. Omit for no limit (all items)").nullable().optional(),
-    offset: int().default(0).describe("Number of items to skip before returning results"),
-    q: string2().describe("Case-insensitive substring search on `name`. Blank is a no-op").nullable().optional(),
+  SetTaskEvaluatorsAnnotationTasksTaskUuidEvaluatorsPutRequest$zodSchema = object({
+    body: EvaluatorSetRequest$zodSchema,
+    task_uuid: string2().describe("Annotation task to act on"),
     xAPIKey: string2().nullable().optional()
   });
-  ListAnnotationTasksAnnotationTasksGetResponse$zodSchema = union([
-    PaginatedResponseAnnotationTaskResponse$zodSchema,
+  SetTaskEvaluatorsAnnotationTasksTaskUuidEvaluatorsPutResponse$zodSchema = union([
+    EvaluatorSetResponse$zodSchema,
     HTTPValidationError$zodSchema
   ]);
 });
 
-// src/funcs/annotationTasksList.ts
-function annotationTasksList(client$, request, options) {
+// src/funcs/annotationTasksSetEvaluators.ts
+function annotationTasksSetEvaluators(client$, request, options) {
   return new APIPromise($do23(client$, request, options));
 }
 async function $do23(client$, request, options) {
-  const parsed$ = safeParse4(request, (value$) => ListAnnotationTasksAnnotationTasksGetRequest$zodSchema.optional().parse(value$), "Input validation failed");
+  const parsed$ = safeParse4(request, (value$) => SetTaskEvaluatorsAnnotationTasksTaskUuidEvaluatorsPutRequest$zodSchema.parse(value$), "Input validation failed");
   if (!parsed$.ok) {
     return [parsed$, { status: "invalid" }];
   }
   const payload$ = parsed$.value;
-  const body$ = null;
-  const path$ = pathToFunc("/annotation-tasks")();
-  const query$ = encodeFormQuery({
-    limit: payload$?.limit,
-    offset: payload$?.offset,
-    q: payload$?.q
-  });
+  const body$ = encodeJSON("body", payload$.body, { explode: true });
+  const pathParams$ = {
+    task_uuid: encodeSimple("task_uuid", payload$.task_uuid, {
+      explode: false,
+      charEncoding: "percent"
+    })
+  };
+  const path$ = pathToFunc("/annotation-tasks/{task_uuid}/evaluators")(pathParams$);
   const headers$ = new Headers(compactMap({
+    "Content-Type": "application/json",
     Accept: "application/json",
-    "X-API-Key": encodeSimple("X-API-Key", payload$?.xAPIKey, {
+    "X-API-Key": encodeSimple("X-API-Key", payload$.xAPIKey, {
       explode: false,
       charEncoding: "none"
     })
@@ -56759,7 +56781,7 @@ async function $do23(client$, request, options) {
   const context = {
     options: client$._options,
     baseURL: options?.serverURL ?? client$._baseURL ?? "",
-    operationID: "list_annotation_tasks_annotation_tasks_get",
+    operationID: "set_task_evaluators_annotation_tasks__task_uuid__evaluators_put",
     oAuth2Scopes: null,
     resolvedSecurity: requestSecurity,
     securitySource: client$._options.security,
@@ -56774,11 +56796,10 @@ async function $do23(client$, request, options) {
   };
   const requestRes = client$._createRequest(context, {
     security: requestSecurity,
-    method: "GET",
+    method: "PUT",
     baseURL: options?.serverURL,
     path: path$,
     headers: headers$,
-    query: query$,
     body: body$,
     userAgent: client$._options.userAgent,
     timeoutMs: options?.timeoutMs || client$._options.timeoutMs || -1
@@ -56802,40 +56823,44 @@ async function $do23(client$, request, options) {
     response: doResult.value
   }];
 }
-var init_annotationTasksList = __esm(() => {
+var init_annotationTasksSetEvaluators = __esm(() => {
   init_encodings();
   init_primitives();
   init_schemas4();
   init_security();
   init_url();
-  init_listannotationtasksannotationtasksgetop();
+  init_settaskevaluatorsannotationtaskstaskuuidevaluatorsputop();
   init_async();
 });
 
-// src/mcp-server/tools/annotationTasksList.ts
-var args23, tool$annotationTasksList;
-var init_annotationTasksList2 = __esm(() => {
-  init_annotationTasksList();
-  init_listannotationtasksannotationtasksgetop();
+// src/mcp-server/tools/annotationTasksSetEvaluators.ts
+var args23, tool$annotationTasksSetEvaluators;
+var init_annotationTasksSetEvaluators2 = __esm(() => {
+  init_annotationTasksSetEvaluators();
+  init_settaskevaluatorsannotationtaskstaskuuidevaluatorsputop();
   init_tools();
   args23 = {
-    request: ListAnnotationTasksAnnotationTasksGetRequest$zodSchema.optional()
-  };
-  tool$annotationTasksList = {
-    name: "list-annotation-tasks",
-    description: `List your annotation tasks with their linked evaluators.
+    request: SetTaskEvaluatorsAnnotationTasksTaskUuidEvaluatorsPutRequest$zodSchema
+  };
+  tool$annotationTasksSetEvaluators = {
+    name: "set-task-evaluators",
+    description: `Replace the task's linked evaluators with exactly the given ordered
+set: new ids are linked, omitted ids are unlinked, and the list order
+sets their display position. Send an empty list to unlink all. Each
+evaluator must be one you created or a built-in default, and the task
+must be in your workspace.
 `,
-    scopes: ["read"],
+    scopes: ["write"],
     annotations: {
-      title: "List annotation tasks",
+      title: "Set an annotation task's evaluators",
       destructiveHint: false,
-      idempotentHint: false,
+      idempotentHint: true,
       openWorldHint: true,
-      readOnlyHint: true
+      readOnlyHint: false
     },
     args: args23,
     tool: async (client, args24, ctx) => {
-      const [result] = await annotationTasksList(client, args24.request, { fetchOptions: { signal: ctx.signal } }).$inspect();
+      const [result] = await annotationTasksSetEvaluators(client, args24.request, { fetchOptions: { signal: ctx.signal } }).$inspect();
       if (!result.ok) {
         return {
           content: [{ type: "text", text: result.error.message }],
@@ -57389,56 +57414,56 @@ var init_evaluatorsCreateVersion2 = __esm(() => {
   };
 });
 
-// src/models/evaluatorversionresponse.ts
-var EvaluatorVersionResponse$zodSchema;
-var init_evaluatorversionresponse = __esm(() => {
+// src/models/evaluatorversioncompact.ts
+var EvaluatorVersionCompact$zodSchema;
+var init_evaluatorversioncompact = __esm(() => {
   init_zod();
   init_outputconfig();
   init_variablespec();
-  EvaluatorVersionResponse$zodSchema = object({
+  EvaluatorVersionCompact$zodSchema = object({
     created_at: string2().describe("When the version was created (ISO 8601 UTC)"),
     judge_model: string2().describe("The model that runs the judge, named the way its provider does, for example `openai/gpt-4.1` or `anthropic/claude-sonnet-4`"),
     output_config: OutputConfig$zodSchema.nullable().optional().describe("The scale points and their labels. Required for a `rating` evaluator. A `binary` evaluator uses the default Correct/Wrong labels unless you set your own"),
-    system_prompt: string2().describe('Judge system prompt, with `{{"{{"}}variable}}` placeholders unrendered'),
+    system_prompt: string2().nullable().optional().describe('Judge system prompt, with `{{"{{"}}variable}}` placeholders unrendered'),
     uuid: string2().describe("Version ID"),
     variables: array(VariableSpec$zodSchema).nullable().optional().describe("Declared prompt variables"),
     version_number: int().describe("The version's number. The first version is 1, and it goes up by one for each new version of the evaluator")
   });
 });
 
-// src/models/evaluatordetailresponse.ts
-var EvaluatorDetailResponseEvaluatorType$zodSchema, EvaluatorDetailResponseDataType$zodSchema, EvaluatorDetailResponseOutputType$zodSchema, EvaluatorDetailResponse$zodSchema;
-var init_evaluatordetailresponse = __esm(() => {
+// src/models/evaluatordetailresponsecompact.ts
+var EvaluatorDetailResponseCompactEvaluatorType$zodSchema, EvaluatorDetailResponseCompactDataType$zodSchema, EvaluatorDetailResponseCompactOutputType$zodSchema, EvaluatorDetailResponseCompact$zodSchema;
+var init_evaluatordetailresponsecompact = __esm(() => {
   init_zod();
-  init_evaluatorversionresponse();
-  EvaluatorDetailResponseEvaluatorType$zodSchema = _enum([
+  init_evaluatorversioncompact();
+  EvaluatorDetailResponseCompactEvaluatorType$zodSchema = _enum([
     "tts",
     "stt",
     "llm",
     "llm-general",
     "conversation"
   ]).describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n");
-  EvaluatorDetailResponseDataType$zodSchema = _enum([
+  EvaluatorDetailResponseCompactDataType$zodSchema = _enum([
     "text",
     "audio"
   ]).describe("The modality the judge reads:\n\n- `text`\n- `audio`\n");
-  EvaluatorDetailResponseOutputType$zodSchema = _enum([
+  EvaluatorDetailResponseCompactOutputType$zodSchema = _enum([
     "binary",
     "rating"
   ]).describe("How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n");
-  EvaluatorDetailResponse$zodSchema = object({
+  EvaluatorDetailResponseCompact$zodSchema = object({
     created_at: string2().describe("When the evaluator was created (ISO 8601 UTC)"),
-    data_type: EvaluatorDetailResponseDataType$zodSchema.describe("The modality the judge reads:\n\n- `text`\n- `audio`\n"),
+    data_type: EvaluatorDetailResponseCompactDataType$zodSchema.describe("The modality the judge reads:\n\n- `text`\n- `audio`\n"),
     description: string2().nullable().optional().describe("What the evaluator checks"),
-    evaluator_type: EvaluatorDetailResponseEvaluatorType$zodSchema.describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n"),
+    evaluator_type: EvaluatorDetailResponseCompactEvaluatorType$zodSchema.describe("What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n"),
     is_default: boolean2().describe("True for a built-in default evaluator, which you can't edit. False for an evaluator you created, which you can edit and add versions to"),
     live_version_id: string2().nullable().optional().describe("ID of the version that is currently live"),
     name: string2().describe("Evaluator name"),
-    output_type: EvaluatorDetailResponseOutputType$zodSchema.describe("How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n"),
+    output_type: EvaluatorDetailResponseCompactOutputType$zodSchema.describe("How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n"),
     slug: string2().nullable().optional().describe("Stable slug for a built-in default evaluator"),
     updated_at: string2().describe("When the evaluator was last updated (ISO 8601 UTC)"),
     uuid: string2().describe("Evaluator ID"),
-    versions: array(EvaluatorVersionResponse$zodSchema).describe("Full version history, oldest first")
+    versions: array(EvaluatorVersionCompact$zodSchema).describe("Full version history, oldest first")
   });
 });
 
@@ -57446,14 +57471,15 @@ var init_evaluatordetailresponse = __esm(() => {
 var GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetRequest$zodSchema, GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetResponse$zodSchema;
 var init_getevaluatorendpointevaluatorsevaluatoruuidgetop = __esm(() => {
   init_zod();
-  init_evaluatordetailresponse();
+  init_evaluatordetailresponsecompact();
   init_httpvalidationerror();
   GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetRequest$zodSchema = object({
+    compact: boolean2().default(false).describe("Return a compact response that omits heavy detail fields (`versions.system_prompt`, `versions.output_config`, `versions.variables`), keeping only the lightweight decision fields. Omit for full detail"),
     evaluator_uuid: string2().describe("Evaluator to retrieve"),
     xAPIKey: string2().nullable().optional()
   });
   GetEvaluatorEndpointEvaluatorsEvaluatorUuidGetResponse$zodSchema = union([
-    EvaluatorDetailResponse$zodSchema,
+    EvaluatorDetailResponseCompact$zodSchema,
     HTTPValidationError$zodSchema
   ]);
 });
@@ -57476,6 +57502,9 @@ async function $do27(client$, request, options) {
     })
   };
   const path$ = pathToFunc("/evaluators/{evaluator_uuid}")(pathParams$);
+  const query$ = encodeFormQuery({
+    compact: payload$.compact
+  });
   const headers$ = new Headers(compactMap({
     Accept: "application/json",
     "X-API-Key": encodeSimple("X-API-Key", payload$.xAPIKey, {
@@ -57507,6 +57536,7 @@ async function $do27(client$, request, options) {
     baseURL: options?.serverURL,
     path: path$,
     headers: headers$,
+    query: query$,
     body: body$,
     userAgent: client$._options.userAgent,
     timeoutMs: options?.timeoutMs || client$._options.timeoutMs || -1
@@ -58636,7 +58666,7 @@ fields to change (name, config). Returns the updated test. Fails with
 function createMCPServer(deps) {
   const server = new McpServer({
     name: "CalibrateMcp",
-    version: "0.0.24"
+    version: "0.0.26"
   });
   const getClient = deps.getSDK || (() => new CalibrateMcpCore({
     security: deps.security,
@@ -58681,7 +58711,7 @@ function createMCPServer(deps) {
   tool(tool$annotationTasksCreate);
   tool(tool$annotationTasksList);
   tool(tool$annotationTasksGet);
-  tool(tool$annotationTasksLinkEvaluator);
+  tool(tool$annotationTasksSetEvaluators);
   tool(tool$annotationTasksAddItems);
   tool(tool$annotationTasksUpdateItems);
   tool(tool$annotationTasksCreateEvaluatorRun);
@@ -58719,8 +58749,8 @@ var init_server2 = __esm(() => {
   init_annotationTasksGetAgreement2();
   init_annotationTasksGetEvaluatorRun2();
   init_annotationTasksGetSummary2();
-  init_annotationTasksLinkEvaluator2();
   init_annotationTasksList2();
+  init_annotationTasksSetEvaluators2();
   init_annotationTasksUpdateItems2();
   init_evaluatorsCreate2();
   init_evaluatorsCreateVersion2();
@@ -58902,8 +58932,12 @@ run-agent-tests-batch.
 `
     },
     {
-      name: "link-evaluator-to-task",
-      description: `Link an evaluator to an annotation task, appending it to the display order.
+      name: "set-task-evaluators",
+      description: `Replace the task's linked evaluators with exactly the given ordered
+set: new ids are linked, omitted ids are unlinked, and the list order
+sets their display position. Send an empty list to unlink all. Each
+evaluator must be one you created or a built-in default, and the task
+must be in your workspace.
 `
     },
     {
@@ -59842,7 +59876,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
         <h1>Instructions</h1>
         <p>One-click installation for Claude Desktop users</p>
         <div class="instruction-item">
-          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.24/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
+          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.26/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
             \uD83D\uDCE5 Download MCP Bundle
           </a>
         </div>
@@ -62494,7 +62528,7 @@ var routes = buildRouteMap({
 var app = buildApplication(routes, {
   name: "mcp",
   versionInfo: {
-    currentVersion: "0.0.24"
+    currentVersion: "0.0.26"
   }
 });
 run(app, process4.argv.slice(2), buildContext(process4));
@@ -62502,5 +62536,5 @@ export {
   app
 };
 
-//# debugId=F30249DEC7355B7B64756E2164756E21
+//# debugId=DE4BD7689C1357ED64756E2164756E21
 //# sourceMappingURL=mcp-server.js.map