@dalmia/calibrate-mcp 0.0.85 → 0.0.86

package/bin/mcp-server.js

@@ -53279,9 +53279,9 @@ var init_config = __esm(() => {
   SDK_METADATA = {
     language: "typescript",
     openapiDocVersion: "0.1.0",
-    sdkVersion: "0.0.85",
+    sdkVersion: "0.0.86",
     genVersion: "2.915.1",
-    userAgent: "speakeasy-sdk/mcp-typescript 0.0.85 2.915.1 0.1.0 @dalmia/calibrate-mcp"
+    userAgent: "speakeasy-sdk/mcp-typescript 0.0.86 2.915.1 0.1.0 @dalmia/calibrate-mcp"
   };
 });
 
@@ -58958,6 +58958,82 @@ var init_annotationTasksGet2 = __esm(() => {
   };
 });
 
+// src/models/answersummary.ts
+var AnswerSummary$zodSchema;
+var init_answersummary = __esm(() => {
+  init_zod();
+  AnswerSummary$zodSchema = object({
+    count: int().describe("How many answers there are"),
+    mean: number2().nullable().describe("Mean of the rating answers"),
+    true_count: int().nullable().describe("How many pass/fail answers are a pass")
+  });
+});
+
+// src/models/interannotatoragreement.ts
+var InterAnnotatorAgreementKappaType$zodSchema, InterAnnotatorAgreementKappaBlankReason$zodSchema, InterAnnotatorAgreement$zodSchema;
+var init_interannotatoragreement = __esm(() => {
+  init_zod();
+  InterAnnotatorAgreementKappaType$zodSchema = _enum([
+    "fleiss",
+    "linear_weighted_fleiss",
+    "cohen",
+    "linear_weighted_cohen"
+  ]);
+  InterAnnotatorAgreementKappaBlankReason$zodSchema = _enum([
+    "too_few_items",
+    "everyone_agreed"
+  ]);
+  InterAnnotatorAgreement$zodSchema = object({
+    chance_agreement: number2().nullable().describe("Agreement expected by chance alone, which `kappa` corrects for, on the same scale as the agreement"),
+    current: number2().nullable().describe("Current mean pairwise agreement among annotators on this evaluator"),
+    kappa: number2().nullable().describe("Agreement corrected for the agreement expected by chance, where 1 is perfect agreement, 0 is chance level, and below 0 is worse than chance"),
+    kappa_blank_reason: InterAnnotatorAgreementKappaBlankReason$zodSchema.nullable().describe("Why `kappa` is blank when answers were compared:\n\n- `too_few_items`: every evaluator compared these answers on only one item, and one item cannot show the agreement expected by chance\n- `everyone_agreed`: every compared answer is the same, so agreement cannot be told apart from chance\n"),
+    kappa_type: InterAnnotatorAgreementKappaType$zodSchema.nullable().describe("The kind of kappa reported in `kappa`:\n\n- `fleiss`: Fleiss' kappa between three or more annotators\n- `linear_weighted_fleiss`: Fleiss' kappa between three or more annotators, giving partial credit to close ratings\n- `cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others\n- `linear_weighted_cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others, giving partial credit to close ratings\n"),
+    pair_count: int().describe("How many annotator pairs the current value averages over")
+  });
+});
+
+// src/models/evaluatoragreement.ts
+var EvaluatorAgreementKappaType$zodSchema, EvaluatorAgreementKappaBlankReason$zodSchema, EvaluatorAgreement$zodSchema;
+var init_evaluatoragreement = __esm(() => {
+  init_zod();
+  init_answersummary();
+  init_interannotatoragreement();
+  EvaluatorAgreementKappaType$zodSchema = _enum([
+    "fleiss",
+    "linear_weighted_fleiss",
+    "cohen",
+    "linear_weighted_cohen"
+  ]);
+  EvaluatorAgreementKappaBlankReason$zodSchema = _enum([
+    "too_few_items",
+    "everyone_agreed"
+  ]);
+  EvaluatorAgreement$zodSchema = object({
+    chance_agreement: number2().nullable().describe("Agreement expected by chance alone, which `kappa` corrects for, on the same scale as the agreement"),
+    current: number2().nullable().describe("Current mean agreement between this evaluator's judge and the annotators"),
+    evaluator_id: string2().describe("ID of the evaluator"),
+    human_result: AnswerSummary$zodSchema.nullable().describe("The annotators' answers for this evaluator"),
+    inter_annotator: InterAnnotatorAgreement$zodSchema,
+    kappa: number2().nullable().describe("Agreement corrected for the agreement expected by chance, where 1 is perfect agreement, 0 is chance level, and below 0 is worse than chance"),
+    kappa_blank_reason: EvaluatorAgreementKappaBlankReason$zodSchema.nullable().describe("Why `kappa` is blank when answers were compared:\n\n- `too_few_items`: every evaluator compared these answers on only one item, and one item cannot show the agreement expected by chance\n- `everyone_agreed`: every compared answer is the same, so agreement cannot be told apart from chance\n"),
+    kappa_type: EvaluatorAgreementKappaType$zodSchema.nullable().describe("The kind of kappa reported in `kappa`:\n\n- `fleiss`: Fleiss' kappa between three or more annotators\n- `linear_weighted_fleiss`: Fleiss' kappa between three or more annotators, giving partial credit to close ratings\n- `cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others\n- `linear_weighted_cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others, giving partial credit to close ratings\n"),
+    name: string2().nullable().describe("Name of the evaluator"),
+    pair_count: int().describe("How many judge and annotator pairs the current value averages over"),
+    result: AnswerSummary$zodSchema
+  });
+});
+
+// src/models/evaluatorswithhumansblock.ts
+var EvaluatorsWithHumansBlock$zodSchema;
+var init_evaluatorswithhumansblock = __esm(() => {
+  init_zod();
+  EvaluatorsWithHumansBlock$zodSchema = object({
+    current: number2().nullable().describe("Current mean agreement between the evaluators and the annotators, weighted by how many pairs each evaluator compared"),
+    pair_count: int().describe("How many evaluator and annotator pairs the current value averages over")
+  });
+});
+
 // src/models/humanagreementblock.ts
 var HumanAgreementBlock$zodSchema, HumanAgreementBlock$zodSchemaOutbound;
 var init_humanagreementblock = __esm(() => {
@@ -58965,14 +59041,12 @@ var init_humanagreementblock = __esm(() => {
   init_schemas4();
   HumanAgreementBlock$zodSchema = collectExtraKeys(object({
     current: number2().nullable().describe("Current mean pairwise agreement among annotators"),
-    pair_count: int().describe("How many annotator pairs the current value averages over"),
-    series: array(record(string2(), any())).describe("Agreement over time, one point per time bucket")
+    pair_count: int().describe("How many annotator pairs the current value averages over")
   }).catchall(any()), "AdditionalProperties", true);
   HumanAgreementBlock$zodSchemaOutbound = object({
     AdditionalProperties: record(string2(), any()).optional(),
     current: nullable(number2()),
-    pair_count: int(),
-    series: array(record(string2(), any()))
+    pair_count: int()
   }).transform((v) => {
     const { AdditionalProperties, ...rest } = v;
     return {
@@ -58987,19 +59061,19 @@ var TaskAgreementResponse$zodSchema, TaskAgreementResponse$zodSchemaOutbound;
 var init_taskagreementresponse = __esm(() => {
   init_zod();
   init_schemas4();
+  init_evaluatoragreement();
+  init_evaluatorswithhumansblock();
   init_humanagreementblock();
   TaskAgreementResponse$zodSchema = collectExtraKeys(object({
-    bucket: string2().describe("Time bucket used for the trend series"),
-    days: int().describe("Length of the trailing window in days"),
-    evaluators: array(record(string2(), any())).describe("Agreement between each evaluator and the annotators, plus that evaluator's own results and the human results, one entry per linked evaluator"),
+    evaluators: array(EvaluatorAgreement$zodSchema).describe("Agreement between each evaluator and the annotators, plus that evaluator's own results and the human results, one entry per linked evaluator"),
+    evaluators_with_humans: EvaluatorsWithHumansBlock$zodSchema,
     human_human: HumanAgreementBlock$zodSchema,
     task_id: string2().describe("ID of the task")
   }).catchall(any()), "AdditionalProperties", true);
   TaskAgreementResponse$zodSchemaOutbound = object({
     AdditionalProperties: record(string2(), any()).optional(),
-    bucket: string2(),
-    days: int(),
-    evaluators: array(record(string2(), any())),
+    evaluators: array(EvaluatorAgreement$zodSchema),
+    evaluators_with_humans: EvaluatorsWithHumansBlock$zodSchema,
     human_human: HumanAgreementBlock$zodSchemaOutbound,
     task_id: string2()
   }).transform((v) => {
@@ -59012,19 +59086,12 @@ var init_taskagreementresponse = __esm(() => {
 });
 
 // src/models/taskagreementannotationtaskstaskuuidagreementgetop.ts
-var Bucket$zodSchema, TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest$zodSchema, TaskAgreementAnnotationTasksTaskUuidAgreementGetResponse$zodSchema;
+var TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest$zodSchema, TaskAgreementAnnotationTasksTaskUuidAgreementGetResponse$zodSchema;
 var init_taskagreementannotationtaskstaskuuidagreementgetop = __esm(() => {
   init_zod();
   init_httpvalidationerror();
   init_taskagreementresponse();
-  Bucket$zodSchema = _enum([
-    "week",
-    "month",
-    "year"
-  ]).describe("How to bucket points in the trend series");
   TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest$zodSchema = object({
-    bucket: Bucket$zodSchema.default("week").describe("How to bucket points in the trend series"),
-    days: int().default(90).describe("Trailing window in days for the trend series"),
     task_uuid: string2().describe("Annotation task to act on"),
     xAPIKey: string2().nullable().optional()
   });
@@ -59052,10 +59119,6 @@ async function $do24(client$, request, options) {
     })
   };
   const path$ = pathToFunc("/annotation-tasks/{task_uuid}/agreement")(pathParams$);
-  const query$ = encodeFormQuery({
-    bucket: payload$.bucket,
-    days: payload$.days
-  });
   const headers$ = new Headers(compactMap({
     Accept: "application/json",
     "X-API-Key": encodeSimple("X-API-Key", payload$.xAPIKey, {
@@ -59087,7 +59150,6 @@ async function $do24(client$, request, options) {
     baseURL: options?.serverURL,
     path: path$,
     headers: headers$,
-    query: query$,
     body: body$,
     userAgent: client$._options.userAgent,
     timeoutMs: options?.timeoutMs || client$._options.timeoutMs || -1
@@ -59132,7 +59194,7 @@ var init_annotationTasksGetAgreement2 = __esm(() => {
   };
   tool$annotationTasksGetAgreement = {
     name: "get-annotation-agreement",
-    description: `Get human-vs-human and human-vs-evaluator agreement metrics for a task, with a trend series.
+    description: `Get human-vs-human and human-vs-evaluator agreement metrics for a task, with a chance-corrected kappa for each evaluator.
 `,
     scopes: ["read"],
     annotations: {
@@ -60175,14 +60237,45 @@ var init_annotatorsCreate2 = __esm(() => {
   };
 });
 
+// src/models/kappaofanswerkind.ts
+var KappaOfAnswerKindKappaType$zodSchema, KappaOfAnswerKindKappaBlankReason$zodSchema, AnswerKind$zodSchema, KappaOfAnswerKind$zodSchema;
+var init_kappaofanswerkind = __esm(() => {
+  init_zod();
+  KappaOfAnswerKindKappaType$zodSchema = _enum([
+    "fleiss",
+    "linear_weighted_fleiss",
+    "cohen",
+    "linear_weighted_cohen"
+  ]);
+  KappaOfAnswerKindKappaBlankReason$zodSchema = _enum([
+    "too_few_items",
+    "everyone_agreed"
+  ]);
+  AnswerKind$zodSchema = _enum([
+    "binary",
+    "rating"
+  ]).describe("Which answers this entry covers:\n\n- `binary`: answers that either match or do not, such as pass or fail\n- `rating`: ratings on a scale\n");
+  KappaOfAnswerKind$zodSchema = object({
+    agreement: number2().nullable().describe("Mean agreement with the other annotators over the pairs `kappa` is worked out from, from 0 to 1. When `kappa_blank_reason` is `too_few_items`, over every compared pair of this kind"),
+    answer_kind: AnswerKind$zodSchema.describe("Which answers this entry covers:\n\n- `binary`: answers that either match or do not, such as pass or fail\n- `rating`: ratings on a scale\n"),
+    chance_agreement: number2().nullable().describe("Agreement expected by chance alone, which `kappa` corrects for, on the same scale as the agreement"),
+    kappa: number2().nullable().describe("Agreement corrected for the agreement expected by chance, where 1 is perfect agreement, 0 is chance level, and below 0 is worse than chance"),
+    kappa_blank_reason: KappaOfAnswerKindKappaBlankReason$zodSchema.nullable().describe("Why `kappa` is blank when answers were compared:\n\n- `too_few_items`: every evaluator compared these answers on only one item, and one item cannot show the agreement expected by chance\n- `everyone_agreed`: every compared answer is the same, so agreement cannot be told apart from chance\n"),
+    kappa_type: KappaOfAnswerKindKappaType$zodSchema.nullable().describe("The kind of kappa reported in `kappa`:\n\n- `fleiss`: Fleiss' kappa between three or more annotators\n- `linear_weighted_fleiss`: Fleiss' kappa between three or more annotators, giving partial credit to close ratings\n- `cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others\n- `linear_weighted_cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others, giving partial credit to close ratings\n"),
+    pair_count: int().describe("How many compared pairs `agreement` averages over")
+  });
+});
+
 // src/models/annotatorresponse.ts
 var AnnotatorResponse$zodSchema;
 var init_annotatorresponse = __esm(() => {
   init_zod();
+  init_kappaofanswerkind();
   AnnotatorResponse$zodSchema = object({
     created_at: string2().describe("When the annotator was created (ISO 8601 UTC)"),
     current_agreement: number2().nullable().optional().describe("Latest pairwise mean agreement with other annotators, from 0 to 1"),
     jobs_count: int().nullable().optional().describe("Number of labelling jobs assigned to this annotator"),
+    kappa_by_kind: array(KappaOfAnswerKind$zodSchema).nullable().optional().describe("Agreement with the other annotators and its kappa, one entry for each kind of answer compared, binary answers first. The kinds are never combined into one kappa"),
     name: string2().describe("Annotator name"),
     pair_count: int().nullable().optional().describe("Number of comparable annotation pairs behind the current agreement score"),
     updated_at: string2().describe("When the annotator was last updated (ISO 8601 UTC)"),
@@ -62954,7 +63047,7 @@ last. Requires a Calibrate API key (CALIBRATE_API_KEY).
 function createMCPServer(deps) {
   const server = new McpServer({
     name: "CalibrateMcp",
-    version: "0.0.85"
+    version: "0.0.86"
   });
   const getClient = deps.getSDK || (() => new CalibrateMcpCore({
     security: deps.security,
@@ -63332,7 +63425,7 @@ must be in your workspace.
     },
     {
       name: "get-annotation-agreement",
-      description: `Get human-vs-human and human-vs-evaluator agreement metrics for a task, with a trend series.
+      description: `Get human-vs-human and human-vs-evaluator agreement metrics for a task, with a chance-corrected kappa for each evaluator.
 `
     },
     {
@@ -64279,7 +64372,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
         <h1>Instructions</h1>
         <p>One-click installation for Claude Desktop users</p>
         <div class="instruction-item">
-          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.85/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
+          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.86/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
             \uD83D\uDCE5 Download MCP Bundle
           </a>
         </div>
@@ -67233,7 +67326,7 @@ var routes = buildRouteMap({
 var app = buildApplication(routes, {
   name: "mcp",
   versionInfo: {
-    currentVersion: "0.0.85"
+    currentVersion: "0.0.86"
   }
 });
 run(app, process3.argv.slice(2), buildContext(process3));
@@ -67241,5 +67334,5 @@ export {
   app
 };
 
-//# debugId=A1741B450B8DD19464756E2164756E21
+//# debugId=F390FB93EA6317C264756E2164756E21
 //# sourceMappingURL=mcp-server.js.map