@dalmia/calibrate-mcp 0.0.84 → 0.0.86

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/bin/mcp-server.js +152 -46
  2. package/bin/mcp-server.js.map +23 -18
  3. package/esm/funcs/annotationTasksGetAgreement.js +1 -6
  4. package/esm/funcs/annotationTasksGetAgreement.js.map +1 -1
  5. package/esm/landing-page.js +1 -1
  6. package/esm/lib/config.d.ts +2 -2
  7. package/esm/lib/config.js +2 -2
  8. package/esm/mcp-server/mcp-server.js +1 -1
  9. package/esm/mcp-server/server.js +1 -1
  10. package/esm/mcp-server/tools/annotationTasksGetAgreement.js +1 -1
  11. package/esm/models/annotatorresponse.d.ts +2 -0
  12. package/esm/models/annotatorresponse.d.ts.map +1 -1
  13. package/esm/models/annotatorresponse.js +3 -0
  14. package/esm/models/annotatorresponse.js.map +1 -1
  15. package/esm/models/answersummary.d.ts +8 -0
  16. package/esm/models/answersummary.d.ts.map +1 -0
  17. package/esm/models/answersummary.js +10 -0
  18. package/esm/models/answersummary.js.map +1 -0
  19. package/esm/models/evaluatoragreement.d.ts +41 -0
  20. package/esm/models/evaluatoragreement.d.ts.map +1 -0
  21. package/esm/models/evaluatoragreement.js +42 -0
  22. package/esm/models/evaluatoragreement.js.map +1 -0
  23. package/esm/models/evaluatorswithhumansblock.d.ts +7 -0
  24. package/esm/models/evaluatorswithhumansblock.d.ts.map +1 -0
  25. package/esm/models/evaluatorswithhumansblock.js +9 -0
  26. package/esm/models/evaluatorswithhumansblock.js.map +1 -0
  27. package/esm/models/humanagreementblock.d.ts +0 -3
  28. package/esm/models/humanagreementblock.d.ts.map +1 -1
  29. package/esm/models/humanagreementblock.js +0 -2
  30. package/esm/models/humanagreementblock.js.map +1 -1
  31. package/esm/models/interannotatoragreement.d.ts +34 -0
  32. package/esm/models/interannotatoragreement.d.ts.map +1 -0
  33. package/esm/models/interannotatoragreement.js +34 -0
  34. package/esm/models/interannotatoragreement.js.map +1 -0
  35. package/esm/models/kappaofanswerkind.d.ts +60 -0
  36. package/esm/models/kappaofanswerkind.d.ts.map +1 -0
  37. package/esm/models/kappaofanswerkind.js +52 -0
  38. package/esm/models/kappaofanswerkind.js.map +1 -0
  39. package/esm/models/taskagreementannotationtaskstaskuuidagreementgetop.d.ts +0 -20
  40. package/esm/models/taskagreementannotationtaskstaskuuidagreementgetop.d.ts.map +1 -1
  41. package/esm/models/taskagreementannotationtaskstaskuuidagreementgetop.js +0 -15
  42. package/esm/models/taskagreementannotationtaskstaskuuidagreementgetop.js.map +1 -1
  43. package/esm/models/taskagreementresponse.d.ts +4 -5
  44. package/esm/models/taskagreementresponse.d.ts.map +1 -1
  45. package/esm/models/taskagreementresponse.js +6 -6
  46. package/esm/models/taskagreementresponse.js.map +1 -1
  47. package/esm/tool-names.js +1 -1
  48. package/esm/tool-names.js.map +1 -1
  49. package/package.json +1 -1
  50. package/src/funcs/annotationTasksGetAgreement.ts +1 -6
  51. package/src/landing-page.ts +1 -1
  52. package/src/lib/config.ts +2 -2
  53. package/src/mcp-server/mcp-server.ts +1 -1
  54. package/src/mcp-server/server.ts +1 -1
  55. package/src/mcp-server/tools/annotationTasksGetAgreement.ts +1 -1
  56. package/src/models/annotatorresponse.ts +9 -0
  57. package/src/models/answersummary.ts +19 -0
  58. package/src/models/evaluatoragreement.ts +85 -0
  59. package/src/models/evaluatorswithhumansblock.ts +21 -0
  60. package/src/models/humanagreementblock.ts +0 -5
  61. package/src/models/interannotatoragreement.ts +69 -0
  62. package/src/models/kappaofanswerkind.ts +101 -0
  63. package/src/models/taskagreementannotationtaskstaskuuidagreementgetop.ts +0 -28
  64. package/src/models/taskagreementresponse.ts +14 -9
  65. package/src/tool-names.ts +1 -1
package/bin/mcp-server.js CHANGED
@@ -25889,7 +25889,8 @@ var require_formats = __commonJS(function(exports, module) {
25889
25889
  var percentTwenties = /%20/g;
25890
25890
  var Format = {
25891
25891
  RFC1738: "RFC1738",
25892
- RFC3986: "RFC3986"
25892
+ RFC3986: "RFC3986",
25893
+ WHATWG: "WHATWG"
25893
25894
  };
25894
25895
  module.exports = {
25895
25896
  default: Format.RFC3986,
@@ -25899,10 +25900,14 @@ var require_formats = __commonJS(function(exports, module) {
25899
25900
  },
25900
25901
  RFC3986: function(value) {
25901
25902
  return String(value);
25903
+ },
25904
+ WHATWG: function(value) {
25905
+ return replace.call(value, percentTwenties, "+");
25902
25906
  }
25903
25907
  },
25904
25908
  RFC1738: Format.RFC1738,
25905
- RFC3986: Format.RFC3986
25909
+ RFC3986: Format.RFC3986,
25910
+ WHATWG: Format.WHATWG
25906
25911
  };
25907
25912
  });
25908
25913
 
@@ -26206,7 +26211,7 @@ var require_utils2 = __commonJS(function(exports, module) {
26206
26211
  var arr = [];
26207
26212
  for (var i = 0;i < segment.length; ++i) {
26208
26213
  var c = segment.charCodeAt(i);
26209
- if (c === 45 || c === 46 || c === 95 || c === 126 || c >= 48 && c <= 57 || c >= 65 && c <= 90 || c >= 97 && c <= 122 || format === formats.RFC1738 && (c === 40 || c === 41)) {
26214
+ if (c === 45 || c === 46 || c === 95 || c === 126 && format !== formats.WHATWG || c >= 48 && c <= 57 || c >= 65 && c <= 90 || c >= 97 && c <= 122 || format === formats.RFC1738 && (c === 40 || c === 41) || format === formats.WHATWG && c === 42) {
26210
26215
  arr[arr.length] = segment.charAt(i);
26211
26216
  continue;
26212
26217
  }
@@ -26222,9 +26227,14 @@ var require_utils2 = __commonJS(function(exports, module) {
26222
26227
  arr[arr.length] = hexTable[224 | c >> 12] + hexTable[128 | c >> 6 & 63] + hexTable[128 | c & 63];
26223
26228
  continue;
26224
26229
  }
26225
- i += 1;
26226
- c = 65536 + ((c & 1023) << 10 | segment.charCodeAt(i) & 1023);
26227
- arr[arr.length] = hexTable[240 | c >> 18] + hexTable[128 | c >> 12 & 63] + hexTable[128 | c >> 6 & 63] + hexTable[128 | c & 63];
26230
+ var next = segment.charCodeAt(i + 1);
26231
+ if (c <= 56319 && next >= 56320 && next <= 57343) {
26232
+ i += 1;
26233
+ c = 65536 + ((c & 1023) << 10 | next & 1023);
26234
+ arr[arr.length] = hexTable[240 | c >> 18] + hexTable[128 | c >> 12 & 63] + hexTable[128 | c >> 6 & 63] + hexTable[128 | c & 63];
26235
+ } else {
26236
+ arr[arr.length] = "%EF%BF%BD";
26237
+ }
26228
26238
  }
26229
26239
  out += arr.join("");
26230
26240
  }
@@ -26304,6 +26314,7 @@ var require_utils2 = __commonJS(function(exports, module) {
26304
26314
  markOverflow,
26305
26315
  maybeMap,
26306
26316
  merge,
26317
+ setProperty,
26307
26318
  toCoercible
26308
26319
  };
26309
26320
  });
@@ -26428,7 +26439,7 @@ var require_stringify = __commonJS(function(exports, module) {
26428
26439
  var encodedPrefix = String(prefix);
26429
26440
  var adjustedPrefix = commaRoundTrip && isArray(obj) && obj.length === 1 ? encodedPrefix + "[]" : encodedPrefix;
26430
26441
  if (allowEmptyArrays && isArray(obj) && obj.length === 0 && Object.keys(obj).length === 0) {
26431
- return adjustedPrefix + "[]";
26442
+ return formatter(encoder && !encodeValuesOnly ? encoder(adjustedPrefix, defaults.encoder, charset, "key", format) : adjustedPrefix) + "[]";
26432
26443
  }
26433
26444
  for (var j = 0;j < objKeys.length; ++j) {
26434
26445
  var key = objKeys[j];
@@ -26518,7 +26529,7 @@ var require_stringify = __commonJS(function(exports, module) {
26518
26529
  filter = options.filter;
26519
26530
  obj = filter("", obj);
26520
26531
  } else if (isArray(options.filter)) {
26521
- filter = options.filter;
26532
+ filter = options.filter.slice();
26522
26533
  objKeys = filter;
26523
26534
  }
26524
26535
  var keys = [];
@@ -26544,7 +26555,7 @@ var require_stringify = __commonJS(function(exports, module) {
26544
26555
  continue;
26545
26556
  }
26546
26557
  var encodedKey = options.encodeDotInKeys ? String(key).replace(/\./g, "%2E") : String(key);
26547
- pushToArray(keys, stringify(value, encodedKey, generateArrayPrefix, commaRoundTrip, options.allowEmptyArrays, options.strictNullHandling, options.skipNulls, options.encodeDotInKeys, options.encode ? options.encoder : null, options.filter, options.sort, options.allowDots, options.serializeDate, options.format, options.formatter, options.encodeValuesOnly, options.charset, sideChannel, options.depth, 0));
26558
+ pushToArray(keys, stringify(value, encodedKey, generateArrayPrefix, commaRoundTrip, options.allowEmptyArrays, options.strictNullHandling, options.skipNulls, options.encodeDotInKeys, options.encode ? options.encoder : null, filter, options.sort, options.allowDots, options.serializeDate, options.format, options.formatter, options.encodeValuesOnly, options.charset, sideChannel, options.depth, 0));
26548
26559
  }
26549
26560
  var joined = keys.join(options.delimiter);
26550
26561
  var prefix = options.addQueryPrefix === true ? "?" : "";
@@ -26600,9 +26611,9 @@ var require_parse = __commonJS(function(exports, module) {
26600
26611
  copies.set(value, copy);
26601
26612
  queue[queue.length] = { source: value, target: copy };
26602
26613
  }
26603
- target[keys[j]] = copies.get(value);
26614
+ utils.setProperty(target, keys[j], copies.get(value));
26604
26615
  } else {
26605
- target[keys[j]] = value;
26616
+ utils.setProperty(target, keys[j], value);
26606
26617
  }
26607
26618
  }
26608
26619
  }
@@ -26652,7 +26663,7 @@ var require_parse = __commonJS(function(exports, module) {
26652
26663
  }
26653
26664
  return val.split(",");
26654
26665
  }
26655
- if (options.throwOnLimitExceeded && currentArrayLength >= options.arrayLimit) {
26666
+ if (options.throwOnLimitExceeded && currentArrayLength >= options.arrayLimit && currentArrayLength > 0) {
26656
26667
  throw new RangeError("Array limit exceeded. Only " + options.arrayLimit + " element" + (options.arrayLimit === 1 ? "" : "s") + " allowed in an array.");
26657
26668
  }
26658
26669
  return val;
@@ -26700,13 +26711,15 @@ var require_parse = __commonJS(function(exports, module) {
26700
26711
  } else {
26701
26712
  key = options.decoder(part.slice(0, pos), defaults.decoder, charset, "key");
26702
26713
  if (key !== null) {
26703
- val = utils.maybeMap(parseArrayValue(part.slice(pos + 1), options, isArray(obj[key]) ? obj[key].length : 0), function(encodedVal) {
26714
+ val = utils.maybeMap(parseArrayValue(part.slice(pos + 1), options, isArray(obj[key]) && (options.duplicates === "combine" || isEmptyBracketKey) ? obj[key].length : 0), function(encodedVal) {
26704
26715
  return options.decoder(encodedVal, defaults.decoder, charset, "value");
26705
26716
  });
26706
26717
  }
26707
26718
  }
26708
26719
  if (val && options.interpretNumericEntities && charset === "iso-8859-1") {
26709
- val = interpretNumericEntities(String(val));
26720
+ val = utils.maybeMap(val, function(value) {
26721
+ return value ? interpretNumericEntities(String(value)) : value;
26722
+ });
26710
26723
  }
26711
26724
  if (isEmptyBracketKey) {
26712
26725
  val = isArray(val) ? [val] : val;
@@ -26744,7 +26757,7 @@ var require_parse = __commonJS(function(exports, module) {
26744
26757
  } else {
26745
26758
  obj = options.plainObjects ? { __proto__: null } : {};
26746
26759
  var cleanRoot = root.charAt(0) === "[" && root.charAt(root.length - 1) === "]" ? root.slice(1, -1) : root;
26747
- var decodedRoot = options.decodeDotInKeys ? cleanRoot.replace(/%2E/g, ".") : cleanRoot;
26760
+ var decodedRoot = options.decodeDotInKeys ? cleanRoot.replace(/%2E/gi, ".") : cleanRoot;
26748
26761
  var index = parseInt(decodedRoot, 10);
26749
26762
  var isValidArrayIndex = !isNaN(index) && root !== decodedRoot && String(index) === decodedRoot && index >= 0 && options.parseArrays;
26750
26763
  if (!options.parseArrays && decodedRoot === "") {
@@ -53266,9 +53279,9 @@ var init_config = __esm(() => {
53266
53279
  SDK_METADATA = {
53267
53280
  language: "typescript",
53268
53281
  openapiDocVersion: "0.1.0",
53269
- sdkVersion: "0.0.84",
53282
+ sdkVersion: "0.0.86",
53270
53283
  genVersion: "2.915.1",
53271
- userAgent: "speakeasy-sdk/mcp-typescript 0.0.84 2.915.1 0.1.0 @dalmia/calibrate-mcp"
53284
+ userAgent: "speakeasy-sdk/mcp-typescript 0.0.86 2.915.1 0.1.0 @dalmia/calibrate-mcp"
53272
53285
  };
53273
53286
  });
53274
53287
 
@@ -58945,6 +58958,82 @@ var init_annotationTasksGet2 = __esm(() => {
58945
58958
  };
58946
58959
  });
58947
58960
 
58961
+ // src/models/answersummary.ts
58962
+ var AnswerSummary$zodSchema;
58963
+ var init_answersummary = __esm(() => {
58964
+ init_zod();
58965
+ AnswerSummary$zodSchema = object({
58966
+ count: int().describe("How many answers there are"),
58967
+ mean: number2().nullable().describe("Mean of the rating answers"),
58968
+ true_count: int().nullable().describe("How many pass/fail answers are a pass")
58969
+ });
58970
+ });
58971
+
58972
+ // src/models/interannotatoragreement.ts
58973
+ var InterAnnotatorAgreementKappaType$zodSchema, InterAnnotatorAgreementKappaBlankReason$zodSchema, InterAnnotatorAgreement$zodSchema;
58974
+ var init_interannotatoragreement = __esm(() => {
58975
+ init_zod();
58976
+ InterAnnotatorAgreementKappaType$zodSchema = _enum([
58977
+ "fleiss",
58978
+ "linear_weighted_fleiss",
58979
+ "cohen",
58980
+ "linear_weighted_cohen"
58981
+ ]);
58982
+ InterAnnotatorAgreementKappaBlankReason$zodSchema = _enum([
58983
+ "too_few_items",
58984
+ "everyone_agreed"
58985
+ ]);
58986
+ InterAnnotatorAgreement$zodSchema = object({
58987
+ chance_agreement: number2().nullable().describe("Agreement expected by chance alone, which `kappa` corrects for, on the same scale as the agreement"),
58988
+ current: number2().nullable().describe("Current mean pairwise agreement among annotators on this evaluator"),
58989
+ kappa: number2().nullable().describe("Agreement corrected for the agreement expected by chance, where 1 is perfect agreement, 0 is chance level, and below 0 is worse than chance"),
58990
+ kappa_blank_reason: InterAnnotatorAgreementKappaBlankReason$zodSchema.nullable().describe("Why `kappa` is blank when answers were compared:\n\n- `too_few_items`: every evaluator compared these answers on only one item, and one item cannot show the agreement expected by chance\n- `everyone_agreed`: every compared answer is the same, so agreement cannot be told apart from chance\n"),
58991
+ kappa_type: InterAnnotatorAgreementKappaType$zodSchema.nullable().describe("The kind of kappa reported in `kappa`:\n\n- `fleiss`: Fleiss' kappa between three or more annotators\n- `linear_weighted_fleiss`: Fleiss' kappa between three or more annotators, giving partial credit to close ratings\n- `cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others\n- `linear_weighted_cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others, giving partial credit to close ratings\n"),
58992
+ pair_count: int().describe("How many annotator pairs the current value averages over")
58993
+ });
58994
+ });
58995
+
58996
+ // src/models/evaluatoragreement.ts
58997
+ var EvaluatorAgreementKappaType$zodSchema, EvaluatorAgreementKappaBlankReason$zodSchema, EvaluatorAgreement$zodSchema;
58998
+ var init_evaluatoragreement = __esm(() => {
58999
+ init_zod();
59000
+ init_answersummary();
59001
+ init_interannotatoragreement();
59002
+ EvaluatorAgreementKappaType$zodSchema = _enum([
59003
+ "fleiss",
59004
+ "linear_weighted_fleiss",
59005
+ "cohen",
59006
+ "linear_weighted_cohen"
59007
+ ]);
59008
+ EvaluatorAgreementKappaBlankReason$zodSchema = _enum([
59009
+ "too_few_items",
59010
+ "everyone_agreed"
59011
+ ]);
59012
+ EvaluatorAgreement$zodSchema = object({
59013
+ chance_agreement: number2().nullable().describe("Agreement expected by chance alone, which `kappa` corrects for, on the same scale as the agreement"),
59014
+ current: number2().nullable().describe("Current mean agreement between this evaluator's judge and the annotators"),
59015
+ evaluator_id: string2().describe("ID of the evaluator"),
59016
+ human_result: AnswerSummary$zodSchema.nullable().describe("The annotators' answers for this evaluator"),
59017
+ inter_annotator: InterAnnotatorAgreement$zodSchema,
59018
+ kappa: number2().nullable().describe("Agreement corrected for the agreement expected by chance, where 1 is perfect agreement, 0 is chance level, and below 0 is worse than chance"),
59019
+ kappa_blank_reason: EvaluatorAgreementKappaBlankReason$zodSchema.nullable().describe("Why `kappa` is blank when answers were compared:\n\n- `too_few_items`: every evaluator compared these answers on only one item, and one item cannot show the agreement expected by chance\n- `everyone_agreed`: every compared answer is the same, so agreement cannot be told apart from chance\n"),
59020
+ kappa_type: EvaluatorAgreementKappaType$zodSchema.nullable().describe("The kind of kappa reported in `kappa`:\n\n- `fleiss`: Fleiss' kappa between three or more annotators\n- `linear_weighted_fleiss`: Fleiss' kappa between three or more annotators, giving partial credit to close ratings\n- `cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others\n- `linear_weighted_cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others, giving partial credit to close ratings\n"),
59021
+ name: string2().nullable().describe("Name of the evaluator"),
59022
+ pair_count: int().describe("How many judge and annotator pairs the current value averages over"),
59023
+ result: AnswerSummary$zodSchema
59024
+ });
59025
+ });
59026
+
59027
+ // src/models/evaluatorswithhumansblock.ts
59028
+ var EvaluatorsWithHumansBlock$zodSchema;
59029
+ var init_evaluatorswithhumansblock = __esm(() => {
59030
+ init_zod();
59031
+ EvaluatorsWithHumansBlock$zodSchema = object({
59032
+ current: number2().nullable().describe("Current mean agreement between the evaluators and the annotators, weighted by how many pairs each evaluator compared"),
59033
+ pair_count: int().describe("How many evaluator and annotator pairs the current value averages over")
59034
+ });
59035
+ });
59036
+
58948
59037
  // src/models/humanagreementblock.ts
58949
59038
  var HumanAgreementBlock$zodSchema, HumanAgreementBlock$zodSchemaOutbound;
58950
59039
  var init_humanagreementblock = __esm(() => {
@@ -58952,14 +59041,12 @@ var init_humanagreementblock = __esm(() => {
58952
59041
  init_schemas4();
58953
59042
  HumanAgreementBlock$zodSchema = collectExtraKeys(object({
58954
59043
  current: number2().nullable().describe("Current mean pairwise agreement among annotators"),
58955
- pair_count: int().describe("How many annotator pairs the current value averages over"),
58956
- series: array(record(string2(), any())).describe("Agreement over time, one point per time bucket")
59044
+ pair_count: int().describe("How many annotator pairs the current value averages over")
58957
59045
  }).catchall(any()), "AdditionalProperties", true);
58958
59046
  HumanAgreementBlock$zodSchemaOutbound = object({
58959
59047
  AdditionalProperties: record(string2(), any()).optional(),
58960
59048
  current: nullable(number2()),
58961
- pair_count: int(),
58962
- series: array(record(string2(), any()))
59049
+ pair_count: int()
58963
59050
  }).transform((v) => {
58964
59051
  const { AdditionalProperties, ...rest } = v;
58965
59052
  return {
@@ -58974,19 +59061,19 @@ var TaskAgreementResponse$zodSchema, TaskAgreementResponse$zodSchemaOutbound;
58974
59061
  var init_taskagreementresponse = __esm(() => {
58975
59062
  init_zod();
58976
59063
  init_schemas4();
59064
+ init_evaluatoragreement();
59065
+ init_evaluatorswithhumansblock();
58977
59066
  init_humanagreementblock();
58978
59067
  TaskAgreementResponse$zodSchema = collectExtraKeys(object({
58979
- bucket: string2().describe("Time bucket used for the trend series"),
58980
- days: int().describe("Length of the trailing window in days"),
58981
- evaluators: array(record(string2(), any())).describe("Agreement between each evaluator and the annotators, plus that evaluator's own results and the human results, one entry per linked evaluator"),
59068
+ evaluators: array(EvaluatorAgreement$zodSchema).describe("Agreement between each evaluator and the annotators, plus that evaluator's own results and the human results, one entry per linked evaluator"),
59069
+ evaluators_with_humans: EvaluatorsWithHumansBlock$zodSchema,
58982
59070
  human_human: HumanAgreementBlock$zodSchema,
58983
59071
  task_id: string2().describe("ID of the task")
58984
59072
  }).catchall(any()), "AdditionalProperties", true);
58985
59073
  TaskAgreementResponse$zodSchemaOutbound = object({
58986
59074
  AdditionalProperties: record(string2(), any()).optional(),
58987
- bucket: string2(),
58988
- days: int(),
58989
- evaluators: array(record(string2(), any())),
59075
+ evaluators: array(EvaluatorAgreement$zodSchema),
59076
+ evaluators_with_humans: EvaluatorsWithHumansBlock$zodSchema,
58990
59077
  human_human: HumanAgreementBlock$zodSchemaOutbound,
58991
59078
  task_id: string2()
58992
59079
  }).transform((v) => {
@@ -58999,19 +59086,12 @@ var init_taskagreementresponse = __esm(() => {
58999
59086
  });
59000
59087
 
59001
59088
  // src/models/taskagreementannotationtaskstaskuuidagreementgetop.ts
59002
- var Bucket$zodSchema, TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest$zodSchema, TaskAgreementAnnotationTasksTaskUuidAgreementGetResponse$zodSchema;
59089
+ var TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest$zodSchema, TaskAgreementAnnotationTasksTaskUuidAgreementGetResponse$zodSchema;
59003
59090
  var init_taskagreementannotationtaskstaskuuidagreementgetop = __esm(() => {
59004
59091
  init_zod();
59005
59092
  init_httpvalidationerror();
59006
59093
  init_taskagreementresponse();
59007
- Bucket$zodSchema = _enum([
59008
- "week",
59009
- "month",
59010
- "year"
59011
- ]).describe("How to bucket points in the trend series");
59012
59094
  TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest$zodSchema = object({
59013
- bucket: Bucket$zodSchema.default("week").describe("How to bucket points in the trend series"),
59014
- days: int().default(90).describe("Trailing window in days for the trend series"),
59015
59095
  task_uuid: string2().describe("Annotation task to act on"),
59016
59096
  xAPIKey: string2().nullable().optional()
59017
59097
  });
@@ -59039,10 +59119,6 @@ async function $do24(client$, request, options) {
59039
59119
  })
59040
59120
  };
59041
59121
  const path$ = pathToFunc("/annotation-tasks/{task_uuid}/agreement")(pathParams$);
59042
- const query$ = encodeFormQuery({
59043
- bucket: payload$.bucket,
59044
- days: payload$.days
59045
- });
59046
59122
  const headers$ = new Headers(compactMap({
59047
59123
  Accept: "application/json",
59048
59124
  "X-API-Key": encodeSimple("X-API-Key", payload$.xAPIKey, {
@@ -59074,7 +59150,6 @@ async function $do24(client$, request, options) {
59074
59150
  baseURL: options?.serverURL,
59075
59151
  path: path$,
59076
59152
  headers: headers$,
59077
- query: query$,
59078
59153
  body: body$,
59079
59154
  userAgent: client$._options.userAgent,
59080
59155
  timeoutMs: options?.timeoutMs || client$._options.timeoutMs || -1
@@ -59119,7 +59194,7 @@ var init_annotationTasksGetAgreement2 = __esm(() => {
59119
59194
  };
59120
59195
  tool$annotationTasksGetAgreement = {
59121
59196
  name: "get-annotation-agreement",
59122
- description: `Get human-vs-human and human-vs-evaluator agreement metrics for a task, with a trend series.
59197
+ description: `Get human-vs-human and human-vs-evaluator agreement metrics for a task, with a chance-corrected kappa for each evaluator.
59123
59198
  `,
59124
59199
  scopes: ["read"],
59125
59200
  annotations: {
@@ -60162,14 +60237,45 @@ var init_annotatorsCreate2 = __esm(() => {
60162
60237
  };
60163
60238
  });
60164
60239
 
60240
+ // src/models/kappaofanswerkind.ts
60241
+ var KappaOfAnswerKindKappaType$zodSchema, KappaOfAnswerKindKappaBlankReason$zodSchema, AnswerKind$zodSchema, KappaOfAnswerKind$zodSchema;
60242
+ var init_kappaofanswerkind = __esm(() => {
60243
+ init_zod();
60244
+ KappaOfAnswerKindKappaType$zodSchema = _enum([
60245
+ "fleiss",
60246
+ "linear_weighted_fleiss",
60247
+ "cohen",
60248
+ "linear_weighted_cohen"
60249
+ ]);
60250
+ KappaOfAnswerKindKappaBlankReason$zodSchema = _enum([
60251
+ "too_few_items",
60252
+ "everyone_agreed"
60253
+ ]);
60254
+ AnswerKind$zodSchema = _enum([
60255
+ "binary",
60256
+ "rating"
60257
+ ]).describe("Which answers this entry covers:\n\n- `binary`: answers that either match or do not, such as pass or fail\n- `rating`: ratings on a scale\n");
60258
+ KappaOfAnswerKind$zodSchema = object({
60259
+ agreement: number2().nullable().describe("Mean agreement with the other annotators over the pairs `kappa` is worked out from, from 0 to 1. When `kappa_blank_reason` is `too_few_items`, over every compared pair of this kind"),
60260
+ answer_kind: AnswerKind$zodSchema.describe("Which answers this entry covers:\n\n- `binary`: answers that either match or do not, such as pass or fail\n- `rating`: ratings on a scale\n"),
60261
+ chance_agreement: number2().nullable().describe("Agreement expected by chance alone, which `kappa` corrects for, on the same scale as the agreement"),
60262
+ kappa: number2().nullable().describe("Agreement corrected for the agreement expected by chance, where 1 is perfect agreement, 0 is chance level, and below 0 is worse than chance"),
60263
+ kappa_blank_reason: KappaOfAnswerKindKappaBlankReason$zodSchema.nullable().describe("Why `kappa` is blank when answers were compared:\n\n- `too_few_items`: every evaluator compared these answers on only one item, and one item cannot show the agreement expected by chance\n- `everyone_agreed`: every compared answer is the same, so agreement cannot be told apart from chance\n"),
60264
+ kappa_type: KappaOfAnswerKindKappaType$zodSchema.nullable().describe("The kind of kappa reported in `kappa`:\n\n- `fleiss`: Fleiss' kappa between three or more annotators\n- `linear_weighted_fleiss`: Fleiss' kappa between three or more annotators, giving partial credit to close ratings\n- `cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others\n- `linear_weighted_cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others, giving partial credit to close ratings\n"),
60265
+ pair_count: int().describe("How many compared pairs `agreement` averages over")
60266
+ });
60267
+ });
60268
+
60165
60269
  // src/models/annotatorresponse.ts
60166
60270
  var AnnotatorResponse$zodSchema;
60167
60271
  var init_annotatorresponse = __esm(() => {
60168
60272
  init_zod();
60273
+ init_kappaofanswerkind();
60169
60274
  AnnotatorResponse$zodSchema = object({
60170
60275
  created_at: string2().describe("When the annotator was created (ISO 8601 UTC)"),
60171
60276
  current_agreement: number2().nullable().optional().describe("Latest pairwise mean agreement with other annotators, from 0 to 1"),
60172
60277
  jobs_count: int().nullable().optional().describe("Number of labelling jobs assigned to this annotator"),
60278
+ kappa_by_kind: array(KappaOfAnswerKind$zodSchema).nullable().optional().describe("Agreement with the other annotators and its kappa, one entry for each kind of answer compared, binary answers first. The kinds are never combined into one kappa"),
60173
60279
  name: string2().describe("Annotator name"),
60174
60280
  pair_count: int().nullable().optional().describe("Number of comparable annotation pairs behind the current agreement score"),
60175
60281
  updated_at: string2().describe("When the annotator was last updated (ISO 8601 UTC)"),
@@ -62941,7 +63047,7 @@ last. Requires a Calibrate API key (CALIBRATE_API_KEY).
62941
63047
  function createMCPServer(deps) {
62942
63048
  const server = new McpServer({
62943
63049
  name: "CalibrateMcp",
62944
- version: "0.0.84"
63050
+ version: "0.0.86"
62945
63051
  });
62946
63052
  const getClient = deps.getSDK || (() => new CalibrateMcpCore({
62947
63053
  security: deps.security,
@@ -63319,7 +63425,7 @@ must be in your workspace.
63319
63425
  },
63320
63426
  {
63321
63427
  name: "get-annotation-agreement",
63322
- description: `Get human-vs-human and human-vs-evaluator agreement metrics for a task, with a trend series.
63428
+ description: `Get human-vs-human and human-vs-evaluator agreement metrics for a task, with a chance-corrected kappa for each evaluator.
63323
63429
  `
63324
63430
  },
63325
63431
  {
@@ -64266,7 +64372,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
64266
64372
  <h1>Instructions</h1>
64267
64373
  <p>One-click installation for Claude Desktop users</p>
64268
64374
  <div class="instruction-item">
64269
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.84/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
64375
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.86/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
64270
64376
  \uD83D\uDCE5 Download MCP Bundle
64271
64377
  </a>
64272
64378
  </div>
@@ -67220,7 +67326,7 @@ var routes = buildRouteMap({
67220
67326
  var app = buildApplication(routes, {
67221
67327
  name: "mcp",
67222
67328
  versionInfo: {
67223
- currentVersion: "0.0.84"
67329
+ currentVersion: "0.0.86"
67224
67330
  }
67225
67331
  });
67226
67332
  run(app, process3.argv.slice(2), buildContext(process3));
@@ -67228,5 +67334,5 @@ export {
67228
67334
  app
67229
67335
  };
67230
67336
 
67231
- //# debugId=836868EA84BFD5EF64756E2164756E21
67337
+ //# debugId=F390FB93EA6317C264756E2164756E21
67232
67338
  //# sourceMappingURL=mcp-server.js.map