@dalmia/calibrate-mcp 0.0.84 → 0.0.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/mcp-server.js +152 -46
- package/bin/mcp-server.js.map +23 -18
- package/esm/funcs/annotationTasksGetAgreement.js +1 -6
- package/esm/funcs/annotationTasksGetAgreement.js.map +1 -1
- package/esm/landing-page.js +1 -1
- package/esm/lib/config.d.ts +2 -2
- package/esm/lib/config.js +2 -2
- package/esm/mcp-server/mcp-server.js +1 -1
- package/esm/mcp-server/server.js +1 -1
- package/esm/mcp-server/tools/annotationTasksGetAgreement.js +1 -1
- package/esm/models/annotatorresponse.d.ts +2 -0
- package/esm/models/annotatorresponse.d.ts.map +1 -1
- package/esm/models/annotatorresponse.js +3 -0
- package/esm/models/annotatorresponse.js.map +1 -1
- package/esm/models/answersummary.d.ts +8 -0
- package/esm/models/answersummary.d.ts.map +1 -0
- package/esm/models/answersummary.js +10 -0
- package/esm/models/answersummary.js.map +1 -0
- package/esm/models/evaluatoragreement.d.ts +41 -0
- package/esm/models/evaluatoragreement.d.ts.map +1 -0
- package/esm/models/evaluatoragreement.js +42 -0
- package/esm/models/evaluatoragreement.js.map +1 -0
- package/esm/models/evaluatorswithhumansblock.d.ts +7 -0
- package/esm/models/evaluatorswithhumansblock.d.ts.map +1 -0
- package/esm/models/evaluatorswithhumansblock.js +9 -0
- package/esm/models/evaluatorswithhumansblock.js.map +1 -0
- package/esm/models/humanagreementblock.d.ts +0 -3
- package/esm/models/humanagreementblock.d.ts.map +1 -1
- package/esm/models/humanagreementblock.js +0 -2
- package/esm/models/humanagreementblock.js.map +1 -1
- package/esm/models/interannotatoragreement.d.ts +34 -0
- package/esm/models/interannotatoragreement.d.ts.map +1 -0
- package/esm/models/interannotatoragreement.js +34 -0
- package/esm/models/interannotatoragreement.js.map +1 -0
- package/esm/models/kappaofanswerkind.d.ts +60 -0
- package/esm/models/kappaofanswerkind.d.ts.map +1 -0
- package/esm/models/kappaofanswerkind.js +52 -0
- package/esm/models/kappaofanswerkind.js.map +1 -0
- package/esm/models/taskagreementannotationtaskstaskuuidagreementgetop.d.ts +0 -20
- package/esm/models/taskagreementannotationtaskstaskuuidagreementgetop.d.ts.map +1 -1
- package/esm/models/taskagreementannotationtaskstaskuuidagreementgetop.js +0 -15
- package/esm/models/taskagreementannotationtaskstaskuuidagreementgetop.js.map +1 -1
- package/esm/models/taskagreementresponse.d.ts +4 -5
- package/esm/models/taskagreementresponse.d.ts.map +1 -1
- package/esm/models/taskagreementresponse.js +6 -6
- package/esm/models/taskagreementresponse.js.map +1 -1
- package/esm/tool-names.js +1 -1
- package/esm/tool-names.js.map +1 -1
- package/package.json +1 -1
- package/src/funcs/annotationTasksGetAgreement.ts +1 -6
- package/src/landing-page.ts +1 -1
- package/src/lib/config.ts +2 -2
- package/src/mcp-server/mcp-server.ts +1 -1
- package/src/mcp-server/server.ts +1 -1
- package/src/mcp-server/tools/annotationTasksGetAgreement.ts +1 -1
- package/src/models/annotatorresponse.ts +9 -0
- package/src/models/answersummary.ts +19 -0
- package/src/models/evaluatoragreement.ts +85 -0
- package/src/models/evaluatorswithhumansblock.ts +21 -0
- package/src/models/humanagreementblock.ts +0 -5
- package/src/models/interannotatoragreement.ts +69 -0
- package/src/models/kappaofanswerkind.ts +101 -0
- package/src/models/taskagreementannotationtaskstaskuuidagreementgetop.ts +0 -28
- package/src/models/taskagreementresponse.ts +14 -9
- package/src/tool-names.ts +1 -1
package/bin/mcp-server.js
CHANGED
|
@@ -25889,7 +25889,8 @@ var require_formats = __commonJS(function(exports, module) {
|
|
|
25889
25889
|
var percentTwenties = /%20/g;
|
|
25890
25890
|
var Format = {
|
|
25891
25891
|
RFC1738: "RFC1738",
|
|
25892
|
-
RFC3986: "RFC3986"
|
|
25892
|
+
RFC3986: "RFC3986",
|
|
25893
|
+
WHATWG: "WHATWG"
|
|
25893
25894
|
};
|
|
25894
25895
|
module.exports = {
|
|
25895
25896
|
default: Format.RFC3986,
|
|
@@ -25899,10 +25900,14 @@ var require_formats = __commonJS(function(exports, module) {
|
|
|
25899
25900
|
},
|
|
25900
25901
|
RFC3986: function(value) {
|
|
25901
25902
|
return String(value);
|
|
25903
|
+
},
|
|
25904
|
+
WHATWG: function(value) {
|
|
25905
|
+
return replace.call(value, percentTwenties, "+");
|
|
25902
25906
|
}
|
|
25903
25907
|
},
|
|
25904
25908
|
RFC1738: Format.RFC1738,
|
|
25905
|
-
RFC3986: Format.RFC3986
|
|
25909
|
+
RFC3986: Format.RFC3986,
|
|
25910
|
+
WHATWG: Format.WHATWG
|
|
25906
25911
|
};
|
|
25907
25912
|
});
|
|
25908
25913
|
|
|
@@ -26206,7 +26211,7 @@ var require_utils2 = __commonJS(function(exports, module) {
|
|
|
26206
26211
|
var arr = [];
|
|
26207
26212
|
for (var i = 0;i < segment.length; ++i) {
|
|
26208
26213
|
var c = segment.charCodeAt(i);
|
|
26209
|
-
if (c === 45 || c === 46 || c === 95 || c === 126 || c >= 48 && c <= 57 || c >= 65 && c <= 90 || c >= 97 && c <= 122 || format === formats.RFC1738 && (c === 40 || c === 41)) {
|
|
26214
|
+
if (c === 45 || c === 46 || c === 95 || c === 126 && format !== formats.WHATWG || c >= 48 && c <= 57 || c >= 65 && c <= 90 || c >= 97 && c <= 122 || format === formats.RFC1738 && (c === 40 || c === 41) || format === formats.WHATWG && c === 42) {
|
|
26210
26215
|
arr[arr.length] = segment.charAt(i);
|
|
26211
26216
|
continue;
|
|
26212
26217
|
}
|
|
@@ -26222,9 +26227,14 @@ var require_utils2 = __commonJS(function(exports, module) {
|
|
|
26222
26227
|
arr[arr.length] = hexTable[224 | c >> 12] + hexTable[128 | c >> 6 & 63] + hexTable[128 | c & 63];
|
|
26223
26228
|
continue;
|
|
26224
26229
|
}
|
|
26225
|
-
i
|
|
26226
|
-
c
|
|
26227
|
-
|
|
26230
|
+
var next = segment.charCodeAt(i + 1);
|
|
26231
|
+
if (c <= 56319 && next >= 56320 && next <= 57343) {
|
|
26232
|
+
i += 1;
|
|
26233
|
+
c = 65536 + ((c & 1023) << 10 | next & 1023);
|
|
26234
|
+
arr[arr.length] = hexTable[240 | c >> 18] + hexTable[128 | c >> 12 & 63] + hexTable[128 | c >> 6 & 63] + hexTable[128 | c & 63];
|
|
26235
|
+
} else {
|
|
26236
|
+
arr[arr.length] = "%EF%BF%BD";
|
|
26237
|
+
}
|
|
26228
26238
|
}
|
|
26229
26239
|
out += arr.join("");
|
|
26230
26240
|
}
|
|
@@ -26304,6 +26314,7 @@ var require_utils2 = __commonJS(function(exports, module) {
|
|
|
26304
26314
|
markOverflow,
|
|
26305
26315
|
maybeMap,
|
|
26306
26316
|
merge,
|
|
26317
|
+
setProperty,
|
|
26307
26318
|
toCoercible
|
|
26308
26319
|
};
|
|
26309
26320
|
});
|
|
@@ -26428,7 +26439,7 @@ var require_stringify = __commonJS(function(exports, module) {
|
|
|
26428
26439
|
var encodedPrefix = String(prefix);
|
|
26429
26440
|
var adjustedPrefix = commaRoundTrip && isArray(obj) && obj.length === 1 ? encodedPrefix + "[]" : encodedPrefix;
|
|
26430
26441
|
if (allowEmptyArrays && isArray(obj) && obj.length === 0 && Object.keys(obj).length === 0) {
|
|
26431
|
-
return adjustedPrefix + "[]";
|
|
26442
|
+
return formatter(encoder && !encodeValuesOnly ? encoder(adjustedPrefix, defaults.encoder, charset, "key", format) : adjustedPrefix) + "[]";
|
|
26432
26443
|
}
|
|
26433
26444
|
for (var j = 0;j < objKeys.length; ++j) {
|
|
26434
26445
|
var key = objKeys[j];
|
|
@@ -26518,7 +26529,7 @@ var require_stringify = __commonJS(function(exports, module) {
|
|
|
26518
26529
|
filter = options.filter;
|
|
26519
26530
|
obj = filter("", obj);
|
|
26520
26531
|
} else if (isArray(options.filter)) {
|
|
26521
|
-
filter = options.filter;
|
|
26532
|
+
filter = options.filter.slice();
|
|
26522
26533
|
objKeys = filter;
|
|
26523
26534
|
}
|
|
26524
26535
|
var keys = [];
|
|
@@ -26544,7 +26555,7 @@ var require_stringify = __commonJS(function(exports, module) {
|
|
|
26544
26555
|
continue;
|
|
26545
26556
|
}
|
|
26546
26557
|
var encodedKey = options.encodeDotInKeys ? String(key).replace(/\./g, "%2E") : String(key);
|
|
26547
|
-
pushToArray(keys, stringify(value, encodedKey, generateArrayPrefix, commaRoundTrip, options.allowEmptyArrays, options.strictNullHandling, options.skipNulls, options.encodeDotInKeys, options.encode ? options.encoder : null,
|
|
26558
|
+
pushToArray(keys, stringify(value, encodedKey, generateArrayPrefix, commaRoundTrip, options.allowEmptyArrays, options.strictNullHandling, options.skipNulls, options.encodeDotInKeys, options.encode ? options.encoder : null, filter, options.sort, options.allowDots, options.serializeDate, options.format, options.formatter, options.encodeValuesOnly, options.charset, sideChannel, options.depth, 0));
|
|
26548
26559
|
}
|
|
26549
26560
|
var joined = keys.join(options.delimiter);
|
|
26550
26561
|
var prefix = options.addQueryPrefix === true ? "?" : "";
|
|
@@ -26600,9 +26611,9 @@ var require_parse = __commonJS(function(exports, module) {
|
|
|
26600
26611
|
copies.set(value, copy);
|
|
26601
26612
|
queue[queue.length] = { source: value, target: copy };
|
|
26602
26613
|
}
|
|
26603
|
-
target
|
|
26614
|
+
utils.setProperty(target, keys[j], copies.get(value));
|
|
26604
26615
|
} else {
|
|
26605
|
-
target
|
|
26616
|
+
utils.setProperty(target, keys[j], value);
|
|
26606
26617
|
}
|
|
26607
26618
|
}
|
|
26608
26619
|
}
|
|
@@ -26652,7 +26663,7 @@ var require_parse = __commonJS(function(exports, module) {
|
|
|
26652
26663
|
}
|
|
26653
26664
|
return val.split(",");
|
|
26654
26665
|
}
|
|
26655
|
-
if (options.throwOnLimitExceeded && currentArrayLength >= options.arrayLimit) {
|
|
26666
|
+
if (options.throwOnLimitExceeded && currentArrayLength >= options.arrayLimit && currentArrayLength > 0) {
|
|
26656
26667
|
throw new RangeError("Array limit exceeded. Only " + options.arrayLimit + " element" + (options.arrayLimit === 1 ? "" : "s") + " allowed in an array.");
|
|
26657
26668
|
}
|
|
26658
26669
|
return val;
|
|
@@ -26700,13 +26711,15 @@ var require_parse = __commonJS(function(exports, module) {
|
|
|
26700
26711
|
} else {
|
|
26701
26712
|
key = options.decoder(part.slice(0, pos), defaults.decoder, charset, "key");
|
|
26702
26713
|
if (key !== null) {
|
|
26703
|
-
val = utils.maybeMap(parseArrayValue(part.slice(pos + 1), options, isArray(obj[key]) ? obj[key].length : 0), function(encodedVal) {
|
|
26714
|
+
val = utils.maybeMap(parseArrayValue(part.slice(pos + 1), options, isArray(obj[key]) && (options.duplicates === "combine" || isEmptyBracketKey) ? obj[key].length : 0), function(encodedVal) {
|
|
26704
26715
|
return options.decoder(encodedVal, defaults.decoder, charset, "value");
|
|
26705
26716
|
});
|
|
26706
26717
|
}
|
|
26707
26718
|
}
|
|
26708
26719
|
if (val && options.interpretNumericEntities && charset === "iso-8859-1") {
|
|
26709
|
-
val =
|
|
26720
|
+
val = utils.maybeMap(val, function(value) {
|
|
26721
|
+
return value ? interpretNumericEntities(String(value)) : value;
|
|
26722
|
+
});
|
|
26710
26723
|
}
|
|
26711
26724
|
if (isEmptyBracketKey) {
|
|
26712
26725
|
val = isArray(val) ? [val] : val;
|
|
@@ -26744,7 +26757,7 @@ var require_parse = __commonJS(function(exports, module) {
|
|
|
26744
26757
|
} else {
|
|
26745
26758
|
obj = options.plainObjects ? { __proto__: null } : {};
|
|
26746
26759
|
var cleanRoot = root.charAt(0) === "[" && root.charAt(root.length - 1) === "]" ? root.slice(1, -1) : root;
|
|
26747
|
-
var decodedRoot = options.decodeDotInKeys ? cleanRoot.replace(/%2E/
|
|
26760
|
+
var decodedRoot = options.decodeDotInKeys ? cleanRoot.replace(/%2E/gi, ".") : cleanRoot;
|
|
26748
26761
|
var index = parseInt(decodedRoot, 10);
|
|
26749
26762
|
var isValidArrayIndex = !isNaN(index) && root !== decodedRoot && String(index) === decodedRoot && index >= 0 && options.parseArrays;
|
|
26750
26763
|
if (!options.parseArrays && decodedRoot === "") {
|
|
@@ -53266,9 +53279,9 @@ var init_config = __esm(() => {
|
|
|
53266
53279
|
SDK_METADATA = {
|
|
53267
53280
|
language: "typescript",
|
|
53268
53281
|
openapiDocVersion: "0.1.0",
|
|
53269
|
-
sdkVersion: "0.0.
|
|
53282
|
+
sdkVersion: "0.0.86",
|
|
53270
53283
|
genVersion: "2.915.1",
|
|
53271
|
-
userAgent: "speakeasy-sdk/mcp-typescript 0.0.
|
|
53284
|
+
userAgent: "speakeasy-sdk/mcp-typescript 0.0.86 2.915.1 0.1.0 @dalmia/calibrate-mcp"
|
|
53272
53285
|
};
|
|
53273
53286
|
});
|
|
53274
53287
|
|
|
@@ -58945,6 +58958,82 @@ var init_annotationTasksGet2 = __esm(() => {
|
|
|
58945
58958
|
};
|
|
58946
58959
|
});
|
|
58947
58960
|
|
|
58961
|
+
// src/models/answersummary.ts
|
|
58962
|
+
var AnswerSummary$zodSchema;
|
|
58963
|
+
var init_answersummary = __esm(() => {
|
|
58964
|
+
init_zod();
|
|
58965
|
+
AnswerSummary$zodSchema = object({
|
|
58966
|
+
count: int().describe("How many answers there are"),
|
|
58967
|
+
mean: number2().nullable().describe("Mean of the rating answers"),
|
|
58968
|
+
true_count: int().nullable().describe("How many pass/fail answers are a pass")
|
|
58969
|
+
});
|
|
58970
|
+
});
|
|
58971
|
+
|
|
58972
|
+
// src/models/interannotatoragreement.ts
|
|
58973
|
+
var InterAnnotatorAgreementKappaType$zodSchema, InterAnnotatorAgreementKappaBlankReason$zodSchema, InterAnnotatorAgreement$zodSchema;
|
|
58974
|
+
var init_interannotatoragreement = __esm(() => {
|
|
58975
|
+
init_zod();
|
|
58976
|
+
InterAnnotatorAgreementKappaType$zodSchema = _enum([
|
|
58977
|
+
"fleiss",
|
|
58978
|
+
"linear_weighted_fleiss",
|
|
58979
|
+
"cohen",
|
|
58980
|
+
"linear_weighted_cohen"
|
|
58981
|
+
]);
|
|
58982
|
+
InterAnnotatorAgreementKappaBlankReason$zodSchema = _enum([
|
|
58983
|
+
"too_few_items",
|
|
58984
|
+
"everyone_agreed"
|
|
58985
|
+
]);
|
|
58986
|
+
InterAnnotatorAgreement$zodSchema = object({
|
|
58987
|
+
chance_agreement: number2().nullable().describe("Agreement expected by chance alone, which `kappa` corrects for, on the same scale as the agreement"),
|
|
58988
|
+
current: number2().nullable().describe("Current mean pairwise agreement among annotators on this evaluator"),
|
|
58989
|
+
kappa: number2().nullable().describe("Agreement corrected for the agreement expected by chance, where 1 is perfect agreement, 0 is chance level, and below 0 is worse than chance"),
|
|
58990
|
+
kappa_blank_reason: InterAnnotatorAgreementKappaBlankReason$zodSchema.nullable().describe("Why `kappa` is blank when answers were compared:\n\n- `too_few_items`: every evaluator compared these answers on only one item, and one item cannot show the agreement expected by chance\n- `everyone_agreed`: every compared answer is the same, so agreement cannot be told apart from chance\n"),
|
|
58991
|
+
kappa_type: InterAnnotatorAgreementKappaType$zodSchema.nullable().describe("The kind of kappa reported in `kappa`:\n\n- `fleiss`: Fleiss' kappa between three or more annotators\n- `linear_weighted_fleiss`: Fleiss' kappa between three or more annotators, giving partial credit to close ratings\n- `cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others\n- `linear_weighted_cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others, giving partial credit to close ratings\n"),
|
|
58992
|
+
pair_count: int().describe("How many annotator pairs the current value averages over")
|
|
58993
|
+
});
|
|
58994
|
+
});
|
|
58995
|
+
|
|
58996
|
+
// src/models/evaluatoragreement.ts
|
|
58997
|
+
var EvaluatorAgreementKappaType$zodSchema, EvaluatorAgreementKappaBlankReason$zodSchema, EvaluatorAgreement$zodSchema;
|
|
58998
|
+
var init_evaluatoragreement = __esm(() => {
|
|
58999
|
+
init_zod();
|
|
59000
|
+
init_answersummary();
|
|
59001
|
+
init_interannotatoragreement();
|
|
59002
|
+
EvaluatorAgreementKappaType$zodSchema = _enum([
|
|
59003
|
+
"fleiss",
|
|
59004
|
+
"linear_weighted_fleiss",
|
|
59005
|
+
"cohen",
|
|
59006
|
+
"linear_weighted_cohen"
|
|
59007
|
+
]);
|
|
59008
|
+
EvaluatorAgreementKappaBlankReason$zodSchema = _enum([
|
|
59009
|
+
"too_few_items",
|
|
59010
|
+
"everyone_agreed"
|
|
59011
|
+
]);
|
|
59012
|
+
EvaluatorAgreement$zodSchema = object({
|
|
59013
|
+
chance_agreement: number2().nullable().describe("Agreement expected by chance alone, which `kappa` corrects for, on the same scale as the agreement"),
|
|
59014
|
+
current: number2().nullable().describe("Current mean agreement between this evaluator's judge and the annotators"),
|
|
59015
|
+
evaluator_id: string2().describe("ID of the evaluator"),
|
|
59016
|
+
human_result: AnswerSummary$zodSchema.nullable().describe("The annotators' answers for this evaluator"),
|
|
59017
|
+
inter_annotator: InterAnnotatorAgreement$zodSchema,
|
|
59018
|
+
kappa: number2().nullable().describe("Agreement corrected for the agreement expected by chance, where 1 is perfect agreement, 0 is chance level, and below 0 is worse than chance"),
|
|
59019
|
+
kappa_blank_reason: EvaluatorAgreementKappaBlankReason$zodSchema.nullable().describe("Why `kappa` is blank when answers were compared:\n\n- `too_few_items`: every evaluator compared these answers on only one item, and one item cannot show the agreement expected by chance\n- `everyone_agreed`: every compared answer is the same, so agreement cannot be told apart from chance\n"),
|
|
59020
|
+
kappa_type: EvaluatorAgreementKappaType$zodSchema.nullable().describe("The kind of kappa reported in `kappa`:\n\n- `fleiss`: Fleiss' kappa between three or more annotators\n- `linear_weighted_fleiss`: Fleiss' kappa between three or more annotators, giving partial credit to close ratings\n- `cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others\n- `linear_weighted_cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others, giving partial credit to close ratings\n"),
|
|
59021
|
+
name: string2().nullable().describe("Name of the evaluator"),
|
|
59022
|
+
pair_count: int().describe("How many judge and annotator pairs the current value averages over"),
|
|
59023
|
+
result: AnswerSummary$zodSchema
|
|
59024
|
+
});
|
|
59025
|
+
});
|
|
59026
|
+
|
|
59027
|
+
// src/models/evaluatorswithhumansblock.ts
|
|
59028
|
+
var EvaluatorsWithHumansBlock$zodSchema;
|
|
59029
|
+
var init_evaluatorswithhumansblock = __esm(() => {
|
|
59030
|
+
init_zod();
|
|
59031
|
+
EvaluatorsWithHumansBlock$zodSchema = object({
|
|
59032
|
+
current: number2().nullable().describe("Current mean agreement between the evaluators and the annotators, weighted by how many pairs each evaluator compared"),
|
|
59033
|
+
pair_count: int().describe("How many evaluator and annotator pairs the current value averages over")
|
|
59034
|
+
});
|
|
59035
|
+
});
|
|
59036
|
+
|
|
58948
59037
|
// src/models/humanagreementblock.ts
|
|
58949
59038
|
var HumanAgreementBlock$zodSchema, HumanAgreementBlock$zodSchemaOutbound;
|
|
58950
59039
|
var init_humanagreementblock = __esm(() => {
|
|
@@ -58952,14 +59041,12 @@ var init_humanagreementblock = __esm(() => {
|
|
|
58952
59041
|
init_schemas4();
|
|
58953
59042
|
HumanAgreementBlock$zodSchema = collectExtraKeys(object({
|
|
58954
59043
|
current: number2().nullable().describe("Current mean pairwise agreement among annotators"),
|
|
58955
|
-
pair_count: int().describe("How many annotator pairs the current value averages over")
|
|
58956
|
-
series: array(record(string2(), any())).describe("Agreement over time, one point per time bucket")
|
|
59044
|
+
pair_count: int().describe("How many annotator pairs the current value averages over")
|
|
58957
59045
|
}).catchall(any()), "AdditionalProperties", true);
|
|
58958
59046
|
HumanAgreementBlock$zodSchemaOutbound = object({
|
|
58959
59047
|
AdditionalProperties: record(string2(), any()).optional(),
|
|
58960
59048
|
current: nullable(number2()),
|
|
58961
|
-
pair_count: int()
|
|
58962
|
-
series: array(record(string2(), any()))
|
|
59049
|
+
pair_count: int()
|
|
58963
59050
|
}).transform((v) => {
|
|
58964
59051
|
const { AdditionalProperties, ...rest } = v;
|
|
58965
59052
|
return {
|
|
@@ -58974,19 +59061,19 @@ var TaskAgreementResponse$zodSchema, TaskAgreementResponse$zodSchemaOutbound;
|
|
|
58974
59061
|
var init_taskagreementresponse = __esm(() => {
|
|
58975
59062
|
init_zod();
|
|
58976
59063
|
init_schemas4();
|
|
59064
|
+
init_evaluatoragreement();
|
|
59065
|
+
init_evaluatorswithhumansblock();
|
|
58977
59066
|
init_humanagreementblock();
|
|
58978
59067
|
TaskAgreementResponse$zodSchema = collectExtraKeys(object({
|
|
58979
|
-
|
|
58980
|
-
|
|
58981
|
-
evaluators: array(record(string2(), any())).describe("Agreement between each evaluator and the annotators, plus that evaluator's own results and the human results, one entry per linked evaluator"),
|
|
59068
|
+
evaluators: array(EvaluatorAgreement$zodSchema).describe("Agreement between each evaluator and the annotators, plus that evaluator's own results and the human results, one entry per linked evaluator"),
|
|
59069
|
+
evaluators_with_humans: EvaluatorsWithHumansBlock$zodSchema,
|
|
58982
59070
|
human_human: HumanAgreementBlock$zodSchema,
|
|
58983
59071
|
task_id: string2().describe("ID of the task")
|
|
58984
59072
|
}).catchall(any()), "AdditionalProperties", true);
|
|
58985
59073
|
TaskAgreementResponse$zodSchemaOutbound = object({
|
|
58986
59074
|
AdditionalProperties: record(string2(), any()).optional(),
|
|
58987
|
-
|
|
58988
|
-
|
|
58989
|
-
evaluators: array(record(string2(), any())),
|
|
59075
|
+
evaluators: array(EvaluatorAgreement$zodSchema),
|
|
59076
|
+
evaluators_with_humans: EvaluatorsWithHumansBlock$zodSchema,
|
|
58990
59077
|
human_human: HumanAgreementBlock$zodSchemaOutbound,
|
|
58991
59078
|
task_id: string2()
|
|
58992
59079
|
}).transform((v) => {
|
|
@@ -58999,19 +59086,12 @@ var init_taskagreementresponse = __esm(() => {
|
|
|
58999
59086
|
});
|
|
59000
59087
|
|
|
59001
59088
|
// src/models/taskagreementannotationtaskstaskuuidagreementgetop.ts
|
|
59002
|
-
var
|
|
59089
|
+
var TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest$zodSchema, TaskAgreementAnnotationTasksTaskUuidAgreementGetResponse$zodSchema;
|
|
59003
59090
|
var init_taskagreementannotationtaskstaskuuidagreementgetop = __esm(() => {
|
|
59004
59091
|
init_zod();
|
|
59005
59092
|
init_httpvalidationerror();
|
|
59006
59093
|
init_taskagreementresponse();
|
|
59007
|
-
Bucket$zodSchema = _enum([
|
|
59008
|
-
"week",
|
|
59009
|
-
"month",
|
|
59010
|
-
"year"
|
|
59011
|
-
]).describe("How to bucket points in the trend series");
|
|
59012
59094
|
TaskAgreementAnnotationTasksTaskUuidAgreementGetRequest$zodSchema = object({
|
|
59013
|
-
bucket: Bucket$zodSchema.default("week").describe("How to bucket points in the trend series"),
|
|
59014
|
-
days: int().default(90).describe("Trailing window in days for the trend series"),
|
|
59015
59095
|
task_uuid: string2().describe("Annotation task to act on"),
|
|
59016
59096
|
xAPIKey: string2().nullable().optional()
|
|
59017
59097
|
});
|
|
@@ -59039,10 +59119,6 @@ async function $do24(client$, request, options) {
|
|
|
59039
59119
|
})
|
|
59040
59120
|
};
|
|
59041
59121
|
const path$ = pathToFunc("/annotation-tasks/{task_uuid}/agreement")(pathParams$);
|
|
59042
|
-
const query$ = encodeFormQuery({
|
|
59043
|
-
bucket: payload$.bucket,
|
|
59044
|
-
days: payload$.days
|
|
59045
|
-
});
|
|
59046
59122
|
const headers$ = new Headers(compactMap({
|
|
59047
59123
|
Accept: "application/json",
|
|
59048
59124
|
"X-API-Key": encodeSimple("X-API-Key", payload$.xAPIKey, {
|
|
@@ -59074,7 +59150,6 @@ async function $do24(client$, request, options) {
|
|
|
59074
59150
|
baseURL: options?.serverURL,
|
|
59075
59151
|
path: path$,
|
|
59076
59152
|
headers: headers$,
|
|
59077
|
-
query: query$,
|
|
59078
59153
|
body: body$,
|
|
59079
59154
|
userAgent: client$._options.userAgent,
|
|
59080
59155
|
timeoutMs: options?.timeoutMs || client$._options.timeoutMs || -1
|
|
@@ -59119,7 +59194,7 @@ var init_annotationTasksGetAgreement2 = __esm(() => {
|
|
|
59119
59194
|
};
|
|
59120
59195
|
tool$annotationTasksGetAgreement = {
|
|
59121
59196
|
name: "get-annotation-agreement",
|
|
59122
|
-
description: `Get human-vs-human and human-vs-evaluator agreement metrics for a task, with a
|
|
59197
|
+
description: `Get human-vs-human and human-vs-evaluator agreement metrics for a task, with a chance-corrected kappa for each evaluator.
|
|
59123
59198
|
`,
|
|
59124
59199
|
scopes: ["read"],
|
|
59125
59200
|
annotations: {
|
|
@@ -60162,14 +60237,45 @@ var init_annotatorsCreate2 = __esm(() => {
|
|
|
60162
60237
|
};
|
|
60163
60238
|
});
|
|
60164
60239
|
|
|
60240
|
+
// src/models/kappaofanswerkind.ts
|
|
60241
|
+
var KappaOfAnswerKindKappaType$zodSchema, KappaOfAnswerKindKappaBlankReason$zodSchema, AnswerKind$zodSchema, KappaOfAnswerKind$zodSchema;
|
|
60242
|
+
var init_kappaofanswerkind = __esm(() => {
|
|
60243
|
+
init_zod();
|
|
60244
|
+
KappaOfAnswerKindKappaType$zodSchema = _enum([
|
|
60245
|
+
"fleiss",
|
|
60246
|
+
"linear_weighted_fleiss",
|
|
60247
|
+
"cohen",
|
|
60248
|
+
"linear_weighted_cohen"
|
|
60249
|
+
]);
|
|
60250
|
+
KappaOfAnswerKindKappaBlankReason$zodSchema = _enum([
|
|
60251
|
+
"too_few_items",
|
|
60252
|
+
"everyone_agreed"
|
|
60253
|
+
]);
|
|
60254
|
+
AnswerKind$zodSchema = _enum([
|
|
60255
|
+
"binary",
|
|
60256
|
+
"rating"
|
|
60257
|
+
]).describe("Which answers this entry covers:\n\n- `binary`: answers that either match or do not, such as pass or fail\n- `rating`: ratings on a scale\n");
|
|
60258
|
+
KappaOfAnswerKind$zodSchema = object({
|
|
60259
|
+
agreement: number2().nullable().describe("Mean agreement with the other annotators over the pairs `kappa` is worked out from, from 0 to 1. When `kappa_blank_reason` is `too_few_items`, over every compared pair of this kind"),
|
|
60260
|
+
answer_kind: AnswerKind$zodSchema.describe("Which answers this entry covers:\n\n- `binary`: answers that either match or do not, such as pass or fail\n- `rating`: ratings on a scale\n"),
|
|
60261
|
+
chance_agreement: number2().nullable().describe("Agreement expected by chance alone, which `kappa` corrects for, on the same scale as the agreement"),
|
|
60262
|
+
kappa: number2().nullable().describe("Agreement corrected for the agreement expected by chance, where 1 is perfect agreement, 0 is chance level, and below 0 is worse than chance"),
|
|
60263
|
+
kappa_blank_reason: KappaOfAnswerKindKappaBlankReason$zodSchema.nullable().describe("Why `kappa` is blank when answers were compared:\n\n- `too_few_items`: every evaluator compared these answers on only one item, and one item cannot show the agreement expected by chance\n- `everyone_agreed`: every compared answer is the same, so agreement cannot be told apart from chance\n"),
|
|
60264
|
+
kappa_type: KappaOfAnswerKindKappaType$zodSchema.nullable().describe("The kind of kappa reported in `kappa`:\n\n- `fleiss`: Fleiss' kappa between three or more annotators\n- `linear_weighted_fleiss`: Fleiss' kappa between three or more annotators, giving partial credit to close ratings\n- `cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others\n- `linear_weighted_cohen`: Cohen's kappa between exactly two annotators, between a judge and the annotators, or between one annotator and the others, giving partial credit to close ratings\n"),
|
|
60265
|
+
pair_count: int().describe("How many compared pairs `agreement` averages over")
|
|
60266
|
+
});
|
|
60267
|
+
});
|
|
60268
|
+
|
|
60165
60269
|
// src/models/annotatorresponse.ts
|
|
60166
60270
|
var AnnotatorResponse$zodSchema;
|
|
60167
60271
|
var init_annotatorresponse = __esm(() => {
|
|
60168
60272
|
init_zod();
|
|
60273
|
+
init_kappaofanswerkind();
|
|
60169
60274
|
AnnotatorResponse$zodSchema = object({
|
|
60170
60275
|
created_at: string2().describe("When the annotator was created (ISO 8601 UTC)"),
|
|
60171
60276
|
current_agreement: number2().nullable().optional().describe("Latest pairwise mean agreement with other annotators, from 0 to 1"),
|
|
60172
60277
|
jobs_count: int().nullable().optional().describe("Number of labelling jobs assigned to this annotator"),
|
|
60278
|
+
kappa_by_kind: array(KappaOfAnswerKind$zodSchema).nullable().optional().describe("Agreement with the other annotators and its kappa, one entry for each kind of answer compared, binary answers first. The kinds are never combined into one kappa"),
|
|
60173
60279
|
name: string2().describe("Annotator name"),
|
|
60174
60280
|
pair_count: int().nullable().optional().describe("Number of comparable annotation pairs behind the current agreement score"),
|
|
60175
60281
|
updated_at: string2().describe("When the annotator was last updated (ISO 8601 UTC)"),
|
|
@@ -62941,7 +63047,7 @@ last. Requires a Calibrate API key (CALIBRATE_API_KEY).
|
|
|
62941
63047
|
function createMCPServer(deps) {
|
|
62942
63048
|
const server = new McpServer({
|
|
62943
63049
|
name: "CalibrateMcp",
|
|
62944
|
-
version: "0.0.
|
|
63050
|
+
version: "0.0.86"
|
|
62945
63051
|
});
|
|
62946
63052
|
const getClient = deps.getSDK || (() => new CalibrateMcpCore({
|
|
62947
63053
|
security: deps.security,
|
|
@@ -63319,7 +63425,7 @@ must be in your workspace.
|
|
|
63319
63425
|
},
|
|
63320
63426
|
{
|
|
63321
63427
|
name: "get-annotation-agreement",
|
|
63322
|
-
description: `Get human-vs-human and human-vs-evaluator agreement metrics for a task, with a
|
|
63428
|
+
description: `Get human-vs-human and human-vs-evaluator agreement metrics for a task, with a chance-corrected kappa for each evaluator.
|
|
63323
63429
|
`
|
|
63324
63430
|
},
|
|
63325
63431
|
{
|
|
@@ -64266,7 +64372,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
|
|
|
64266
64372
|
<h1>Instructions</h1>
|
|
64267
64373
|
<p>One-click installation for Claude Desktop users</p>
|
|
64268
64374
|
<div class="instruction-item">
|
|
64269
|
-
<a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.
|
|
64375
|
+
<a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.86/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
|
|
64270
64376
|
\uD83D\uDCE5 Download MCP Bundle
|
|
64271
64377
|
</a>
|
|
64272
64378
|
</div>
|
|
@@ -67220,7 +67326,7 @@ var routes = buildRouteMap({
|
|
|
67220
67326
|
var app = buildApplication(routes, {
|
|
67221
67327
|
name: "mcp",
|
|
67222
67328
|
versionInfo: {
|
|
67223
|
-
currentVersion: "0.0.
|
|
67329
|
+
currentVersion: "0.0.86"
|
|
67224
67330
|
}
|
|
67225
67331
|
});
|
|
67226
67332
|
run(app, process3.argv.slice(2), buildContext(process3));
|
|
@@ -67228,5 +67334,5 @@ export {
|
|
|
67228
67334
|
app
|
|
67229
67335
|
};
|
|
67230
67336
|
|
|
67231
|
-
//# debugId=
|
|
67337
|
+
//# debugId=F390FB93EA6317C264756E2164756E21
|
|
67232
67338
|
//# sourceMappingURL=mcp-server.js.map
|