@dalmia/calibrate-mcp 0.0.47 → 0.0.49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/bin/mcp-server.js +210 -46
  2. package/bin/mcp-server.js.map +18 -18
  3. package/esm/landing-page.js +1 -1
  4. package/esm/lib/config.d.ts +2 -2
  5. package/esm/lib/config.js +2 -2
  6. package/esm/mcp-server/mcp-server.js +1 -1
  7. package/esm/mcp-server/server.js +1 -1
  8. package/esm/models/agenttestrunlistitem.d.ts +2 -0
  9. package/esm/models/agenttestrunlistitem.d.ts.map +1 -1
  10. package/esm/models/agenttestrunlistitem.js +2 -0
  11. package/esm/models/agenttestrunlistitem.js.map +1 -1
  12. package/esm/models/benchmarkstatusresponse.d.ts +1 -0
  13. package/esm/models/benchmarkstatusresponse.d.ts.map +1 -1
  14. package/esm/models/benchmarkstatusresponse.js +1 -0
  15. package/esm/models/benchmarkstatusresponse.js.map +1 -1
  16. package/esm/models/testcaseresult.d.ts +2 -0
  17. package/esm/models/testcaseresult.d.ts.map +1 -1
  18. package/esm/models/testcaseresult.js +2 -0
  19. package/esm/models/testcaseresult.js.map +1 -1
  20. package/esm/models/testruncasesummary.js +1 -1
  21. package/esm/models/testruncasesummary.js.map +1 -1
  22. package/esm/models/testrunstatusresponse.d.ts +3 -0
  23. package/esm/models/testrunstatusresponse.d.ts.map +1 -1
  24. package/esm/models/testrunstatusresponse.js +3 -0
  25. package/esm/models/testrunstatusresponse.js.map +1 -1
  26. package/package.json +1 -1
  27. package/src/landing-page.ts +1 -1
  28. package/src/lib/config.ts +2 -2
  29. package/src/mcp-server/mcp-server.ts +1 -1
  30. package/src/mcp-server/server.ts +1 -1
  31. package/src/models/agenttestrunlistitem.ts +8 -0
  32. package/src/models/benchmarkstatusresponse.ts +4 -0
  33. package/src/models/testcaseresult.ts +8 -0
  34. package/src/models/testruncasesummary.ts +1 -1
  35. package/src/models/testrunstatusresponse.ts +12 -0
package/bin/mcp-server.js CHANGED
@@ -1759,7 +1759,7 @@ var init_versions = __esm(() => {
1759
1759
  version2 = {
1760
1760
  major: 4,
1761
1761
  minor: 5,
1762
- patch: 2
1762
+ patch: 4
1763
1763
  };
1764
1764
  });
1765
1765
 
@@ -3412,22 +3412,95 @@ function isRecursive(inst, stack) {
3412
3412
  result = true;
3413
3413
  };
3414
3414
  const def = inst._zod.def;
3415
- if (def.type === "lazy") {
3416
- check(inst._zod.innerType);
3417
- } else {
3418
- const shape = def.shape;
3419
- if (shape)
3420
- for (const key of Reflect.ownKeys(shape))
3421
- check(shape[key]);
3422
- for (const key in def) {
3423
- const value = def[key];
3424
- if (!value || typeof value !== "object")
3425
- continue;
3426
- if (value._zod)
3427
- check(value);
3428
- else if (Array.isArray(value))
3429
- for (const el of value)
3430
- check(el);
3415
+ const kind = def.type;
3416
+ switch (kind) {
3417
+ case "object": {
3418
+ for (const key of Reflect.ownKeys(def.shape))
3419
+ check(def.shape[key]);
3420
+ check(def.catchall);
3421
+ break;
3422
+ }
3423
+ case "array":
3424
+ check(def.element);
3425
+ break;
3426
+ case "tuple":
3427
+ for (const el of def.items)
3428
+ check(el);
3429
+ check(def.rest);
3430
+ break;
3431
+ case "record":
3432
+ case "map":
3433
+ check(def.keyType);
3434
+ check(def.valueType);
3435
+ break;
3436
+ case "set":
3437
+ check(def.valueType);
3438
+ break;
3439
+ case "union":
3440
+ for (const el of def.options)
3441
+ check(el);
3442
+ break;
3443
+ case "intersection":
3444
+ check(def.left);
3445
+ check(def.right);
3446
+ break;
3447
+ case "optional":
3448
+ case "nullable":
3449
+ case "default":
3450
+ case "prefault":
3451
+ case "catch":
3452
+ case "readonly":
3453
+ case "nonoptional":
3454
+ case "promise":
3455
+ case "success":
3456
+ check(def.innerType);
3457
+ break;
3458
+ case "pipe":
3459
+ check(def.in);
3460
+ check(def.out);
3461
+ break;
3462
+ case "function":
3463
+ check(def.input);
3464
+ check(def.output);
3465
+ break;
3466
+ case "lazy":
3467
+ check(inst._zod.innerType);
3468
+ break;
3469
+ case "template_literal":
3470
+ case "string":
3471
+ case "number":
3472
+ case "int":
3473
+ case "boolean":
3474
+ case "bigint":
3475
+ case "symbol":
3476
+ case "undefined":
3477
+ case "null":
3478
+ case "void":
3479
+ case "never":
3480
+ case "any":
3481
+ case "unknown":
3482
+ case "date":
3483
+ case "nan":
3484
+ case "enum":
3485
+ case "literal":
3486
+ case "file":
3487
+ case "transform":
3488
+ case "custom":
3489
+ break;
3490
+ default: {
3491
+ for (const key in def) {
3492
+ const desc = Object.getOwnPropertyDescriptor(def, key);
3493
+ if (!desc || desc.get)
3494
+ continue;
3495
+ const value = desc.value;
3496
+ if (!value || typeof value !== "object")
3497
+ continue;
3498
+ if (value._zod)
3499
+ check(value);
3500
+ else if (Array.isArray(value))
3501
+ for (const el of value)
3502
+ check(el);
3503
+ }
3431
3504
  }
3432
3505
  }
3433
3506
  stack.delete(inst);
@@ -4319,6 +4392,7 @@ function initializeContext(params) {
4319
4392
  cycles: params?.cycles ?? "ref",
4320
4393
  reused: params?.reused ?? "inline",
4321
4394
  intersections: [],
4395
+ deferred: [],
4322
4396
  external: params?.external ?? undefined
4323
4397
  };
4324
4398
  }
@@ -4663,6 +4737,8 @@ function finalize(ctx, schema) {
4663
4737
  compactTypeUnion(entry[1].def ?? entry[1].schema);
4664
4738
  }
4665
4739
  }
4740
+ for (const rewrite of ctx.deferred)
4741
+ rewrite();
4666
4742
  if (ctx.intersections.length) {
4667
4743
  const carriers = new Map;
4668
4744
  for (const seen of ctx.seen.values()) {
@@ -4824,6 +4900,68 @@ function inputOptin(schema) {
4824
4900
  }
4825
4901
  return schema._zod.optin;
4826
4902
  }
4903
+ function stringifyKeyNames(bySchema, json, visited) {
4904
+ if (json.$ref) {
4905
+ if (visited.has(json))
4906
+ return json;
4907
+ visited.add(json);
4908
+ const def = bySchema.get(json)?.def;
4909
+ if (!def)
4910
+ return json;
4911
+ const inlined = stringifyKeyNames(bySchema, def, visited);
4912
+ return inlined === def ? json : inlined;
4913
+ }
4914
+ for (const keyword of ["anyOf", "oneOf"]) {
4915
+ const branches = json[keyword];
4916
+ if (!Array.isArray(branches))
4917
+ continue;
4918
+ const mapped = branches.map((branch) => stringifyKeyNames(bySchema, branch, visited));
4919
+ if (mapped.some((branch, i) => branch !== branches[i]))
4920
+ json = { ...json, [keyword]: mapped };
4921
+ }
4922
+ const types = Array.isArray(json.type) ? json.type : [json.type];
4923
+ const numericType = !types.includes("string") && types.some((t) => t === "number" || t === "integer");
4924
+ const values = json.enum ?? (json.const !== undefined ? [json.const] : undefined);
4925
+ if (!numericType && !values?.some((v) => typeof v === "number"))
4926
+ return json;
4927
+ const { minimum, maximum: maximum2, exclusiveMinimum, exclusiveMaximum, multipleOf, format, id, ...rest } = json;
4928
+ if (rest.enum)
4929
+ rest.enum = rest.enum.map((v) => typeof v === "number" ? String(v) : v);
4930
+ else if (typeof rest.const === "number")
4931
+ rest.const = String(rest.const);
4932
+ if (!numericType)
4933
+ return rest;
4934
+ rest.type = "string";
4935
+ if (!values)
4936
+ rest.pattern = (types.includes("number") ? number : integer).source;
4937
+ return rest;
4938
+ }
4939
+ function rewriteKeyNames(ctx) {
4940
+ const bySchema = new Map;
4941
+ for (const entry of ctx.seen.values()) {
4942
+ if (entry.def && !bySchema.has(entry.schema))
4943
+ bySchema.set(entry.schema, entry);
4944
+ }
4945
+ const rewrites = new Map;
4946
+ for (const record of pendingRecords.get(ctx) ?? []) {
4947
+ const seen = ctx.seen.get(record);
4948
+ const names = (seen?.def ?? seen?.schema)?.propertyNames;
4949
+ if (!names || names === true || rewrites.has(names))
4950
+ continue;
4951
+ const rewritten = stringifyKeyNames(bySchema, names, new Set);
4952
+ if (rewritten !== names)
4953
+ rewrites.set(names, rewritten);
4954
+ }
4955
+ if (!rewrites.size)
4956
+ return;
4957
+ for (const entry of ctx.seen.values()) {
4958
+ for (const carrier of [entry.schema, entry.def]) {
4959
+ const rewritten = carrier && rewrites.get(carrier.propertyNames);
4960
+ if (rewritten)
4961
+ carrier.propertyNames = rewritten;
4962
+ }
4963
+ }
4964
+ }
4827
4965
  function serializeDefaultValue(value, schema, ctx, json, params) {
4828
4966
  let unrepresentable = false;
4829
4967
  const serialized = JSON.stringify(value, (_, val) => {
@@ -4890,12 +5028,12 @@ var formatMap, stringProcessor = (schema, ctx, _json, _params) => {
4890
5028
  if (contentEncoding)
4891
5029
  json.contentEncoding = contentEncoding;
4892
5030
  if (patterns && patterns.size > 0) {
4893
- const regexes = [...patterns];
4894
- if (regexes.length === 1)
4895
- json.pattern = regexes[0].source;
4896
- else if (regexes.length > 1) {
5031
+ const patternList = [...patterns];
5032
+ if (patternList.length === 1)
5033
+ json.pattern = patternList[0].source;
5034
+ else if (patternList.length > 1) {
4897
5035
  json.allOf = [
4898
- ...regexes.map((regex) => ({
5036
+ ...patternList.map((regex) => ({
4899
5037
  ...ctx.target === "draft-07" || ctx.target === "draft-04" || ctx.target === "openapi-3.0" ? { type: "string" } : {},
4900
5038
  pattern: regex.source
4901
5039
  }))
@@ -5198,7 +5336,7 @@ var formatMap, stringProcessor = (schema, ctx, _json, _params) => {
5198
5336
  json.minItems = minimum;
5199
5337
  if (typeof maximum2 === "number")
5200
5338
  json.maxItems = maximum2;
5201
- }, recordProcessor = (schema, ctx, _json, params) => {
5339
+ }, pendingRecords, recordProcessor = (schema, ctx, _json, params) => {
5202
5340
  const json = _json;
5203
5341
  const def = schema._zod.def;
5204
5342
  json.type = "object";
@@ -5220,6 +5358,13 @@ var formatMap, stringProcessor = (schema, ctx, _json, _params) => {
5220
5358
  ...params,
5221
5359
  path: [...params.path, "propertyNames"]
5222
5360
  });
5361
+ let pending = pendingRecords.get(ctx);
5362
+ if (!pending) {
5363
+ pending = [];
5364
+ pendingRecords.set(ctx, pending);
5365
+ ctx.deferred.push(() => rewriteKeyNames(ctx));
5366
+ }
5367
+ pending.push(schema);
5223
5368
  }
5224
5369
  json.additionalProperties = process2(def.valueType, ctx, {
5225
5370
  ...params,
@@ -5231,7 +5376,7 @@ var formatMap, stringProcessor = (schema, ctx, _json, _params) => {
5231
5376
  if (keyValues && !def.partial && !omittableOnInput) {
5232
5377
  const validKeyValues = [...keyValues].filter((v) => typeof v === "string" || typeof v === "number");
5233
5378
  if (validKeyValues.length > 0) {
5234
- json.required = validKeyValues;
5379
+ json.required = validKeyValues.map(String);
5235
5380
  }
5236
5381
  }
5237
5382
  }, nullableProcessor = (schema, ctx, json, params) => {
@@ -5310,6 +5455,7 @@ var formatMap, stringProcessor = (schema, ctx, _json, _params) => {
5310
5455
  seen.ref = innerType;
5311
5456
  }, allProcessors;
5312
5457
  var init_json_schema_processors = __esm(() => {
5458
+ init_regexes();
5313
5459
  init_to_json_schema();
5314
5460
  init_util();
5315
5461
  formatMap = {
@@ -5319,6 +5465,7 @@ var init_json_schema_processors = __esm(() => {
5319
5465
  json_string: "json-string",
5320
5466
  regex: ""
5321
5467
  };
5468
+ pendingRecords = new WeakMap;
5322
5469
  UNREPRESENTABLE_DEFAULT = Symbol();
5323
5470
  allProcessors = {
5324
5471
  string: stringProcessor,
@@ -26219,15 +26366,19 @@ var require_utils2 = __commonJS(function(exports, module) {
26219
26366
  if (!obj || typeof obj !== "object") {
26220
26367
  return false;
26221
26368
  }
26222
- return !!(obj.constructor && obj.constructor.isBuffer && obj.constructor.isBuffer(obj));
26369
+ return !!(obj.constructor && typeof obj.constructor.isBuffer === "function" && obj.constructor.isBuffer(obj));
26223
26370
  };
26224
26371
  var combine = function combine2(a, b, arrayLimit, plainObjects, throwOnLimitExceeded) {
26225
26372
  if (isOverflow(a)) {
26226
26373
  if (throwOnLimitExceeded) {
26227
26374
  throw new RangeError("Array limit exceeded. Only " + arrayLimit + " element" + (arrayLimit === 1 ? "" : "s") + " allowed in an array.");
26228
26375
  }
26229
- var newIndex = getMaxIndex(a) + 1;
26230
- a[newIndex] = b;
26376
+ var bValues = isArray(b) ? b : [b];
26377
+ var newIndex = getMaxIndex(a);
26378
+ for (var i = 0;i < bValues.length; ++i) {
26379
+ newIndex += 1;
26380
+ a[newIndex] = bValues[i];
26381
+ }
26231
26382
  setMaxIndex(a, newIndex);
26232
26383
  return a;
26233
26384
  }
@@ -26300,6 +26451,7 @@ var require_stringify = __commonJS(function(exports, module) {
26300
26451
  charsetSentinel: false,
26301
26452
  commaRoundTrip: false,
26302
26453
  delimiter: "&",
26454
+ depth: Infinity,
26303
26455
  encode: true,
26304
26456
  encodeDotInKeys: false,
26305
26457
  encoder: utils.encode,
@@ -26318,8 +26470,11 @@ var require_stringify = __commonJS(function(exports, module) {
26318
26470
  return typeof v === "string" || typeof v === "number" || typeof v === "boolean" || typeof v === "symbol" || typeof v === "bigint";
26319
26471
  };
26320
26472
  var sentinel = {};
26321
- var stringify = function stringify2(object2, prefix, generateArrayPrefix, commaRoundTrip, allowEmptyArrays, strictNullHandling, skipNulls, encodeDotInKeys, encoder, filter, sort, allowDots, serializeDate, format, formatter, encodeValuesOnly, charset, sideChannel) {
26473
+ var stringify = function stringify2(object2, prefix, generateArrayPrefix, commaRoundTrip, allowEmptyArrays, strictNullHandling, skipNulls, encodeDotInKeys, encoder, filter, sort, allowDots, serializeDate, format, formatter, encodeValuesOnly, charset, sideChannel, depth, currentDepth) {
26322
26474
  var obj = object2;
26475
+ if (currentDepth > depth) {
26476
+ throw new RangeError("Input depth exceeded depth option of " + depth);
26477
+ }
26323
26478
  var tmpSc = sideChannel;
26324
26479
  var step = 0;
26325
26480
  var findFlag = false;
@@ -26337,9 +26492,8 @@ var require_stringify = __commonJS(function(exports, module) {
26337
26492
  step = 0;
26338
26493
  }
26339
26494
  }
26340
- if (typeof filter === "function") {
26341
- obj = filter(prefix, obj);
26342
- } else if (obj instanceof Date) {
26495
+ obj = typeof filter === "function" ? filter(prefix, obj) : obj;
26496
+ if (obj instanceof Date) {
26343
26497
  obj = serializeDate(obj);
26344
26498
  } else if (generateArrayPrefix === "comma" && isArray(obj)) {
26345
26499
  obj = utils.maybeMap(obj, function(value2) {
@@ -26382,7 +26536,7 @@ var require_stringify = __commonJS(function(exports, module) {
26382
26536
  }
26383
26537
  var encodedPrefix = encodeDotInKeys ? String(prefix).replace(/\./g, "%2E") : String(prefix);
26384
26538
  var adjustedPrefix = commaRoundTrip && isArray(obj) && obj.length === 1 ? encodedPrefix + "[]" : encodedPrefix;
26385
- if (allowEmptyArrays && isArray(obj) && obj.length === 0) {
26539
+ if (allowEmptyArrays && isArray(obj) && obj.length === 0 && Object.keys(obj).length === 0) {
26386
26540
  return adjustedPrefix + "[]";
26387
26541
  }
26388
26542
  for (var j = 0;j < objKeys.length; ++j) {
@@ -26396,7 +26550,7 @@ var require_stringify = __commonJS(function(exports, module) {
26396
26550
  sideChannel.set(object2, step);
26397
26551
  var valueSideChannel = getSideChannel();
26398
26552
  valueSideChannel.set(sentinel, sideChannel);
26399
- pushToArray(values, stringify2(value, keyPrefix, generateArrayPrefix, commaRoundTrip, allowEmptyArrays, strictNullHandling, skipNulls, encodeDotInKeys, generateArrayPrefix === "comma" && encodeValuesOnly && isArray(obj) ? null : encoder, filter, sort, allowDots, serializeDate, format, formatter, encodeValuesOnly, charset, valueSideChannel));
26553
+ pushToArray(values, stringify2(value, keyPrefix, generateArrayPrefix, commaRoundTrip, allowEmptyArrays, strictNullHandling, skipNulls, encodeDotInKeys, generateArrayPrefix === "comma" && encodeValuesOnly && isArray(obj) ? null : encoder, filter, sort, allowDots, serializeDate, format, formatter, encodeValuesOnly, charset, valueSideChannel, depth, currentDepth + 1));
26400
26554
  }
26401
26555
  return values;
26402
26556
  };
@@ -26450,6 +26604,7 @@ var require_stringify = __commonJS(function(exports, module) {
26450
26604
  charsetSentinel: typeof opts.charsetSentinel === "boolean" ? opts.charsetSentinel : defaults.charsetSentinel,
26451
26605
  commaRoundTrip: !!opts.commaRoundTrip,
26452
26606
  delimiter: typeof opts.delimiter === "undefined" ? defaults.delimiter : opts.delimiter,
26607
+ depth: typeof opts.depth === "number" ? opts.depth : defaults.depth,
26453
26608
  encode: typeof opts.encode === "boolean" ? opts.encode : defaults.encode,
26454
26609
  encodeDotInKeys: typeof opts.encodeDotInKeys === "boolean" ? opts.encodeDotInKeys : defaults.encodeDotInKeys,
26455
26610
  encoder: typeof opts.encoder === "function" ? opts.encoder : defaults.encoder,
@@ -26497,7 +26652,8 @@ var require_stringify = __commonJS(function(exports, module) {
26497
26652
  if (options.skipNulls && value === null) {
26498
26653
  continue;
26499
26654
  }
26500
- pushToArray(keys, stringify(value, key, generateArrayPrefix, commaRoundTrip, options.allowEmptyArrays, options.strictNullHandling, options.skipNulls, options.encodeDotInKeys, options.encode ? options.encoder : null, options.filter, options.sort, options.allowDots, options.serializeDate, options.format, options.formatter, options.encodeValuesOnly, options.charset, sideChannel));
26655
+ var encodedKey = options.encodeDotInKeys ? String(key).replace(/\./g, "%2E") : String(key);
26656
+ pushToArray(keys, stringify(value, encodedKey, generateArrayPrefix, commaRoundTrip, options.allowEmptyArrays, options.strictNullHandling, options.skipNulls, options.encodeDotInKeys, options.encode ? options.encoder : null, options.filter, options.sort, options.allowDots, options.serializeDate, options.format, options.formatter, options.encodeValuesOnly, options.charset, sideChannel, options.depth, 0));
26501
26657
  }
26502
26658
  var joined = keys.join(options.delimiter);
26503
26659
  var prefix = options.addQueryPrefix === true ? "?" : "";
@@ -26546,9 +26702,9 @@ var require_parse = __commonJS(function(exports, module) {
26546
26702
  return String.fromCharCode(parseInt(numberStr, 10));
26547
26703
  });
26548
26704
  };
26549
- var parseArrayValue = function(val, options, currentArrayLength, isFlatArrayValue) {
26705
+ var parseArrayValue = function(val, options, currentArrayLength) {
26550
26706
  if (val && typeof val === "string" && options.comma && val.indexOf(",") > -1) {
26551
- if (isFlatArrayValue && options.throwOnLimitExceeded) {
26707
+ if (options.throwOnLimitExceeded) {
26552
26708
  var commaCount = 0;
26553
26709
  var commaIndex = val.indexOf(",");
26554
26710
  while (commaIndex > -1) {
@@ -26608,7 +26764,7 @@ var require_parse = __commonJS(function(exports, module) {
26608
26764
  } else {
26609
26765
  key = options.decoder(part.slice(0, pos), defaults.decoder, charset, "key");
26610
26766
  if (key !== null) {
26611
- val = utils.maybeMap(parseArrayValue(part.slice(pos + 1), options, isArray(obj[key]) ? obj[key].length : 0, part.indexOf("[]=") === -1), function(encodedVal) {
26767
+ val = utils.maybeMap(parseArrayValue(part.slice(pos + 1), options, isArray(obj[key]) ? obj[key].length : 0), function(encodedVal) {
26612
26768
  return options.decoder(encodedVal, defaults.decoder, charset, "value");
26613
26769
  });
26614
26770
  }
@@ -52864,9 +53020,9 @@ var init_config = __esm(() => {
52864
53020
  SDK_METADATA = {
52865
53021
  language: "typescript",
52866
53022
  openapiDocVersion: "0.1.0",
52867
- sdkVersion: "0.0.47",
53023
+ sdkVersion: "0.0.49",
52868
53024
  genVersion: "2.915.1",
52869
- userAgent: "speakeasy-sdk/mcp-typescript 0.0.47 2.915.1 0.1.0 @dalmia/calibrate-mcp"
53025
+ userAgent: "speakeasy-sdk/mcp-typescript 0.0.49 2.915.1 0.1.0 @dalmia/calibrate-mcp"
52870
53026
  };
52871
53027
  });
52872
53028
 
@@ -56058,11 +56214,13 @@ var init_testcaseresult = __esm(() => {
56058
56214
  judge_results: array(JudgeResult$zodSchema).nullable().optional().describe("One verdict for each evaluator"),
56059
56215
  latency_ms: number2().nullable().optional().describe("How long the agent took to respond, in milliseconds"),
56060
56216
  name: string2().nullable().optional().describe("Name of the test"),
56217
+ not_run: boolean2().default(false).describe("Whether this case never started, because a user stopped the run first. It is counted neither as passed nor as failed"),
56061
56218
  output: TestOutput$zodSchema.nullable().optional().describe("The agent's output for this case"),
56062
56219
  passed: boolean2().nullable().optional().describe("Whether the case passed"),
56063
56220
  reasoning: string2().nullable().optional().describe("The judge's reasoning, or the tool-call diff for a tool-call test"),
56064
56221
  test_case: record(string2(), any()).nullable().optional().describe("The test case definition that was run"),
56065
- test_case_id: string2().nullable().optional().describe("ID of the test case within the run")
56222
+ test_case_id: string2().nullable().optional().describe("ID of the test case within the run"),
56223
+ unanswered: boolean2().default(false).describe("Whether this case produced no answer because the agent or the judge could not be reached, in which case `reasoning` carries the error and `passed` is not a verdict on the agent")
56066
56224
  });
56067
56225
  });
56068
56226
 
@@ -56127,6 +56285,7 @@ var init_benchmarkstatusresponse = __esm(() => {
56127
56285
  init_taskstatus();
56128
56286
  init_testrunevaluator();
56129
56287
  BenchmarkStatusResponse$zodSchema = object({
56288
+ aborted: boolean2().default(false).describe("Whether a user stopped this run before it finished. The results collected up to that point are kept, and test cases that never ran are counted neither as passed nor as failed"),
56130
56289
  error: boolean2().default(false).describe("True if the run failed"),
56131
56290
  evaluators: array(TestRunEvaluator$zodSchema).nullable().optional().describe("The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`"),
56132
56291
  is_public: boolean2().default(false).describe("Whether the run is shared publicly"),
@@ -56287,6 +56446,7 @@ var init_testrunstatusresponse = __esm(() => {
56287
56446
  init_testcaseresult();
56288
56447
  init_testrunevaluator();
56289
56448
  TestRunStatusResponse$zodSchema = object({
56449
+ aborted: boolean2().default(false).describe("Whether a user stopped this run before it finished. The results collected up to that point are kept, and test cases that never ran are counted neither as passed nor as failed"),
56290
56450
  cost: record(string2(), any()).nullable().optional().describe("Aggregated cost as `{mean, min, max, count}` (USD)"),
56291
56451
  error: boolean2().default(false).describe("True if the run failed"),
56292
56452
  evaluators: array(TestRunEvaluator$zodSchema).nullable().optional().describe("The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`"),
@@ -56297,10 +56457,12 @@ var init_testrunstatusresponse = __esm(() => {
56297
56457
  results: array(TestCaseResult$zodSchema).nullable().optional().describe("Results for each test case"),
56298
56458
  share_token: string2().nullable().optional().describe("Token for building the public share URL"),
56299
56459
  status: TaskStatus$zodSchema,
56460
+ stopped_early: boolean2().default(false).describe("Whether the run stopped before starting every test case, after too many failed in a row"),
56300
56461
  task_id: string2().describe("Test run job ID"),
56301
56462
  test_uuids: array(string2()).nullable().optional().describe("IDs of the tests this run executed, in run order"),
56302
56463
  total_tests: int().nullable().optional().describe("Total number of test cases"),
56303
- total_tokens: record(string2(), any()).nullable().optional().describe("Aggregated token usage as `{mean, min, max, count}`")
56464
+ total_tokens: record(string2(), any()).nullable().optional().describe("Aggregated token usage as `{mean, min, max, count}`"),
56465
+ unanswered_tests: int().nullable().optional().describe("Number of test cases that produced no answer because the agent or the judge could not be reached, which makes the pass rate an unfair measure of the agent")
56304
56466
  });
56305
56467
  });
56306
56468
 
@@ -56822,7 +56984,7 @@ var init_testruncasesummary = __esm(() => {
56822
56984
  init_zod();
56823
56985
  TestRunCaseSummary$zodSchema = object({
56824
56986
  name: string2().nullable().optional().describe("Name of the test case"),
56825
- passed: boolean2().nullable().optional().describe("Whether the case passed (null if it errored or is still running)")
56987
+ passed: boolean2().nullable().optional().describe("Whether the case passed (null while the case is still running)")
56826
56988
  }).describe(`Flat summary for one test case in the run-LIST endpoints. Carries only
56827
56989
  enough to render a run's pass/fail breakdown and a case name. The full detail
56828
56990
  for each case (agent output, judge verdicts, reasoning, latency, cost, the
@@ -56843,6 +57005,7 @@ var init_agenttestrunlistitem = __esm(() => {
56843
57005
  "llm-benchmark"
56844
57006
  ]).describe("What kind of run this is:\n- `llm-unit-test`: a single run of the agent's tests\n- `llm-benchmark`: a multi-model comparison");
56845
57007
  AgentTestRunListItem$zodSchema = object({
57008
+ aborted: boolean2().default(false).describe("Whether a user stopped this run before it finished. The results collected up to that point are kept, and test cases that never ran are counted neither as passed nor as failed"),
56846
57009
  cost: record(string2(), any()).nullable().optional().describe("Aggregated cost as `{mean, min, max, count}` (USD)"),
56847
57010
  created_at: string2().describe("When the run was created (ISO 8601 UTC)"),
56848
57011
  error: boolean2().default(false).describe("True if the run failed"),
@@ -56859,6 +57022,7 @@ var init_agenttestrunlistitem = __esm(() => {
56859
57022
  total_tests: int().nullable().optional().describe("Total number of test cases"),
56860
57023
  total_tokens: record(string2(), any()).nullable().optional().describe("Aggregated token usage as `{mean, min, max, count}`"),
56861
57024
  type: AgentTestRunListItemType$zodSchema.describe("What kind of run this is:\n- `llm-unit-test`: a single run of the agent's tests\n- `llm-benchmark`: a multi-model comparison"),
57025
+ unanswered_tests: int().nullable().optional().describe("Number of test cases that produced no answer because the agent or the judge could not be reached, which makes the pass rate an unfair measure of the agent"),
56862
57026
  updated_at: string2().describe("When the run was last updated (ISO 8601 UTC)"),
56863
57027
  uuid: string2().describe("Test run job ID")
56864
57028
  });
@@ -61482,7 +61646,7 @@ hits its trace limit.
61482
61646
  function createMCPServer(deps) {
61483
61647
  const server = new McpServer({
61484
61648
  name: "CalibrateMcp",
61485
- version: "0.0.47"
61649
+ version: "0.0.49"
61486
61650
  });
61487
61651
  const getClient = deps.getSDK || (() => new CalibrateMcpCore({
61488
61652
  security: deps.security,
@@ -62760,7 +62924,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
62760
62924
  <h1>Instructions</h1>
62761
62925
  <p>One-click installation for Claude Desktop users</p>
62762
62926
  <div class="instruction-item">
62763
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.47/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
62927
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.49/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
62764
62928
  \uD83D\uDCE5 Download MCP Bundle
62765
62929
  </a>
62766
62930
  </div>
@@ -65647,7 +65811,7 @@ var routes = buildRouteMap({
65647
65811
  var app = buildApplication(routes, {
65648
65812
  name: "mcp",
65649
65813
  versionInfo: {
65650
- currentVersion: "0.0.47"
65814
+ currentVersion: "0.0.49"
65651
65815
  }
65652
65816
  });
65653
65817
  run(app, process4.argv.slice(2), buildContext(process4));
@@ -65655,5 +65819,5 @@ export {
65655
65819
  app
65656
65820
  };
65657
65821
 
65658
- //# debugId=D6CE9DB41F5C613964756E2164756E21
65822
+ //# debugId=F5A9C365B934497264756E2164756E21
65659
65823
  //# sourceMappingURL=mcp-server.js.map