@dalmia/calibrate-mcp 0.0.47 → 0.0.49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/bin/mcp-server.js +210 -46
  2. package/bin/mcp-server.js.map +18 -18
  3. package/esm/landing-page.js +1 -1
  4. package/esm/lib/config.d.ts +2 -2
  5. package/esm/lib/config.js +2 -2
  6. package/esm/mcp-server/mcp-server.js +1 -1
  7. package/esm/mcp-server/server.js +1 -1
  8. package/esm/models/agenttestrunlistitem.d.ts +2 -0
  9. package/esm/models/agenttestrunlistitem.d.ts.map +1 -1
  10. package/esm/models/agenttestrunlistitem.js +2 -0
  11. package/esm/models/agenttestrunlistitem.js.map +1 -1
  12. package/esm/models/benchmarkstatusresponse.d.ts +1 -0
  13. package/esm/models/benchmarkstatusresponse.d.ts.map +1 -1
  14. package/esm/models/benchmarkstatusresponse.js +1 -0
  15. package/esm/models/benchmarkstatusresponse.js.map +1 -1
  16. package/esm/models/testcaseresult.d.ts +2 -0
  17. package/esm/models/testcaseresult.d.ts.map +1 -1
  18. package/esm/models/testcaseresult.js +2 -0
  19. package/esm/models/testcaseresult.js.map +1 -1
  20. package/esm/models/testruncasesummary.js +1 -1
  21. package/esm/models/testruncasesummary.js.map +1 -1
  22. package/esm/models/testrunstatusresponse.d.ts +3 -0
  23. package/esm/models/testrunstatusresponse.d.ts.map +1 -1
  24. package/esm/models/testrunstatusresponse.js +3 -0
  25. package/esm/models/testrunstatusresponse.js.map +1 -1
  26. package/package.json +1 -1
  27. package/src/landing-page.ts +1 -1
  28. package/src/lib/config.ts +2 -2
  29. package/src/mcp-server/mcp-server.ts +1 -1
  30. package/src/mcp-server/server.ts +1 -1
  31. package/src/models/agenttestrunlistitem.ts +8 -0
  32. package/src/models/benchmarkstatusresponse.ts +4 -0
  33. package/src/models/testcaseresult.ts +8 -0
  34. package/src/models/testruncasesummary.ts +1 -1
  35. package/src/models/testrunstatusresponse.ts +12 -0
package/esm/landing-page.js CHANGED
@@ -915,7 +915,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
915
915
  <h1>Instructions</h1>
916
916
  <p>One-click installation for Claude Desktop users</p>
917
917
  <div class="instruction-item">
918
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.47/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
918
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.49/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
919
919
  📥 Download MCP Bundle
920
920
  </a>
921
921
  </div>
package/esm/lib/config.d.ts CHANGED
@@ -35,8 +35,8 @@ export declare function serverURLFromOptions(options: SDKOptions): URL | null;
35
35
  export declare const SDK_METADATA: {
36
36
  readonly language: "typescript";
37
37
  readonly openapiDocVersion: "0.1.0";
38
- readonly sdkVersion: "0.0.47";
38
+ readonly sdkVersion: "0.0.49";
39
39
  readonly genVersion: "2.915.1";
40
- readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.47 2.915.1 0.1.0 @dalmia/calibrate-mcp";
40
+ readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.49 2.915.1 0.1.0 @dalmia/calibrate-mcp";
41
41
  };
42
42
  //# sourceMappingURL=config.d.ts.map
package/esm/lib/config.js CHANGED
@@ -27,8 +27,8 @@ export function serverURLFromOptions(options) {
27
27
  export const SDK_METADATA = {
28
28
  language: "typescript",
29
29
  openapiDocVersion: "0.1.0",
30
- sdkVersion: "0.0.47",
30
+ sdkVersion: "0.0.49",
31
31
  genVersion: "2.915.1",
32
- userAgent: "speakeasy-sdk/mcp-typescript 0.0.47 2.915.1 0.1.0 @dalmia/calibrate-mcp",
32
+ userAgent: "speakeasy-sdk/mcp-typescript 0.0.49 2.915.1 0.1.0 @dalmia/calibrate-mcp",
33
33
  };
34
34
  //# sourceMappingURL=config.js.map
package/esm/mcp-server/mcp-server.js CHANGED
@@ -18,7 +18,7 @@ const routes = buildRouteMap({
18
18
  export const app = buildApplication(routes, {
19
19
  name: "mcp",
20
20
  versionInfo: {
21
- currentVersion: "0.0.47",
21
+ currentVersion: "0.0.49",
22
22
  },
23
23
  });
24
24
  run(app, process.argv.slice(2), buildContext(process));
package/esm/mcp-server/server.js CHANGED
@@ -50,7 +50,7 @@ import { tool$tracesCreate } from "./tools/tracesCreate.js";
50
50
  export function createMCPServer(deps) {
51
51
  const server = new McpServer({
52
52
  name: "CalibrateMcp",
53
- version: "0.0.47",
53
+ version: "0.0.49",
54
54
  });
55
55
  const getClient = deps.getSDK || (() => new CalibrateMcpCore({
56
56
  security: deps.security,
package/esm/models/agenttestrunlistitem.d.ts CHANGED
@@ -49,6 +49,8 @@ export type AgentTestRunListItem = {
49
49
  [k: string]: any;
50
50
  } | null | undefined;
51
51
  model_results?: Array<ModelRunSummary> | null | undefined;
52
+ unanswered_tests?: number | null | undefined;
53
+ aborted?: boolean | undefined;
52
54
  error?: boolean | undefined;
53
55
  is_public?: boolean | undefined;
54
56
  share_token?: string | null | undefined;
package/esm/models/agenttestrunlistitem.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agenttestrunlistitem.d.ts","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,eAAe,EAEhB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EACL,kBAAkB,EAEnB,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB;;;CAG3B,CAAC;AACX;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,UAAU,CAC/C,OAAO,wBAAwB,CAChC,CAAC;AAEF,eAAO,MAAM,kCAAkC;;;EAK9C,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,wBAAwB,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,SAAS,CAAC;IACjD,OAAO,CAAC,EAAE,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,aAAa,CAAC,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC1D,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,8BAA8B,EAAE,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAkDvE,CAAC"}
1
+ {"version":3,"file":"agenttestrunlistitem.d.ts","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,eAAe,EAEhB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EACL,kBAAkB,EAEnB,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB;;;CAG3B,CAAC;AACX;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,UAAU,CAC/C,OAAO,wBAAwB,CAChC,CAAC;AAEF,eAAO,MAAM,kCAAkC;;;EAK9C,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,wBAAwB,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,SAAS,CAAC;IACjD,OAAO,CAAC,EAAE,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,aAAa,CAAC,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC1D,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7C,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,8BAA8B,EAAE,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAwDvE,CAAC"}
package/esm/models/agenttestrunlistitem.js CHANGED
@@ -23,6 +23,7 @@ export const AgentTestRunListItemType$zodSchema = z.enum([
23
23
  ]).describe("What kind of run this is:\n- `llm-unit-test`: a single run of the agent's tests\n- `llm-benchmark`: a multi-model comparison");
24
24
  export const AgentTestRunListItem$zodSchema = z
25
25
  .object({
26
+ aborted: z.boolean().default(false).describe("Whether a user stopped this run before it finished. The results collected up to that point are kept, and test cases that never ran are counted neither as passed nor as failed"),
26
27
  cost: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated cost as `{mean, min, max, count}` (USD)"),
27
28
  created_at: z.string().describe("When the run was created (ISO 8601 UTC)"),
28
29
  error: z.boolean().default(false).describe("True if the run failed"),
@@ -41,6 +42,7 @@ export const AgentTestRunListItem$zodSchema = z
41
42
  total_tests: z.int().nullable().optional().describe("Total number of test cases"),
42
43
  total_tokens: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated token usage as `{mean, min, max, count}`"),
43
44
  type: AgentTestRunListItemType$zodSchema.describe("What kind of run this is:\n- `llm-unit-test`: a single run of the agent's tests\n- `llm-benchmark`: a multi-model comparison"),
45
+ unanswered_tests: z.int().nullable().optional().describe("Number of test cases that produced no answer because the agent or the judge could not be reached, which makes the pass rate an unfair measure of the agent"),
44
46
  updated_at: z.string().describe("When the run was last updated (ISO 8601 UTC)"),
45
47
  uuid: z.string().describe("Test run job ID"),
46
48
  });
package/esm/models/agenttestrunlistitem.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agenttestrunlistitem.js","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,4BAA4B,GAC7B,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,WAAW,EAAE,eAAe;IAC5B,YAAY,EAAE,eAAe;CACrB,CAAC;AAYX,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,CAAC,IAAI,CAAC;IACvD,eAAe;IACf,eAAe;CAChB,CAAC,CAAC,QAAQ,CACT,8HAA8H,CAC/H,CAAC;AAuBF,MAAM,CAAC,MAAM,8BAA8B,GAAoC,CAAC;KAC7E,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IAC1E,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjE,sQAAsQ,CACvQ;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,iEAAiE,CAClE;IACD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACpE,QAAQ,CACP,8FAA8F,CAC/F;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,gFAAgF,CACjF;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACjE,QAAQ,CACP,mFAAmF,CACpF;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,IAAI,EAAE,kCAAkC,CAAC,QAAQ,CAC/C,8HAA8H,CAC/H;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,8CAA8C,CAC/C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;CAC7C,CAAC,CAAC"}
1
+ {"version":3,"file":"agenttestrunlistitem.js","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,4BAA4B,GAC7B,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,WAAW,EAAE,eAAe;IAC5B,YAAY,EAAE,eAAe;CACrB,CAAC;AAYX,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,CAAC,IAAI,CAAC;IACvD,eAAe;IACf,eAAe;CAChB,CAAC,CAAC,QAAQ,CACT,8HAA8H,CAC/H,CAAC;AAyBF,MAAM,CAAC,MAAM,8BAA8B,GAAoC,CAAC;KAC7E,MAAM,CAAC;IACN,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC1C,gLAAgL,CACjL;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IAC1E,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjE,sQAAsQ,CACvQ;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,iEAAiE,CAClE;IACD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACpE,QAAQ,CACP,8FAA8F,CAC/F;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,gFAAgF,CACjF;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACjE,QAAQ,CACP,mFAAmF,CACpF;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,IAAI,EAAE,kCAAkC,CAAC,QAAQ,CAC/C,8HAA8H,CAC/H;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,8CAA8C,CAC/C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;CAC7C,CAAC,CAAC"}
package/esm/models/benchmarkstatusresponse.d.ts CHANGED
@@ -11,6 +11,7 @@ export type BenchmarkStatusResponse = {
11
11
  leaderboard_summary?: Array<{
12
12
  [k: string]: any;
13
13
  }> | null | undefined;
14
+ aborted?: boolean | undefined;
14
15
  error?: boolean | undefined;
15
16
  is_public?: boolean | undefined;
16
17
  share_token?: string | null | undefined;
package/esm/models/benchmarkstatusresponse.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"benchmarkstatusresponse.d.ts","sourceRoot":"","sources":["../../src/models/benchmarkstatusresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,uBAAuB,GAAG;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,aAAa,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,mBAAmB,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACrE,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,iCAAiC,EAAE,CAAC,CAAC,OAAO,CACvD,uBAAuB,CAyBvB,CAAC"}
1
+ {"version":3,"file":"benchmarkstatusresponse.d.ts","sourceRoot":"","sources":["../../src/models/benchmarkstatusresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,uBAAuB,GAAG;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,aAAa,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,mBAAmB,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACrE,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,iCAAiC,EAAE,CAAC,CAAC,OAAO,CACvD,uBAAuB,CA4BvB,CAAC"}
package/esm/models/benchmarkstatusresponse.js CHANGED
@@ -6,6 +6,7 @@ import { ModelResult$zodSchema } from "./modelresult.js";
6
6
  import { TaskStatus$zodSchema } from "./taskstatus.js";
7
7
  import { TestRunEvaluator$zodSchema, } from "./testrunevaluator.js";
8
8
  export const BenchmarkStatusResponse$zodSchema = z.object({
9
+ aborted: z.boolean().default(false).describe("Whether a user stopped this run before it finished. The results collected up to that point are kept, and test cases that never ran are counted neither as passed nor as failed"),
9
10
  error: z.boolean().default(false).describe("True if the run failed"),
10
11
  evaluators: z.array(TestRunEvaluator$zodSchema).nullable().optional()
11
12
  .describe("The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`"),
package/esm/models/benchmarkstatusresponse.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"benchmarkstatusresponse.js","sourceRoot":"","sources":["../../src/models/benchmarkstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAc/B,MAAM,CAAC,MAAM,iCAAiC,GAE1C,CAAC,CAAC,MAAM,CAAC;IACX,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,mBAAmB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;SACnE,QAAQ,EAAE,CAAC,QAAQ,CAClB,oMAAoM,CACrM;IACH,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC1E,wBAAwB,CACzB;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;IACpD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,wDAAwD,CACzD;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"benchmarkstatusresponse.js","sourceRoot":"","sources":["../../src/models/benchmarkstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAe/B,MAAM,CAAC,MAAM,iCAAiC,GAE1C,CAAC,CAAC,MAAM,CAAC;IACX,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC1C,gLAAgL,CACjL;IACD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,mBAAmB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;SACnE,QAAQ,EAAE,CAAC,QAAQ,CAClB,oMAAoM,CACrM;IACH,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC1E,wBAAwB,CACzB;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;IACpD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,wDAAwD,CACzD;CACF,CAAC,CAAC"}
package/esm/models/testcaseresult.d.ts CHANGED
@@ -16,6 +16,8 @@ export type TestCaseResult = {
16
16
  judge_results?: Array<JudgeResult> | null | undefined;
17
17
  latency_ms?: number | null | undefined;
18
18
  cost?: number | null | undefined;
19
+ unanswered?: boolean | undefined;
20
+ not_run?: boolean | undefined;
19
21
  };
20
22
  export declare const TestCaseResult$zodSchema: z.ZodType<TestCaseResult>;
21
23
  //# sourceMappingURL=testcaseresult.d.ts.map
package/esm/models/testcaseresult.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testcaseresult.d.ts","sourceRoot":"","sources":["../../src/models/testcaseresult.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AAEnE,MAAM,MAAM,cAAc,GAAG;IAC3B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACzC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,MAAM,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACtC,MAAM,CAAC,EAAE,UAAU,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,SAAS,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACpD,MAAM,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,aAAa,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,eAAO,MAAM,wBAAwB,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAyB7D,CAAC"}
1
+ {"version":3,"file":"testcaseresult.d.ts","sourceRoot":"","sources":["../../src/models/testcaseresult.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AAEnE,MAAM,MAAM,cAAc,GAAG;IAC3B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACzC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,MAAM,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACtC,MAAM,CAAC,EAAE,UAAU,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,SAAS,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACpD,MAAM,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,aAAa,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,UAAU,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACjC,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAC/B,CAAC;AAEF,eAAO,MAAM,wBAAwB,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CA+B7D,CAAC"}
package/esm/models/testcaseresult.js CHANGED
@@ -10,10 +10,12 @@ export const TestCaseResult$zodSchema = z.object({
10
10
  judge_results: z.array(JudgeResult$zodSchema).nullable().optional().describe("One verdict for each evaluator"),
11
11
  latency_ms: z.number().nullable().optional().describe("How long the agent took to respond, in milliseconds"),
12
12
  name: z.string().nullable().optional().describe("Name of the test"),
13
+ not_run: z.boolean().default(false).describe("Whether this case never started, because a user stopped the run first. It is counted neither as passed nor as failed"),
13
14
  output: TestOutput$zodSchema.nullable().optional().describe("The agent's output for this case"),
14
15
  passed: z.boolean().nullable().optional().describe("Whether the case passed"),
15
16
  reasoning: z.string().nullable().optional().describe("The judge's reasoning, or the tool-call diff for a tool-call test"),
16
17
  test_case: z.record(z.string(), z.any()).nullable().optional().describe("The test case definition that was run"),
17
18
  test_case_id: z.string().nullable().optional().describe("ID of the test case within the run"),
19
+ unanswered: z.boolean().default(false).describe("Whether this case produced no answer because the agent or the judge could not be reached, in which case `reasoning` carries the error and `passed` is not a verdict on the agent"),
18
20
  });
19
21
  //# sourceMappingURL=testcaseresult.js.map
package/esm/models/testcaseresult.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testcaseresult.js","sourceRoot":"","sources":["../../src/models/testcaseresult.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAenE,MAAM,CAAC,MAAM,wBAAwB,GAA8B,CAAC,CAAC,MAAM,CAAC;IAC1E,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC1E,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,uHAAuH,CACxH;IACD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC1E,gCAAgC,CACjC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnD,qDAAqD,CACtD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACnE,MAAM,EAAE,oBAAoB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACzD,kCAAkC,CACnC;IACD,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC7E,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClD,mEAAmE,CACpE;IACD,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACrE,uCAAuC,CACxC;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACrD,oCAAoC,CACrC;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"testcaseresult.js","sourceRoot":"","sources":["../../src/models/testcaseresult.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAiBnE,MAAM,CAAC,MAAM,wBAAwB,GAA8B,CAAC,CAAC,MAAM,CAAC;IAC1E,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC1E,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,uHAAuH,CACxH;IACD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC1E,gCAAgC,CACjC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnD,qDAAqD,CACtD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACnE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC1C,sHAAsH,CACvH;IACD,MAAM,EAAE,oBAAoB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACzD,kCAAkC,CACnC;IACD,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC7E,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClD,mEAAmE,CACpE;IACD,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACrE,uCAAuC,CACxC;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACrD,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC7C,kLAAkL,CACnL;CACF,CAAC,CAAC"}
package/esm/models/testruncasesummary.js CHANGED
@@ -5,6 +5,6 @@ import * as z from "zod";
5
5
  export const TestRunCaseSummary$zodSchema = z
6
6
  .object({
7
7
  name: z.string().nullable().optional().describe("Name of the test case"),
8
- passed: z.boolean().nullable().optional().describe("Whether the case passed (null if it errored or is still running)"),
8
+ passed: z.boolean().nullable().optional().describe("Whether the case passed (null while the case is still running)"),
9
9
  }).describe("Flat summary for one test case in the run-LIST endpoints. Carries only\nenough to render a run's pass/fail breakdown and a case name. The full detail\nfor each case (agent output, judge verdicts, reasoning, latency, cost, the\ntest-case definition) lives on the run-DETAIL endpoint\n(`GET /agent-tests/run/{task_id}`).");
10
10
  //# sourceMappingURL=testruncasesummary.js.map
package/esm/models/testruncasesummary.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testruncasesummary.js","sourceRoot":"","sources":["../../src/models/testruncasesummary.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAgBzB,MAAM,CAAC,MAAM,4BAA4B,GAAkC,CAAC;KACzE,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACxE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChD,kEAAkE,CACnE;CACF,CAAC,CAAC,QAAQ,CACT,gUAAgU,CACjU,CAAC"}
1
+ {"version":3,"file":"testruncasesummary.js","sourceRoot":"","sources":["../../src/models/testruncasesummary.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAgBzB,MAAM,CAAC,MAAM,4BAA4B,GAAkC,CAAC;KACzE,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACxE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChD,gEAAgE,CACjE;CACF,CAAC,CAAC,QAAQ,CACT,gUAAgU,CACjU,CAAC"}
package/esm/models/testrunstatusresponse.d.ts CHANGED
@@ -20,6 +20,9 @@ export type TestRunStatusResponse = {
20
20
  } | null | undefined;
21
21
  evaluators?: Array<TestRunEvaluator> | null | undefined;
22
22
  results?: Array<TestCaseResult> | null | undefined;
23
+ unanswered_tests?: number | null | undefined;
24
+ stopped_early?: boolean | undefined;
25
+ aborted?: boolean | undefined;
23
26
  error?: boolean | undefined;
24
27
  is_public?: boolean | undefined;
25
28
  share_token?: string | null | undefined;
package/esm/models/testrunstatusresponse.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testrunstatusresponse.d.ts","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,cAAc,EAA4B,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,OAAO,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnD,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAuCzE,CAAC"}
1
+ {"version":3,"file":"testrunstatusresponse.d.ts","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,cAAc,EAA4B,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,OAAO,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnD,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7C,aAAa,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACpC,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAgDzE,CAAC"}
package/esm/models/testrunstatusresponse.js CHANGED
@@ -6,6 +6,7 @@ import { TaskStatus$zodSchema } from "./taskstatus.js";
6
6
  import { TestCaseResult$zodSchema } from "./testcaseresult.js";
7
7
  import { TestRunEvaluator$zodSchema, } from "./testrunevaluator.js";
8
8
  export const TestRunStatusResponse$zodSchema = z.object({
9
+ aborted: z.boolean().default(false).describe("Whether a user stopped this run before it finished. The results collected up to that point are kept, and test cases that never ran are counted neither as passed nor as failed"),
9
10
  cost: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated cost as `{mean, min, max, count}` (USD)"),
10
11
  error: z.boolean().default(false).describe("True if the run failed"),
11
12
  evaluators: z.array(TestRunEvaluator$zodSchema).nullable().optional()
@@ -17,9 +18,11 @@ export const TestRunStatusResponse$zodSchema = z.object({
17
18
  results: z.array(TestCaseResult$zodSchema).nullable().optional().describe("Results for each test case"),
18
19
  share_token: z.string().nullable().optional().describe("Token for building the public share URL"),
19
20
  status: TaskStatus$zodSchema,
21
+ stopped_early: z.boolean().default(false).describe("Whether the run stopped before starting every test case, after too many failed in a row"),
20
22
  task_id: z.string().describe("Test run job ID"),
21
23
  test_uuids: z.array(z.string()).nullable().optional().describe("IDs of the tests this run executed, in run order"),
22
24
  total_tests: z.int().nullable().optional().describe("Total number of test cases"),
23
25
  total_tokens: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated token usage as `{mean, min, max, count}`"),
26
+ unanswered_tests: z.int().nullable().optional().describe("Number of test cases that produced no answer because the agent or the judge could not be reached, which makes the pass rate an unfair measure of the agent"),
24
27
  });
25
28
  //# sourceMappingURL=testrunstatusresponse.js.map
package/esm/models/testrunstatusresponse.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testrunstatusresponse.js","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAmB/B,MAAM,CAAC,MAAM,+BAA+B,GAC1C,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,0EAA0E,CAC3E;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,4BAA4B,CAC7B;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,kDAAkD,CACnD;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"testrunstatusresponse.js","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAsB/B,MAAM,CAAC,MAAM,+BAA+B,GAC1C,CAAC,CAAC,MAAM,CAAC;IACP,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC1C,gLAAgL,CACjL;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,0EAA0E,CAC3E;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,4BAA4B,CAC7B;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,yFAAyF,CAC1F;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,kDAAkD,CACnD;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;CACF,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dalmia/calibrate-mcp",
3
- "version": "0.0.47",
3
+ "version": "0.0.49",
4
4
  "author": "Speakeasy",
5
5
  "type": "module",
6
6
  "sideEffects": false,
package/src/landing-page.ts CHANGED
@@ -930,7 +930,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
930
930
  <h1>Instructions</h1>
931
931
  <p>One-click installation for Claude Desktop users</p>
932
932
  <div class="instruction-item">
933
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.47/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
933
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.49/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
934
934
  📥 Download MCP Bundle
935
935
  </a>
936
936
  </div>
package/src/lib/config.ts CHANGED
@@ -65,8 +65,8 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
65
65
  export const SDK_METADATA = {
66
66
  language: "typescript",
67
67
  openapiDocVersion: "0.1.0",
68
- sdkVersion: "0.0.47",
68
+ sdkVersion: "0.0.49",
69
69
  genVersion: "2.915.1",
70
70
  userAgent:
71
- "speakeasy-sdk/mcp-typescript 0.0.47 2.915.1 0.1.0 @dalmia/calibrate-mcp",
71
+ "speakeasy-sdk/mcp-typescript 0.0.49 2.915.1 0.1.0 @dalmia/calibrate-mcp",
72
72
  } as const;
package/src/mcp-server/mcp-server.ts CHANGED
@@ -21,7 +21,7 @@ const routes = buildRouteMap({
21
21
  export const app = buildApplication(routes, {
22
22
  name: "mcp",
23
23
  versionInfo: {
24
- currentVersion: "0.0.47",
24
+ currentVersion: "0.0.49",
25
25
  },
26
26
  });
27
27
 
package/src/mcp-server/server.ts CHANGED
@@ -72,7 +72,7 @@ export function createMCPServer(deps: {
72
72
  }) {
73
73
  const server = new McpServer({
74
74
  name: "CalibrateMcp",
75
- version: "0.0.47",
75
+ version: "0.0.49",
76
76
  });
77
77
 
78
78
  const getClient = deps.getSDK || (() =>
package/src/models/agenttestrunlistitem.ts CHANGED
@@ -63,6 +63,8 @@ export type AgentTestRunListItem = {
63
63
  cost?: { [k: string]: any } | null | undefined;
64
64
  total_tokens?: { [k: string]: any } | null | undefined;
65
65
  model_results?: Array<ModelRunSummary> | null | undefined;
66
+ unanswered_tests?: number | null | undefined;
67
+ aborted?: boolean | undefined;
66
68
  error?: boolean | undefined;
67
69
  is_public?: boolean | undefined;
68
70
  share_token?: string | null | undefined;
@@ -70,6 +72,9 @@ export type AgentTestRunListItem = {
70
72
 
71
73
  export const AgentTestRunListItem$zodSchema: z.ZodType<AgentTestRunListItem> = z
72
74
  .object({
75
+ aborted: z.boolean().default(false).describe(
76
+ "Whether a user stopped this run before it finished. The results collected up to that point are kept, and test cases that never ran are counted neither as passed nor as failed",
77
+ ),
73
78
  cost: z.record(z.string(), z.any()).nullable().optional().describe(
74
79
  "Aggregated cost as `{mean, min, max, count}` (USD)",
75
80
  ),
@@ -114,6 +119,9 @@ export const AgentTestRunListItem$zodSchema: z.ZodType<AgentTestRunListItem> = z
114
119
  type: AgentTestRunListItemType$zodSchema.describe(
115
120
  "What kind of run this is:\n- `llm-unit-test`: a single run of the agent's tests\n- `llm-benchmark`: a multi-model comparison",
116
121
  ),
122
+ unanswered_tests: z.int().nullable().optional().describe(
123
+ "Number of test cases that produced no answer because the agent or the judge could not be reached, which makes the pass rate an unfair measure of the agent",
124
+ ),
117
125
  updated_at: z.string().describe(
118
126
  "When the run was last updated (ISO 8601 UTC)",
119
127
  ),
package/src/models/benchmarkstatusresponse.ts CHANGED
@@ -17,6 +17,7 @@ export type BenchmarkStatusResponse = {
17
17
  evaluators?: Array<TestRunEvaluator> | null | undefined;
18
18
  model_results?: Array<ModelResult> | null | undefined;
19
19
  leaderboard_summary?: Array<{ [k: string]: any }> | null | undefined;
20
+ aborted?: boolean | undefined;
20
21
  error?: boolean | undefined;
21
22
  is_public?: boolean | undefined;
22
23
  share_token?: string | null | undefined;
@@ -25,6 +26,9 @@ export type BenchmarkStatusResponse = {
25
26
  export const BenchmarkStatusResponse$zodSchema: z.ZodType<
26
27
  BenchmarkStatusResponse
27
28
  > = z.object({
29
+ aborted: z.boolean().default(false).describe(
30
+ "Whether a user stopped this run before it finished. The results collected up to that point are kept, and test cases that never ran are counted neither as passed nor as failed",
31
+ ),
28
32
  error: z.boolean().default(false).describe("True if the run failed"),
29
33
  evaluators: z.array(TestRunEvaluator$zodSchema).nullable().optional()
30
34
  .describe(
package/src/models/testcaseresult.ts CHANGED
@@ -17,6 +17,8 @@ export type TestCaseResult = {
17
17
  judge_results?: Array<JudgeResult> | null | undefined;
18
18
  latency_ms?: number | null | undefined;
19
19
  cost?: number | null | undefined;
20
+ unanswered?: boolean | undefined;
21
+ not_run?: boolean | undefined;
20
22
  };
21
23
 
22
24
  export const TestCaseResult$zodSchema: z.ZodType<TestCaseResult> = z.object({
@@ -31,6 +33,9 @@ export const TestCaseResult$zodSchema: z.ZodType<TestCaseResult> = z.object({
31
33
  "How long the agent took to respond, in milliseconds",
32
34
  ),
33
35
  name: z.string().nullable().optional().describe("Name of the test"),
36
+ not_run: z.boolean().default(false).describe(
37
+ "Whether this case never started, because a user stopped the run first. It is counted neither as passed nor as failed",
38
+ ),
34
39
  output: TestOutput$zodSchema.nullable().optional().describe(
35
40
  "The agent's output for this case",
36
41
  ),
@@ -44,4 +49,7 @@ export const TestCaseResult$zodSchema: z.ZodType<TestCaseResult> = z.object({
44
49
  test_case_id: z.string().nullable().optional().describe(
45
50
  "ID of the test case within the run",
46
51
  ),
52
+ unanswered: z.boolean().default(false).describe(
53
+ "Whether this case produced no answer because the agent or the judge could not be reached, in which case `reasoning` carries the error and `passed` is not a verdict on the agent",
54
+ ),
47
55
  });
package/src/models/testruncasesummary.ts CHANGED
@@ -22,7 +22,7 @@ export const TestRunCaseSummary$zodSchema: z.ZodType<TestRunCaseSummary> = z
22
22
  .object({
23
23
  name: z.string().nullable().optional().describe("Name of the test case"),
24
24
  passed: z.boolean().nullable().optional().describe(
25
- "Whether the case passed (null if it errored or is still running)",
25
+ "Whether the case passed (null while the case is still running)",
26
26
  ),
27
27
  }).describe(
28
28
  "Flat summary for one test case in the run-LIST endpoints. Carries only\nenough to render a run's pass/fail breakdown and a case name. The full detail\nfor each case (agent output, judge verdicts, reasoning, latency, cost, the\ntest-case definition) lives on the run-DETAIL endpoint\n(`GET /agent-tests/run/{task_id}`).",
package/src/models/testrunstatusresponse.ts CHANGED
@@ -22,6 +22,9 @@ export type TestRunStatusResponse = {
22
22
  total_tokens?: { [k: string]: any } | null | undefined;
23
23
  evaluators?: Array<TestRunEvaluator> | null | undefined;
24
24
  results?: Array<TestCaseResult> | null | undefined;
25
+ unanswered_tests?: number | null | undefined;
26
+ stopped_early?: boolean | undefined;
27
+ aborted?: boolean | undefined;
25
28
  error?: boolean | undefined;
26
29
  is_public?: boolean | undefined;
27
30
  share_token?: string | null | undefined;
@@ -29,6 +32,9 @@ export type TestRunStatusResponse = {
29
32
 
30
33
  export const TestRunStatusResponse$zodSchema: z.ZodType<TestRunStatusResponse> =
31
34
  z.object({
35
+ aborted: z.boolean().default(false).describe(
36
+ "Whether a user stopped this run before it finished. The results collected up to that point are kept, and test cases that never ran are counted neither as passed nor as failed",
37
+ ),
32
38
  cost: z.record(z.string(), z.any()).nullable().optional().describe(
33
39
  "Aggregated cost as `{mean, min, max, count}` (USD)",
34
40
  ),
@@ -56,6 +62,9 @@ export const TestRunStatusResponse$zodSchema: z.ZodType<TestRunStatusResponse> =
56
62
  "Token for building the public share URL",
57
63
  ),
58
64
  status: TaskStatus$zodSchema,
65
+ stopped_early: z.boolean().default(false).describe(
66
+ "Whether the run stopped before starting every test case, after too many failed in a row",
67
+ ),
59
68
  task_id: z.string().describe("Test run job ID"),
60
69
  test_uuids: z.array(z.string()).nullable().optional().describe(
61
70
  "IDs of the tests this run executed, in run order",
@@ -66,4 +75,7 @@ export const TestRunStatusResponse$zodSchema: z.ZodType<TestRunStatusResponse> =
66
75
  total_tokens: z.record(z.string(), z.any()).nullable().optional().describe(
67
76
  "Aggregated token usage as `{mean, min, max, count}`",
68
77
  ),
78
+ unanswered_tests: z.int().nullable().optional().describe(
79
+ "Number of test cases that produced no answer because the agent or the judge could not be reached, which makes the pass rate an unfair measure of the agent",
80
+ ),
69
81
  });