@dalmia/calibrate-mcp 0.0.57 → 0.0.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/bin/mcp-server.js +113 -149
  2. package/bin/mcp-server.js.map +29 -29
  3. package/esm/landing-page.js +1 -1
  4. package/esm/lib/config.d.ts +2 -2
  5. package/esm/lib/config.js +2 -2
  6. package/esm/mcp-server/mcp-server.js +1 -1
  7. package/esm/mcp-server/server.js +1 -1
  8. package/esm/models/agenttestrunlistitem.d.ts +1 -0
  9. package/esm/models/agenttestrunlistitem.d.ts.map +1 -1
  10. package/esm/models/agenttestrunlistitem.js +1 -0
  11. package/esm/models/agenttestrunlistitem.js.map +1 -1
  12. package/esm/models/benchmarkrequest.d.ts +1 -0
  13. package/esm/models/benchmarkrequest.d.ts.map +1 -1
  14. package/esm/models/benchmarkrequest.js +1 -0
  15. package/esm/models/benchmarkrequest.js.map +1 -1
  16. package/esm/models/benchmarkstatusresponse.d.ts +2 -1
  17. package/esm/models/benchmarkstatusresponse.d.ts.map +1 -1
  18. package/esm/models/benchmarkstatusresponse.js +2 -1
  19. package/esm/models/benchmarkstatusresponse.js.map +1 -1
  20. package/esm/models/modelresult.d.ts +2 -0
  21. package/esm/models/modelresult.d.ts.map +1 -1
  22. package/esm/models/modelresult.js +2 -0
  23. package/esm/models/modelresult.js.map +1 -1
  24. package/esm/models/testrunstatusresponse.d.ts +1 -1
  25. package/esm/models/testrunstatusresponse.d.ts.map +1 -1
  26. package/esm/models/testrunstatusresponse.js +1 -1
  27. package/esm/models/testrunstatusresponse.js.map +1 -1
  28. package/package.json +1 -1
  29. package/src/landing-page.ts +1 -1
  30. package/src/lib/config.ts +2 -2
  31. package/src/mcp-server/mcp-server.ts +1 -1
  32. package/src/mcp-server/server.ts +1 -1
  33. package/src/models/agenttestrunlistitem.ts +4 -0
  34. package/src/models/benchmarkrequest.ts +4 -0
  35. package/src/models/benchmarkstatusresponse.ts +8 -2
  36. package/src/models/modelresult.ts +8 -0
  37. package/src/models/testrunstatusresponse.ts +4 -2
package/esm/landing-page.js CHANGED
@@ -915,7 +915,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
915
915
  <h1>Instructions</h1>
916
916
  <p>One-click installation for Claude Desktop users</p>
917
917
  <div class="instruction-item">
918
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.57/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
918
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.59/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
919
919
  📥 Download MCP Bundle
920
920
  </a>
921
921
  </div>
package/esm/lib/config.d.ts CHANGED
@@ -35,8 +35,8 @@ export declare function serverURLFromOptions(options: SDKOptions): URL | null;
35
35
  export declare const SDK_METADATA: {
36
36
  readonly language: "typescript";
37
37
  readonly openapiDocVersion: "0.1.0";
38
- readonly sdkVersion: "0.0.57";
38
+ readonly sdkVersion: "0.0.59";
39
39
  readonly genVersion: "2.915.1";
40
- readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.57 2.915.1 0.1.0 @dalmia/calibrate-mcp";
40
+ readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.59 2.915.1 0.1.0 @dalmia/calibrate-mcp";
41
41
  };
42
42
  //# sourceMappingURL=config.d.ts.map
package/esm/lib/config.js CHANGED
@@ -27,8 +27,8 @@ export function serverURLFromOptions(options) {
27
27
  export const SDK_METADATA = {
28
28
  language: "typescript",
29
29
  openapiDocVersion: "0.1.0",
30
- sdkVersion: "0.0.57",
30
+ sdkVersion: "0.0.59",
31
31
  genVersion: "2.915.1",
32
- userAgent: "speakeasy-sdk/mcp-typescript 0.0.57 2.915.1 0.1.0 @dalmia/calibrate-mcp",
32
+ userAgent: "speakeasy-sdk/mcp-typescript 0.0.59 2.915.1 0.1.0 @dalmia/calibrate-mcp",
33
33
  };
34
34
  //# sourceMappingURL=config.js.map
package/esm/mcp-server/mcp-server.js CHANGED
@@ -18,7 +18,7 @@ const routes = buildRouteMap({
18
18
  export const app = buildApplication(routes, {
19
19
  name: "mcp",
20
20
  versionInfo: {
21
- currentVersion: "0.0.57",
21
+ currentVersion: "0.0.59",
22
22
  },
23
23
  });
24
24
  run(app, process.argv.slice(2), buildContext(process));
package/esm/mcp-server/server.js CHANGED
@@ -51,7 +51,7 @@ import { tool$tracesCreate } from "./tools/tracesCreate.js";
51
51
  export function createMCPServer(deps) {
52
52
  const server = new McpServer({
53
53
  name: "CalibrateMcp",
54
- version: "0.0.57",
54
+ version: "0.0.59",
55
55
  });
56
56
  const getClient = deps.getSDK || (() => new CalibrateMcpCore({
57
57
  security: deps.security,
package/esm/models/agenttestrunlistitem.d.ts CHANGED
@@ -50,6 +50,7 @@ export type AgentTestRunListItem = {
50
50
  } | null | undefined;
51
51
  model_results?: Array<ModelRunSummary> | null | undefined;
52
52
  unanswered_tests?: number | null | undefined;
53
+ stopped_early?: boolean | undefined;
53
54
  aborted?: boolean | undefined;
54
55
  error?: boolean | undefined;
55
56
  is_public?: boolean | undefined;
package/esm/models/agenttestrunlistitem.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agenttestrunlistitem.d.ts","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,eAAe,EAEhB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EACL,kBAAkB,EAEnB,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB;;;CAG3B,CAAC;AACX;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,UAAU,CAC/C,OAAO,wBAAwB,CAChC,CAAC;AAEF,eAAO,MAAM,kCAAkC;;;EAK9C,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,wBAAwB,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,SAAS,CAAC;IACjD,OAAO,CAAC,EAAE,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,aAAa,CAAC,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC1D,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7C,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,8BAA8B,EAAE,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAwDvE,CAAC"}
1
+ {"version":3,"file":"agenttestrunlistitem.d.ts","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,eAAe,EAEhB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EACL,kBAAkB,EAEnB,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB;;;CAG3B,CAAC;AACX;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,UAAU,CAC/C,OAAO,wBAAwB,CAChC,CAAC;AAEF,eAAO,MAAM,kCAAkC;;;EAK9C,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,wBAAwB,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,SAAS,CAAC;IACjD,OAAO,CAAC,EAAE,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,aAAa,CAAC,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC1D,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7C,aAAa,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACpC,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,8BAA8B,EAAE,CAAC,CAAC,OAAO,CAAC,oBAAoB,CA2DvE,CAAC"}
package/esm/models/agenttestrunlistitem.js CHANGED
@@ -39,6 +39,7 @@ export const AgentTestRunListItem$zodSchema = z
39
39
  .describe("Flat pass/fail summary for each test case (fetch the run detail for full results)"),
40
40
  share_token: z.string().nullable().optional().describe("Token for building the public share URL"),
41
41
  status: TaskStatus$zodSchema,
42
+ stopped_early: z.boolean().default(false).describe("Whether the run stopped before starting every test case, after too many failed in a row"),
42
43
  total_tests: z.int().nullable().optional().describe("Total number of test cases"),
43
44
  total_tokens: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated token usage as `{mean, min, max, count}`"),
44
45
  type: AgentTestRunListItemType$zodSchema.describe("What kind of run this is:\n- `llm-unit-test`: a single run of the agent's tests\n- `llm-benchmark`: a multi-model comparison"),
package/esm/models/agenttestrunlistitem.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agenttestrunlistitem.js","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,4BAA4B,GAC7B,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,WAAW,EAAE,eAAe;IAC5B,YAAY,EAAE,eAAe;CACrB,CAAC;AAYX,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,CAAC,IAAI,CAAC;IACvD,eAAe;IACf,eAAe;CAChB,CAAC,CAAC,QAAQ,CACT,8HAA8H,CAC/H,CAAC;AAyBF,MAAM,CAAC,MAAM,8BAA8B,GAAoC,CAAC;KAC7E,MAAM,CAAC;IACN,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC1C,gLAAgL,CACjL;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IAC1E,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjE,sQAAsQ,CACvQ;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,iEAAiE,CAClE;IACD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACpE,QAAQ,CACP,8FAA8F,CAC/F;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,qIAAqI,CACtI;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACjE,QAAQ,CACP,mFAAmF,CACpF;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,IAAI,EAAE,kCAAkC,CAAC,QAAQ,CAC/C,8HAA8H,CAC/H;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,8CAA8C,CAC/C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;CAC7C,CAAC,CAAC"}
1
+ {"version":3,"file":"agenttestrunlistitem.js","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,4BAA4B,GAC7B,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,WAAW,EAAE,eAAe;IAC5B,YAAY,EAAE,eAAe;CACrB,CAAC;AAYX,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,CAAC,IAAI,CAAC;IACvD,eAAe;IACf,eAAe;CAChB,CAAC,CAAC,QAAQ,CACT,8HAA8H,CAC/H,CAAC;AA0BF,MAAM,CAAC,MAAM,8BAA8B,GAAoC,CAAC;KAC7E,MAAM,CAAC;IACN,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC1C,gLAAgL,CACjL;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IAC1E,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjE,sQAAsQ,CACvQ;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,iEAAiE,CAClE;IACD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACpE,QAAQ,CACP,8FAA8F,CAC/F;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,qIAAqI,CACtI;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACjE,QAAQ,CACP,mFAAmF,CACpF;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,yFAAyF,CAC1F;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,IAAI,EAAE,kCAAkC,CAAC,QAAQ,CAC/C,8HAA8H,CAC/H;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,8CAA8C,CAC/C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;CAC7C,CAAC,CAAC"}
package/esm/models/benchmarkrequest.d.ts CHANGED
@@ -2,6 +2,7 @@ import * as z from "zod";
2
2
  export type BenchmarkRequest = {
3
3
  models: Array<string>;
4
4
  test_uuids?: Array<string> | null | undefined;
5
+ parallel_models?: boolean | undefined;
5
6
  };
6
7
  export declare const BenchmarkRequest$zodSchema: z.ZodType<BenchmarkRequest>;
7
8
  //# sourceMappingURL=benchmarkrequest.d.ts.map
package/esm/models/benchmarkrequest.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"benchmarkrequest.d.ts","sourceRoot":"","sources":["../../src/models/benchmarkrequest.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACtB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;CAC/C,CAAC;AAEF,eAAO,MAAM,0BAA0B,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAOlE,CAAC"}
1
+ {"version":3,"file":"benchmarkrequest.d.ts","sourceRoot":"","sources":["../../src/models/benchmarkrequest.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACtB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,eAAe,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACvC,CAAC;AAEF,eAAO,MAAM,0BAA0B,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAUlE,CAAC"}
package/esm/models/benchmarkrequest.js CHANGED
@@ -4,6 +4,7 @@
4
4
  import * as z from "zod";
5
5
  export const BenchmarkRequest$zodSchema = z.object({
6
6
  models: z.array(z.string()).describe("Model names to benchmark"),
7
+ parallel_models: z.boolean().default(true).describe("Whether to run the models at the same time. Set false to run them one after another"),
7
8
  test_uuids: z.array(z.string()).nullable().optional().describe("A subset of the agent's linked tests to benchmark. Each ID must be linked to the agent. Omit to run all linked tests"),
8
9
  });
9
10
  //# sourceMappingURL=benchmarkrequest.js.map
package/esm/models/benchmarkrequest.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"benchmarkrequest.js","sourceRoot":"","sources":["../../src/models/benchmarkrequest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAOzB,MAAM,CAAC,MAAM,0BAA0B,GAAgC,CAAC,CAAC,MAAM,CAC7E;IACE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,0BAA0B,CAAC;IAChE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,sHAAsH,CACvH;CACF,CACF,CAAC"}
1
+ {"version":3,"file":"benchmarkrequest.js","sourceRoot":"","sources":["../../src/models/benchmarkrequest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAQzB,MAAM,CAAC,MAAM,0BAA0B,GAAgC,CAAC,CAAC,MAAM,CAC7E;IACE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,0BAA0B,CAAC;IAChE,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CACjD,qFAAqF,CACtF;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,sHAAsH,CACvH;CACF,CACF,CAAC"}
package/esm/models/benchmarkstatusresponse.d.ts CHANGED
@@ -12,8 +12,9 @@ export type BenchmarkStatusResponse = {
12
12
  leaderboard_summary?: Array<{
13
13
  [k: string]: any;
14
14
  }> | null | undefined;
15
+ stopped_early?: boolean | undefined;
15
16
  aborted?: boolean | undefined;
16
- error?: boolean | undefined;
17
+ error?: string | null | undefined;
17
18
  is_public?: boolean | undefined;
18
19
  share_token?: string | null | undefined;
19
20
  };
package/esm/models/benchmarkstatusresponse.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"benchmarkstatusresponse.d.ts","sourceRoot":"","sources":["../../src/models/benchmarkstatusresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,uBAAuB,GAAG;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,aAAa,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,mBAAmB,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACrE,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,iCAAiC,EAAE,CAAC,CAAC,OAAO,CACvD,uBAAuB,CA+BvB,CAAC"}
1
+ {"version":3,"file":"benchmarkstatusresponse.d.ts","sourceRoot":"","sources":["../../src/models/benchmarkstatusresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,uBAAuB,GAAG;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,aAAa,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,mBAAmB,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACrE,aAAa,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACpC,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAClC,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,iCAAiC,EAAE,CAAC,CAAC,OAAO,CACvD,uBAAuB,CAoCvB,CAAC"}
package/esm/models/benchmarkstatusresponse.js CHANGED
@@ -7,7 +7,7 @@ import { TaskStatus$zodSchema } from "./taskstatus.js";
7
7
  import { TestRunEvaluator$zodSchema, } from "./testrunevaluator.js";
8
8
  export const BenchmarkStatusResponse$zodSchema = z.object({
9
9
  aborted: z.boolean().default(false).describe("Whether a user stopped this run before it finished. The results collected up to that point are kept, and test cases that never ran are counted neither as passed nor as failed"),
10
- error: z.boolean().default(false).describe("True if the run failed"),
10
+ error: z.string().nullable().optional().describe("Why the run could not be carried out, when it failed before producing any result"),
11
11
  evaluators: z.array(TestRunEvaluator$zodSchema).nullable().optional()
12
12
  .describe("The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`"),
13
13
  is_public: z.boolean().default(false).describe("Whether the run is shared publicly"),
@@ -17,6 +17,7 @@ export const BenchmarkStatusResponse$zodSchema = z.object({
17
17
  name: z.string().describe("Name of the run. A run nobody has renamed shows its number instead, such as `Run 1` for a test run or `Benchmark 1` for a benchmark"),
18
18
  share_token: z.string().nullable().optional().describe("Token for building the public share URL"),
19
19
  status: TaskStatus$zodSchema,
20
+ stopped_early: z.boolean().default(false).describe("Whether any model's run stopped before starting every test case, after too many failed in a row"),
20
21
  task_id: z.string().describe("Benchmark run job ID"),
21
22
  test_uuids: z.array(z.string()).nullable().optional().describe("IDs of the tests this benchmark executed, in run order"),
22
23
  });
package/esm/models/benchmarkstatusresponse.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"benchmarkstatusresponse.js","sourceRoot":"","sources":["../../src/models/benchmarkstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAgB/B,MAAM,CAAC,MAAM,iCAAiC,GAE1C,CAAC,CAAC,MAAM,CAAC;IACX,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC1C,gLAAgL,CACjL;IACD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,mBAAmB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;SACnE,QAAQ,EAAE,CAAC,QAAQ,CAClB,oMAAoM,CACrM;IACH,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC1E,wBAAwB,CACzB;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,qIAAqI,CACtI;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;IACpD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,wDAAwD,CACzD;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"benchmarkstatusresponse.js","sourceRoot":"","sources":["../../src/models/benchmarkstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAiB/B,MAAM,CAAC,MAAM,iCAAiC,GAE1C,CAAC,CAAC,MAAM,CAAC;IACX,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC1C,gLAAgL,CACjL;IACD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC9C,kFAAkF,CACnF;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,mBAAmB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;SACnE,QAAQ,EAAE,CAAC,QAAQ,CAClB,oMAAoM,CACrM;IACH,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC1E,wBAAwB,CACzB;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,qIAAqI,CACtI;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,iGAAiG,CAClG;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;IACpD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,wDAAwD,CACzD;CACF,CAAC,CAAC"}
package/esm/models/modelresult.d.ts CHANGED
@@ -20,6 +20,8 @@ export type ModelResult = {
20
20
  total_tokens?: {
21
21
  [k: string]: any;
22
22
  } | null | undefined;
23
+ unanswered_tests?: number | null | undefined;
24
+ stopped_early?: boolean | undefined;
23
25
  };
24
26
  export declare const ModelResult$zodSchema: z.ZodType<ModelResult>;
25
27
  //# sourceMappingURL=modelresult.d.ts.map
package/esm/models/modelresult.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"modelresult.d.ts","sourceRoot":"","sources":["../../src/models/modelresult.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,cAAc,EAA4B,MAAM,qBAAqB,CAAC;AAE/E,MAAM,MAAM,WAAW,GAAG;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,iBAAiB,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnE,YAAY,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;CACxD,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CA4BvD,CAAC"}
1
+ {"version":3,"file":"modelresult.d.ts","sourceRoot":"","sources":["../../src/models/modelresult.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,cAAc,EAA4B,MAAM,qBAAqB,CAAC;AAE/E,MAAM,MAAM,WAAW,GAAG;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,iBAAiB,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnE,YAAY,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7C,aAAa,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACrC,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAkCvD,CAAC"}
package/esm/models/modelresult.js CHANGED
@@ -12,10 +12,12 @@ export const ModelResult$zodSchema = z.object({
12
12
  message: z.string().describe("Status or result message for this model"),
13
13
  model: z.string().describe("Model name these results are for"),
14
14
  passed: z.int().nullable().optional().describe("Number of test cases that passed"),
15
+ stopped_early: z.boolean().default(false).describe("Whether this model's run stopped before starting every test case, after too many failed in a row"),
15
16
  success: z.boolean().nullable().optional().describe("Whether this model's run succeeded"),
16
17
  test_results: z.array(TestCaseResult$zodSchema).nullable().optional()
17
18
  .describe("Results for each test case for this model"),
18
19
  total_tests: z.int().nullable().optional().describe("Total test cases for this model"),
19
20
  total_tokens: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated token usage as `{mean, min, max, count}`"),
21
+ unanswered_tests: z.int().nullable().optional().describe("Number of test cases that produced no answer because the agent or the judge could not be reached, which makes the pass rate an unfair measure of the agent"),
20
22
  });
21
23
  //# sourceMappingURL=modelresult.js.map
package/esm/models/modelresult.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"modelresult.js","sourceRoot":"","sources":["../../src/models/modelresult.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAgB/E,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;SACjE,QAAQ,EAAE,CAAC,QAAQ,CAAC,qDAAqD,CAAC;IAC7E,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,iEAAiE,CAClE;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IACvE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;IAC9D,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,oCAAoC,CACrC;IACD,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CAAC,2CAA2C,CAAC;IACxD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,iCAAiC,CAClC;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"modelresult.js","sourceRoot":"","sources":["../../src/models/modelresult.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAkB/E,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;SACjE,QAAQ,EAAE,CAAC,QAAQ,CAAC,qDAAqD,CAAC;IAC7E,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,iEAAiE,CAClE;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IACvE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;IAC9D,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,kGAAkG,CACnG;IACD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,oCAAoC,CACrC;IACD,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CAAC,2CAA2C,CAAC;IACxD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,iCAAiC,CAClC;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;CACF,CAAC,CAAC"}
package/esm/models/testrunstatusresponse.d.ts CHANGED
@@ -27,7 +27,7 @@ export type TestRunStatusResponse = {
27
27
  unanswered_tests?: number | null | undefined;
28
28
  stopped_early?: boolean | undefined;
29
29
  aborted?: boolean | undefined;
30
- error?: boolean | undefined;
30
+ error?: string | null | undefined;
31
31
  is_public?: boolean | undefined;
32
32
  share_token?: string | null | undefined;
33
33
  };
package/esm/models/testrunstatusresponse.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testrunstatusresponse.d.ts","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,cAAc,EAA4B,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,iBAAiB,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnE,OAAO,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnD,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7C,aAAa,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACpC,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAuDzE,CAAC"}
1
+ {"version":3,"file":"testrunstatusresponse.d.ts","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,cAAc,EAA4B,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,iBAAiB,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnE,OAAO,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnD,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7C,aAAa,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACpC,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAClC,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAyDzE,CAAC"}
package/esm/models/testrunstatusresponse.js CHANGED
@@ -8,7 +8,7 @@ import { TestRunEvaluator$zodSchema, } from "./testrunevaluator.js";
8
8
  export const TestRunStatusResponse$zodSchema = z.object({
9
9
  aborted: z.boolean().default(false).describe("Whether a user stopped this run before it finished. The results collected up to that point are kept, and test cases that never ran are counted neither as passed nor as failed"),
10
10
  cost: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated cost as `{mean, min, max, count}` (USD)"),
11
- error: z.boolean().default(false).describe("True if the run failed"),
11
+ error: z.string().nullable().optional().describe("Why the run could not be carried out, when it failed before producing any result"),
12
12
  evaluator_summary: z.array(z.record(z.string(), z.any())).nullable()
13
13
  .optional().describe("Totals for each evaluator over the whole run, matching the shape a benchmark reports for each model. Only evaluators that returned a verdict appear"),
14
14
  evaluators: z.array(TestRunEvaluator$zodSchema).nullable().optional()
package/esm/models/testrunstatusresponse.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testrunstatusresponse.js","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAwB/B,MAAM,CAAC,MAAM,+BAA+B,GAC1C,CAAC,CAAC,MAAM,CAAC;IACP,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC1C,gLAAgL,CACjL;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;SACjE,QAAQ,EAAE,CAAC,QAAQ,CAClB,qJAAqJ,CACtJ;IACH,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,0EAA0E,CAC3E;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,qIAAqI,CACtI;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,4BAA4B,CAC7B;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,yFAAyF,CAC1F;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,kDAAkD,CACnD;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"testrunstatusresponse.js","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAwB/B,MAAM,CAAC,MAAM,+BAA+B,GAC1C,CAAC,CAAC,MAAM,CAAC;IACP,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC1C,gLAAgL,CACjL;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC9C,kFAAkF,CACnF;IACD,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;SACjE,QAAQ,EAAE,CAAC,QAAQ,CAClB,qJAAqJ,CACtJ;IACH,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,0EAA0E,CAC3E;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,qIAAqI,CACtI;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,4BAA4B,CAC7B;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,yFAAyF,CAC1F;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,kDAAkD,CACnD;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;CACF,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dalmia/calibrate-mcp",
3
- "version": "0.0.57",
3
+ "version": "0.0.59",
4
4
  "author": "Speakeasy",
5
5
  "type": "module",
6
6
  "sideEffects": false,
package/src/landing-page.ts CHANGED
@@ -930,7 +930,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
930
930
  <h1>Instructions</h1>
931
931
  <p>One-click installation for Claude Desktop users</p>
932
932
  <div class="instruction-item">
933
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.57/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
933
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.59/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
934
934
  📥 Download MCP Bundle
935
935
  </a>
936
936
  </div>
package/src/lib/config.ts CHANGED
@@ -65,8 +65,8 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
65
65
  export const SDK_METADATA = {
66
66
  language: "typescript",
67
67
  openapiDocVersion: "0.1.0",
68
- sdkVersion: "0.0.57",
68
+ sdkVersion: "0.0.59",
69
69
  genVersion: "2.915.1",
70
70
  userAgent:
71
- "speakeasy-sdk/mcp-typescript 0.0.57 2.915.1 0.1.0 @dalmia/calibrate-mcp",
71
+ "speakeasy-sdk/mcp-typescript 0.0.59 2.915.1 0.1.0 @dalmia/calibrate-mcp",
72
72
  } as const;
package/src/mcp-server/mcp-server.ts CHANGED
@@ -21,7 +21,7 @@ const routes = buildRouteMap({
21
21
  export const app = buildApplication(routes, {
22
22
  name: "mcp",
23
23
  versionInfo: {
24
- currentVersion: "0.0.57",
24
+ currentVersion: "0.0.59",
25
25
  },
26
26
  });
27
27
 
package/src/mcp-server/server.ts CHANGED
@@ -73,7 +73,7 @@ export function createMCPServer(deps: {
73
73
  }) {
74
74
  const server = new McpServer({
75
75
  name: "CalibrateMcp",
76
- version: "0.0.57",
76
+ version: "0.0.59",
77
77
  });
78
78
 
79
79
  const getClient = deps.getSDK || (() =>
package/src/models/agenttestrunlistitem.ts CHANGED
@@ -64,6 +64,7 @@ export type AgentTestRunListItem = {
64
64
  total_tokens?: { [k: string]: any } | null | undefined;
65
65
  model_results?: Array<ModelRunSummary> | null | undefined;
66
66
  unanswered_tests?: number | null | undefined;
67
+ stopped_early?: boolean | undefined;
67
68
  aborted?: boolean | undefined;
68
69
  error?: boolean | undefined;
69
70
  is_public?: boolean | undefined;
@@ -110,6 +111,9 @@ export const AgentTestRunListItem$zodSchema: z.ZodType<AgentTestRunListItem> = z
110
111
  "Token for building the public share URL",
111
112
  ),
112
113
  status: TaskStatus$zodSchema,
114
+ stopped_early: z.boolean().default(false).describe(
115
+ "Whether the run stopped before starting every test case, after too many failed in a row",
116
+ ),
113
117
  total_tests: z.int().nullable().optional().describe(
114
118
  "Total number of test cases",
115
119
  ),
package/src/models/benchmarkrequest.ts CHANGED
@@ -7,11 +7,15 @@ import * as z from "zod";
7
7
  export type BenchmarkRequest = {
8
8
  models: Array<string>;
9
9
  test_uuids?: Array<string> | null | undefined;
10
+ parallel_models?: boolean | undefined;
10
11
  };
11
12
 
12
13
  export const BenchmarkRequest$zodSchema: z.ZodType<BenchmarkRequest> = z.object(
13
14
  {
14
15
  models: z.array(z.string()).describe("Model names to benchmark"),
16
+ parallel_models: z.boolean().default(true).describe(
17
+ "Whether to run the models at the same time. Set false to run them one after another",
18
+ ),
15
19
  test_uuids: z.array(z.string()).nullable().optional().describe(
16
20
  "A subset of the agent's linked tests to benchmark. Each ID must be linked to the agent. Omit to run all linked tests",
17
21
  ),
package/src/models/benchmarkstatusresponse.ts CHANGED
@@ -18,8 +18,9 @@ export type BenchmarkStatusResponse = {
18
18
  evaluators?: Array<TestRunEvaluator> | null | undefined;
19
19
  model_results?: Array<ModelResult> | null | undefined;
20
20
  leaderboard_summary?: Array<{ [k: string]: any }> | null | undefined;
21
+ stopped_early?: boolean | undefined;
21
22
  aborted?: boolean | undefined;
22
- error?: boolean | undefined;
23
+ error?: string | null | undefined;
23
24
  is_public?: boolean | undefined;
24
25
  share_token?: string | null | undefined;
25
26
  };
@@ -30,7 +31,9 @@ export const BenchmarkStatusResponse$zodSchema: z.ZodType<
30
31
  aborted: z.boolean().default(false).describe(
31
32
  "Whether a user stopped this run before it finished. The results collected up to that point are kept, and test cases that never ran are counted neither as passed nor as failed",
32
33
  ),
33
- error: z.boolean().default(false).describe("True if the run failed"),
34
+ error: z.string().nullable().optional().describe(
35
+ "Why the run could not be carried out, when it failed before producing any result",
36
+ ),
34
37
  evaluators: z.array(TestRunEvaluator$zodSchema).nullable().optional()
35
38
  .describe(
36
39
  "The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`",
@@ -52,6 +55,9 @@ export const BenchmarkStatusResponse$zodSchema: z.ZodType<
52
55
  "Token for building the public share URL",
53
56
  ),
54
57
  status: TaskStatus$zodSchema,
58
+ stopped_early: z.boolean().default(false).describe(
59
+ "Whether any model's run stopped before starting every test case, after too many failed in a row",
60
+ ),
55
61
  task_id: z.string().describe("Benchmark run job ID"),
56
62
  test_uuids: z.array(z.string()).nullable().optional().describe(
57
63
  "IDs of the tests this benchmark executed, in run order",
package/src/models/modelresult.ts CHANGED
@@ -17,6 +17,8 @@ export type ModelResult = {
17
17
  latency_ms?: { [k: string]: any } | null | undefined;
18
18
  cost?: { [k: string]: any } | null | undefined;
19
19
  total_tokens?: { [k: string]: any } | null | undefined;
20
+ unanswered_tests?: number | null | undefined;
21
+ stopped_early?: boolean | undefined;
20
22
  };
21
23
 
22
24
  export const ModelResult$zodSchema: z.ZodType<ModelResult> = z.object({
@@ -36,6 +38,9 @@ export const ModelResult$zodSchema: z.ZodType<ModelResult> = z.object({
36
38
  passed: z.int().nullable().optional().describe(
37
39
  "Number of test cases that passed",
38
40
  ),
41
+ stopped_early: z.boolean().default(false).describe(
42
+ "Whether this model's run stopped before starting every test case, after too many failed in a row",
43
+ ),
39
44
  success: z.boolean().nullable().optional().describe(
40
45
  "Whether this model's run succeeded",
41
46
  ),
@@ -47,4 +52,7 @@ export const ModelResult$zodSchema: z.ZodType<ModelResult> = z.object({
47
52
  total_tokens: z.record(z.string(), z.any()).nullable().optional().describe(
48
53
  "Aggregated token usage as `{mean, min, max, count}`",
49
54
  ),
55
+ unanswered_tests: z.int().nullable().optional().describe(
56
+ "Number of test cases that produced no answer because the agent or the judge could not be reached, which makes the pass rate an unfair measure of the agent",
57
+ ),
50
58
  });
package/src/models/testrunstatusresponse.ts CHANGED
@@ -27,7 +27,7 @@ export type TestRunStatusResponse = {
27
27
  unanswered_tests?: number | null | undefined;
28
28
  stopped_early?: boolean | undefined;
29
29
  aborted?: boolean | undefined;
30
- error?: boolean | undefined;
30
+ error?: string | null | undefined;
31
31
  is_public?: boolean | undefined;
32
32
  share_token?: string | null | undefined;
33
33
  };
@@ -40,7 +40,9 @@ export const TestRunStatusResponse$zodSchema: z.ZodType<TestRunStatusResponse> =
40
40
  cost: z.record(z.string(), z.any()).nullable().optional().describe(
41
41
  "Aggregated cost as `{mean, min, max, count}` (USD)",
42
42
  ),
43
- error: z.boolean().default(false).describe("True if the run failed"),
43
+ error: z.string().nullable().optional().describe(
44
+ "Why the run could not be carried out, when it failed before producing any result",
45
+ ),
44
46
  evaluator_summary: z.array(z.record(z.string(), z.any())).nullable()
45
47
  .optional().describe(
46
48
  "Totals for each evaluator over the whole run, matching the shape a benchmark reports for each model. Only evaluators that returned a verdict appear",