@dalmia/calibrate-mcp 0.0.17 → 0.0.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. package/bin/mcp-server.js +38 -13
  2. package/bin/mcp-server.js.map +10 -8
  3. package/esm/landing-page.js +1 -1
  4. package/esm/lib/config.d.ts +2 -2
  5. package/esm/lib/config.js +2 -2
  6. package/esm/mcp-server/mcp-server.js +1 -1
  7. package/esm/mcp-server/server.js +1 -1
  8. package/esm/models/agenttestrunlistitem.d.ts +4 -9
  9. package/esm/models/agenttestrunlistitem.d.ts.map +1 -1
  10. package/esm/models/agenttestrunlistitem.js +6 -10
  11. package/esm/models/agenttestrunlistitem.js.map +1 -1
  12. package/esm/models/modelrunsummary.d.ts +18 -0
  13. package/esm/models/modelrunsummary.d.ts.map +1 -0
  14. package/esm/models/modelrunsummary.js +13 -0
  15. package/esm/models/modelrunsummary.js.map +1 -0
  16. package/esm/models/testruncasesummary.d.ts +16 -0
  17. package/esm/models/testruncasesummary.d.ts.map +1 -0
  18. package/esm/models/testruncasesummary.js +10 -0
  19. package/esm/models/testruncasesummary.js.map +1 -0
  20. package/package.json +1 -1
  21. package/src/landing-page.ts +1 -1
  22. package/src/lib/config.ts +2 -2
  23. package/src/mcp-server/mcp-server.ts +1 -1
  24. package/src/mcp-server/server.ts +1 -1
  25. package/src/models/agenttestrunlistitem.ts +16 -21
  26. package/src/models/modelrunsummary.ts +42 -0
  27. package/src/models/testruncasesummary.ts +29 -0
package/esm/landing-page.js CHANGED
@@ -915,7 +915,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
915
915
  <h1>Instructions</h1>
916
916
  <p>One-click installation for Claude Desktop users</p>
917
917
  <div class="instruction-item">
918
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.17/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
918
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.18/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
919
919
  📥 Download MCP Bundle
920
920
  </a>
921
921
  </div>
package/esm/lib/config.d.ts CHANGED
@@ -35,8 +35,8 @@ export declare function serverURLFromOptions(options: SDKOptions): URL | null;
35
35
  export declare const SDK_METADATA: {
36
36
  readonly language: "typescript";
37
37
  readonly openapiDocVersion: "0.1.0";
38
- readonly sdkVersion: "0.0.17";
38
+ readonly sdkVersion: "0.0.18";
39
39
  readonly genVersion: "2.915.1";
40
- readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.17 2.915.1 0.1.0 @dalmia/calibrate-mcp";
40
+ readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.18 2.915.1 0.1.0 @dalmia/calibrate-mcp";
41
41
  };
42
42
  //# sourceMappingURL=config.d.ts.map
package/esm/lib/config.js CHANGED
@@ -27,8 +27,8 @@ export function serverURLFromOptions(options) {
27
27
  export const SDK_METADATA = {
28
28
  language: "typescript",
29
29
  openapiDocVersion: "0.1.0",
30
- sdkVersion: "0.0.17",
30
+ sdkVersion: "0.0.18",
31
31
  genVersion: "2.915.1",
32
- userAgent: "speakeasy-sdk/mcp-typescript 0.0.17 2.915.1 0.1.0 @dalmia/calibrate-mcp",
32
+ userAgent: "speakeasy-sdk/mcp-typescript 0.0.18 2.915.1 0.1.0 @dalmia/calibrate-mcp",
33
33
  };
34
34
  //# sourceMappingURL=config.js.map
package/esm/mcp-server/mcp-server.js CHANGED
@@ -18,7 +18,7 @@ const routes = buildRouteMap({
18
18
  export const app = buildApplication(routes, {
19
19
  name: "mcp",
20
20
  versionInfo: {
21
- currentVersion: "0.0.17",
21
+ currentVersion: "0.0.18",
22
22
  },
23
23
  });
24
24
  run(app, process.argv.slice(2), buildContext(process));
package/esm/mcp-server/server.js CHANGED
@@ -42,7 +42,7 @@ import { tool$testsUpdate } from "./tools/testsUpdate.js";
42
42
  export function createMCPServer(deps) {
43
43
  const server = new McpServer({
44
44
  name: "CalibrateMcp",
45
- version: "0.0.17",
45
+ version: "0.0.18",
46
46
  });
47
47
  const getClient = deps.getSDK || (() => new CalibrateMcpCore({
48
48
  security: deps.security,
package/esm/models/agenttestrunlistitem.d.ts CHANGED
@@ -1,9 +1,8 @@
1
1
  import * as z from "zod";
2
2
  import { ClosedEnum } from "../types/enums.js";
3
- import { ModelResult } from "./modelresult.js";
3
+ import { ModelRunSummary } from "./modelrunsummary.js";
4
4
  import { TaskStatus } from "./taskstatus.js";
5
- import { TestCaseResult } from "./testcaseresult.js";
6
- import { TestRunEvaluator } from "./testrunevaluator.js";
5
+ import { TestRunCaseSummary } from "./testruncasesummary.js";
7
6
  /**
8
7
  * What kind of run this is:
9
8
  *
@@ -33,11 +32,10 @@ export type AgentTestRunListItem = {
33
32
  status: TaskStatus;
34
33
  type: AgentTestRunListItemType;
35
34
  updated_at: string;
36
- evaluators?: Array<TestRunEvaluator> | null | undefined;
37
35
  total_tests?: number | null | undefined;
38
36
  passed?: number | null | undefined;
39
37
  failed?: number | null | undefined;
40
- results?: Array<TestCaseResult> | null | undefined;
38
+ results?: Array<TestRunCaseSummary> | null | undefined;
41
39
  latency_ms?: {
42
40
  [k: string]: any;
43
41
  } | null | undefined;
@@ -47,10 +45,7 @@ export type AgentTestRunListItem = {
47
45
  total_tokens?: {
48
46
  [k: string]: any;
49
47
  } | null | undefined;
50
- model_results?: Array<ModelResult> | null | undefined;
51
- leaderboard_summary?: Array<{
52
- [k: string]: any;
53
- }> | null | undefined;
48
+ model_results?: Array<ModelRunSummary> | null | undefined;
54
49
  error?: boolean | undefined;
55
50
  is_public?: boolean | undefined;
56
51
  share_token?: string | null | undefined;
package/esm/models/agenttestrunlistitem.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agenttestrunlistitem.d.ts","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,cAAc,EAA4B,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB;;;CAG3B,CAAC;AACX;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,UAAU,CAC/C,OAAO,wBAAwB,CAChC,CAAC;AAEF,eAAO,MAAM,kCAAkC;;;EAK9C,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,wBAAwB,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,OAAO,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnD,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,aAAa,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,mBAAmB,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACrE,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,8BAA8B,EAAE,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAmDvE,CAAC"}
1
+ {"version":3,"file":"agenttestrunlistitem.d.ts","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,eAAe,EAEhB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EACL,kBAAkB,EAEnB,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB;;;CAG3B,CAAC;AACX;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,UAAU,CAC/C,OAAO,wBAAwB,CAChC,CAAC;AAEF,eAAO,MAAM,kCAAkC;;;EAK9C,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,wBAAwB,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,OAAO,CAAC,EAAE,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,aAAa,CAAC,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC1D,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,8BAA8B,EAAE,CAAC,CAAC,OAAO,CAAC,oBAAoB,CA8CvE,CAAC"}
package/esm/models/agenttestrunlistitem.js CHANGED
@@ -2,10 +2,9 @@
2
2
  * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
3
  */
4
4
  import * as z from "zod";
5
- import { ModelResult$zodSchema } from "./modelresult.js";
5
+ import { ModelRunSummary$zodSchema, } from "./modelrunsummary.js";
6
6
  import { TaskStatus$zodSchema } from "./taskstatus.js";
7
- import { TestCaseResult$zodSchema } from "./testcaseresult.js";
8
- import { TestRunEvaluator$zodSchema, } from "./testrunevaluator.js";
7
+ import { TestRunCaseSummary$zodSchema, } from "./testruncasesummary.js";
9
8
  /**
10
9
  * What kind of run this is:
11
10
  *
@@ -25,18 +24,15 @@ export const AgentTestRunListItem$zodSchema = z
25
24
  .object({
26
25
  cost: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated cost as `{mean, min, max, count}` (USD)"),
27
26
  error: z.boolean().default(false).describe("True if the run failed"),
28
- evaluators: z.array(TestRunEvaluator$zodSchema).nullable().optional()
29
- .describe("The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`"),
30
27
  failed: z.int().nullable().optional().describe("Number of test cases that failed"),
31
28
  is_public: z.boolean().default(false).describe("Whether the run is shared publicly"),
32
29
  latency_ms: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated latency in milliseconds, as `{p50, p95, p99, count}`"),
33
- leaderboard_summary: z.array(z.record(z.string(), z.any())).nullable()
34
- .optional().describe("Leaderboard comparing the models, one row per model. Columns vary by benchmark: a `model` column plus pass/fail counts, latency, cost, and one score column per evaluator, keyed by evaluator name"),
35
- model_results: z.array(ModelResult$zodSchema).nullable().optional()
36
- .describe("Results for each model in a benchmark run"),
30
+ model_results: z.array(ModelRunSummary$zodSchema).nullable().optional()
31
+ .describe("Flat summary for each model in a benchmark run (fetch the benchmark detail for full results)"),
37
32
  name: z.string().describe("Display name, such as `Run 1` for a unit test or `Benchmark 1` for a benchmark"),
38
33
  passed: z.int().nullable().optional().describe("Number of test cases that passed"),
39
- results: z.array(TestCaseResult$zodSchema).nullable().optional().describe("Results for each test case"),
34
+ results: z.array(TestRunCaseSummary$zodSchema).nullable().optional()
35
+ .describe("Flat pass/fail summary for each test case (fetch the run detail for full results)"),
40
36
  share_token: z.string().nullable().optional().describe("Token for building the public share URL"),
41
37
  status: TaskStatus$zodSchema,
42
38
  total_tests: z.int().nullable().optional().describe("Total number of test cases"),
package/esm/models/agenttestrunlistitem.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agenttestrunlistitem.js","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAE/B;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,WAAW,EAAE,eAAe;IAC5B,YAAY,EAAE,eAAe;CACrB,CAAC;AAYX,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,CAAC,IAAI,CAAC;IACvD,eAAe;IACf,eAAe;CAChB,CAAC,CAAC,QAAQ,CACT,8HAA8H,CAC/H,CAAC;AAuBF,MAAM,CAAC,MAAM,8BAA8B,GAAoC,CAAC;KAC7E,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,iEAAiE,CAClE;IACD,mBAAmB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;SACnE,QAAQ,EAAE,CAAC,QAAQ,CAClB,oMAAoM,CACrM;IACH,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAChE,QAAQ,CAAC,2CAA2C,CAAC;IACxD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,gFAAgF,CACjF;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,4BAA4B,CAC7B;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,IAAI,EAAE,kCAAkC,CAAC,QAAQ,CAC/C,8HAA8H,CAC/H;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,8CAA8C,CAC/C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;CAC7C,CAAC,CAAC"}
1
+ {"version":3,"file":"agenttestrunlistitem.js","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,4BAA4B,GAC7B,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,WAAW,EAAE,eAAe;IAC5B,YAAY,EAAE,eAAe;CACrB,CAAC;AAYX,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,CAAC,IAAI,CAAC;IACvD,eAAe;IACf,eAAe;CAChB,CAAC,CAAC,QAAQ,CACT,8HAA8H,CAC/H,CAAC;AAqBF,MAAM,CAAC,MAAM,8BAA8B,GAAoC,CAAC;KAC7E,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,iEAAiE,CAClE;IACD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACpE,QAAQ,CACP,8FAA8F,CAC/F;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,gFAAgF,CACjF;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACjE,QAAQ,CACP,mFAAmF,CACpF;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,IAAI,EAAE,kCAAkC,CAAC,QAAQ,CAC/C,8HAA8H,CAC/H;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,8CAA8C,CAC/C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;CAC7C,CAAC,CAAC"}
package/esm/models/modelrunsummary.d.ts ADDED
@@ -0,0 +1,18 @@
1
+ import * as z from "zod";
2
+ /**
3
+ * Flat summary for one model in a benchmark run-LIST item. The full results
4
+ *
5
+ * @remarks
6
+ * for each case of a model live on the benchmark detail endpoint
7
+ * (`GET /agent-tests/benchmark/{task_id}`), not here.
8
+ */
9
+ export type ModelRunSummary = {
10
+ model: string;
11
+ success?: boolean | null | undefined;
12
+ message?: string | undefined;
13
+ total_tests?: number | null | undefined;
14
+ passed?: number | null | undefined;
15
+ failed?: number | null | undefined;
16
+ };
17
+ export declare const ModelRunSummary$zodSchema: z.ZodType<ModelRunSummary>;
18
+ //# sourceMappingURL=modelrunsummary.d.ts.map
package/esm/models/modelrunsummary.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"modelrunsummary.d.ts","sourceRoot":"","sources":["../../src/models/modelrunsummary.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IACrC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACpC,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,CAAC,CAAC,OAAO,CAAC,eAAe,CAmBhE,CAAC"}
package/esm/models/modelrunsummary.js ADDED
@@ -0,0 +1,13 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ */
4
+ import * as z from "zod";
5
+ export const ModelRunSummary$zodSchema = z.object({
6
+ failed: z.int().nullable().optional().describe("Number of test cases that failed for this model"),
7
+ message: z.string().default("").describe("Status or result message for this model"),
8
+ model: z.string().describe("Model name these results are for"),
9
+ passed: z.int().nullable().optional().describe("Number of test cases that passed for this model"),
10
+ success: z.boolean().nullable().optional().describe("Whether this model's run succeeded"),
11
+ total_tests: z.int().nullable().optional().describe("Total test cases for this model"),
12
+ }).describe("Flat summary for one model in a benchmark run-LIST item. The full results\nfor each case of a model live on the benchmark detail endpoint\n(`GET /agent-tests/benchmark/{task_id}`), not here.");
13
+ //# sourceMappingURL=modelrunsummary.js.map
package/esm/models/modelrunsummary.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"modelrunsummary.js","sourceRoot":"","sources":["../../src/models/modelrunsummary.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAkBzB,MAAM,CAAC,MAAM,yBAAyB,GAA+B,CAAC,CAAC,MAAM,CAAC;IAC5E,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,iDAAiD,CAClD;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,QAAQ,CACtC,yCAAyC,CAC1C;IACD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;IAC9D,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,iDAAiD,CAClD;IACD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,oCAAoC,CACrC;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,iCAAiC,CAClC;CACF,CAAC,CAAC,QAAQ,CACT,gMAAgM,CACjM,CAAC"}
package/esm/models/testruncasesummary.d.ts ADDED
@@ -0,0 +1,16 @@
1
+ import * as z from "zod";
2
+ /**
3
+ * Flat summary for one test case in the run-LIST endpoints. Carries only
4
+ *
5
+ * @remarks
6
+ * enough to render a run's pass/fail breakdown and a case name. The full detail
7
+ * for each case (agent output, judge verdicts, reasoning, latency, cost, the
8
+ * test-case definition) lives on the run-DETAIL endpoint
9
+ * (`GET /agent-tests/run/{task_id}`).
10
+ */
11
+ export type TestRunCaseSummary = {
12
+ name?: string | null | undefined;
13
+ passed?: boolean | null | undefined;
14
+ };
15
+ export declare const TestRunCaseSummary$zodSchema: z.ZodType<TestRunCaseSummary>;
16
+ //# sourceMappingURL=testruncasesummary.d.ts.map
package/esm/models/testruncasesummary.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"testruncasesummary.d.ts","sourceRoot":"","sources":["../../src/models/testruncasesummary.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB;;;;;;;;GAQG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,MAAM,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;CACrC,CAAC;AAEF,eAAO,MAAM,4BAA4B,EAAE,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAQpE,CAAC"}
package/esm/models/testruncasesummary.js ADDED
@@ -0,0 +1,10 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ */
4
+ import * as z from "zod";
5
+ export const TestRunCaseSummary$zodSchema = z
6
+ .object({
7
+ name: z.string().nullable().optional().describe("Name of the test case"),
8
+ passed: z.boolean().nullable().optional().describe("Whether the case passed (null if it errored or is still running)"),
9
+ }).describe("Flat summary for one test case in the run-LIST endpoints. Carries only\nenough to render a run's pass/fail breakdown and a case name. The full detail\nfor each case (agent output, judge verdicts, reasoning, latency, cost, the\ntest-case definition) lives on the run-DETAIL endpoint\n(`GET /agent-tests/run/{task_id}`).");
10
+ //# sourceMappingURL=testruncasesummary.js.map
package/esm/models/testruncasesummary.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"testruncasesummary.js","sourceRoot":"","sources":["../../src/models/testruncasesummary.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAgBzB,MAAM,CAAC,MAAM,4BAA4B,GAAkC,CAAC;KACzE,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACxE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChD,kEAAkE,CACnE;CACF,CAAC,CAAC,QAAQ,CACT,gUAAgU,CACjU,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dalmia/calibrate-mcp",
3
- "version": "0.0.17",
3
+ "version": "0.0.18",
4
4
  "author": "Speakeasy",
5
5
  "type": "module",
6
6
  "sideEffects": false,
package/src/landing-page.ts CHANGED
@@ -930,7 +930,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
930
930
  <h1>Instructions</h1>
931
931
  <p>One-click installation for Claude Desktop users</p>
932
932
  <div class="instruction-item">
933
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.17/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
933
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.18/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
934
934
  📥 Download MCP Bundle
935
935
  </a>
936
936
  </div>
package/src/lib/config.ts CHANGED
@@ -65,8 +65,8 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
65
65
  export const SDK_METADATA = {
66
66
  language: "typescript",
67
67
  openapiDocVersion: "0.1.0",
68
- sdkVersion: "0.0.17",
68
+ sdkVersion: "0.0.18",
69
69
  genVersion: "2.915.1",
70
70
  userAgent:
71
- "speakeasy-sdk/mcp-typescript 0.0.17 2.915.1 0.1.0 @dalmia/calibrate-mcp",
71
+ "speakeasy-sdk/mcp-typescript 0.0.18 2.915.1 0.1.0 @dalmia/calibrate-mcp",
72
72
  } as const;
package/src/mcp-server/mcp-server.ts CHANGED
@@ -21,7 +21,7 @@ const routes = buildRouteMap({
21
21
  export const app = buildApplication(routes, {
22
22
  name: "mcp",
23
23
  versionInfo: {
24
- currentVersion: "0.0.17",
24
+ currentVersion: "0.0.18",
25
25
  },
26
26
  });
27
27
 
package/src/mcp-server/server.ts CHANGED
@@ -64,7 +64,7 @@ export function createMCPServer(deps: {
64
64
  }) {
65
65
  const server = new McpServer({
66
66
  name: "CalibrateMcp",
67
- version: "0.0.17",
67
+ version: "0.0.18",
68
68
  });
69
69
 
70
70
  const getClient = deps.getSDK || (() =>
package/src/models/agenttestrunlistitem.ts CHANGED
@@ -4,13 +4,15 @@
4
4
 
5
5
  import * as z from "zod";
6
6
  import { ClosedEnum } from "../types/enums.js";
7
- import { ModelResult, ModelResult$zodSchema } from "./modelresult.js";
7
+ import {
8
+ ModelRunSummary,
9
+ ModelRunSummary$zodSchema,
10
+ } from "./modelrunsummary.js";
8
11
  import { TaskStatus, TaskStatus$zodSchema } from "./taskstatus.js";
9
- import { TestCaseResult, TestCaseResult$zodSchema } from "./testcaseresult.js";
10
12
  import {
11
- TestRunEvaluator,
12
- TestRunEvaluator$zodSchema,
13
- } from "./testrunevaluator.js";
13
+ TestRunCaseSummary,
14
+ TestRunCaseSummary$zodSchema,
15
+ } from "./testruncasesummary.js";
14
16
 
15
17
  /**
16
18
  * What kind of run this is:
@@ -47,16 +49,14 @@ export type AgentTestRunListItem = {
47
49
  status: TaskStatus;
48
50
  type: AgentTestRunListItemType;
49
51
  updated_at: string;
50
- evaluators?: Array<TestRunEvaluator> | null | undefined;
51
52
  total_tests?: number | null | undefined;
52
53
  passed?: number | null | undefined;
53
54
  failed?: number | null | undefined;
54
- results?: Array<TestCaseResult> | null | undefined;
55
+ results?: Array<TestRunCaseSummary> | null | undefined;
55
56
  latency_ms?: { [k: string]: any } | null | undefined;
56
57
  cost?: { [k: string]: any } | null | undefined;
57
58
  total_tokens?: { [k: string]: any } | null | undefined;
58
- model_results?: Array<ModelResult> | null | undefined;
59
- leaderboard_summary?: Array<{ [k: string]: any }> | null | undefined;
59
+ model_results?: Array<ModelRunSummary> | null | undefined;
60
60
  error?: boolean | undefined;
61
61
  is_public?: boolean | undefined;
62
62
  share_token?: string | null | undefined;
@@ -68,10 +68,6 @@ export const AgentTestRunListItem$zodSchema: z.ZodType<AgentTestRunListItem> = z
68
68
  "Aggregated cost as `{mean, min, max, count}` (USD)",
69
69
  ),
70
70
  error: z.boolean().default(false).describe("True if the run failed"),
71
- evaluators: z.array(TestRunEvaluator$zodSchema).nullable().optional()
72
- .describe(
73
- "The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`",
74
- ),
75
71
  failed: z.int().nullable().optional().describe(
76
72
  "Number of test cases that failed",
77
73
  ),
@@ -81,21 +77,20 @@ export const AgentTestRunListItem$zodSchema: z.ZodType<AgentTestRunListItem> = z
81
77
  latency_ms: z.record(z.string(), z.any()).nullable().optional().describe(
82
78
  "Aggregated latency in milliseconds, as `{p50, p95, p99, count}`",
83
79
  ),
84
- leaderboard_summary: z.array(z.record(z.string(), z.any())).nullable()
85
- .optional().describe(
86
- "Leaderboard comparing the models, one row per model. Columns vary by benchmark: a `model` column plus pass/fail counts, latency, cost, and one score column per evaluator, keyed by evaluator name",
80
+ model_results: z.array(ModelRunSummary$zodSchema).nullable().optional()
81
+ .describe(
82
+ "Flat summary for each model in a benchmark run (fetch the benchmark detail for full results)",
87
83
  ),
88
- model_results: z.array(ModelResult$zodSchema).nullable().optional()
89
- .describe("Results for each model in a benchmark run"),
90
84
  name: z.string().describe(
91
85
  "Display name, such as `Run 1` for a unit test or `Benchmark 1` for a benchmark",
92
86
  ),
93
87
  passed: z.int().nullable().optional().describe(
94
88
  "Number of test cases that passed",
95
89
  ),
96
- results: z.array(TestCaseResult$zodSchema).nullable().optional().describe(
97
- "Results for each test case",
98
- ),
90
+ results: z.array(TestRunCaseSummary$zodSchema).nullable().optional()
91
+ .describe(
92
+ "Flat pass/fail summary for each test case (fetch the run detail for full results)",
93
+ ),
99
94
  share_token: z.string().nullable().optional().describe(
100
95
  "Token for building the public share URL",
101
96
  ),
package/src/models/modelrunsummary.ts ADDED
@@ -0,0 +1,42 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ */
4
+
5
+ import * as z from "zod";
6
+
7
+ /**
8
+ * Flat summary for one model in a benchmark run-LIST item. The full results
9
+ *
10
+ * @remarks
11
+ * for each case of a model live on the benchmark detail endpoint
12
+ * (`GET /agent-tests/benchmark/{task_id}`), not here.
13
+ */
14
+ export type ModelRunSummary = {
15
+ model: string;
16
+ success?: boolean | null | undefined;
17
+ message?: string | undefined;
18
+ total_tests?: number | null | undefined;
19
+ passed?: number | null | undefined;
20
+ failed?: number | null | undefined;
21
+ };
22
+
23
+ export const ModelRunSummary$zodSchema: z.ZodType<ModelRunSummary> = z.object({
24
+ failed: z.int().nullable().optional().describe(
25
+ "Number of test cases that failed for this model",
26
+ ),
27
+ message: z.string().default("").describe(
28
+ "Status or result message for this model",
29
+ ),
30
+ model: z.string().describe("Model name these results are for"),
31
+ passed: z.int().nullable().optional().describe(
32
+ "Number of test cases that passed for this model",
33
+ ),
34
+ success: z.boolean().nullable().optional().describe(
35
+ "Whether this model's run succeeded",
36
+ ),
37
+ total_tests: z.int().nullable().optional().describe(
38
+ "Total test cases for this model",
39
+ ),
40
+ }).describe(
41
+ "Flat summary for one model in a benchmark run-LIST item. The full results\nfor each case of a model live on the benchmark detail endpoint\n(`GET /agent-tests/benchmark/{task_id}`), not here.",
42
+ );
package/src/models/testruncasesummary.ts ADDED
@@ -0,0 +1,29 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ */
4
+
5
+ import * as z from "zod";
6
+
7
+ /**
8
+ * Flat summary for one test case in the run-LIST endpoints. Carries only
9
+ *
10
+ * @remarks
11
+ * enough to render a run's pass/fail breakdown and a case name. The full detail
12
+ * for each case (agent output, judge verdicts, reasoning, latency, cost, the
13
+ * test-case definition) lives on the run-DETAIL endpoint
14
+ * (`GET /agent-tests/run/{task_id}`).
15
+ */
16
+ export type TestRunCaseSummary = {
17
+ name?: string | null | undefined;
18
+ passed?: boolean | null | undefined;
19
+ };
20
+
21
+ export const TestRunCaseSummary$zodSchema: z.ZodType<TestRunCaseSummary> = z
22
+ .object({
23
+ name: z.string().nullable().optional().describe("Name of the test case"),
24
+ passed: z.boolean().nullable().optional().describe(
25
+ "Whether the case passed (null if it errored or is still running)",
26
+ ),
27
+ }).describe(
28
+ "Flat summary for one test case in the run-LIST endpoints. Carries only\nenough to render a run's pass/fail breakdown and a case name. The full detail\nfor each case (agent output, judge verdicts, reasoning, latency, cost, the\ntest-case definition) lives on the run-DETAIL endpoint\n(`GET /agent-tests/run/{task_id}`).",
29
+ );