@dalmia/calibrate-mcp 0.0.61 → 0.0.63

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +1 -1
  2. package/bin/mcp-server.js +15 -14
  3. package/bin/mcp-server.js.map +13 -13
  4. package/esm/landing-page.js +1 -1
  5. package/esm/lib/config.d.ts +2 -2
  6. package/esm/lib/config.js +2 -2
  7. package/esm/mcp-server/mcp-server.js +1 -1
  8. package/esm/mcp-server/server.js +1 -1
  9. package/esm/models/agentcreate.js +1 -1
  10. package/esm/models/agentcreate.js.map +1 -1
  11. package/esm/models/agenttestrunlistitem.js +1 -1
  12. package/esm/models/agenttestrunlistitem.js.map +1 -1
  13. package/esm/models/agentupdate.js +1 -1
  14. package/esm/models/agentupdate.js.map +1 -1
  15. package/esm/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.js +1 -1
  16. package/esm/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.js.map +1 -1
  17. package/esm/models/modelresult.js +1 -1
  18. package/esm/models/modelresult.js.map +1 -1
  19. package/esm/models/modelrunsummary.d.ts +1 -0
  20. package/esm/models/modelrunsummary.d.ts.map +1 -1
  21. package/esm/models/modelrunsummary.js +2 -1
  22. package/esm/models/modelrunsummary.js.map +1 -1
  23. package/esm/models/testrunstatusresponse.js +1 -1
  24. package/esm/models/testrunstatusresponse.js.map +1 -1
  25. package/package.json +1 -1
  26. package/src/landing-page.ts +1 -1
  27. package/src/lib/config.ts +2 -2
  28. package/src/mcp-server/mcp-server.ts +1 -1
  29. package/src/mcp-server/server.ts +1 -1
  30. package/src/models/agentcreate.ts +1 -1
  31. package/src/models/agenttestrunlistitem.ts +1 -1
  32. package/src/models/agentupdate.ts +1 -1
  33. package/src/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.ts +1 -1
  34. package/src/models/modelresult.ts +1 -1
  35. package/src/models/modelrunsummary.ts +5 -1
  36. package/src/models/testrunstatusresponse.ts +1 -1
package/esm/landing-page.js CHANGED
@@ -915,7 +915,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
915
915
  <h1>Instructions</h1>
916
916
  <p>One-click installation for Claude Desktop users</p>
917
917
  <div class="instruction-item">
918
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.61/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
918
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.63/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
919
919
  📥 Download MCP Bundle
920
920
  </a>
921
921
  </div>
package/esm/lib/config.d.ts CHANGED
@@ -35,8 +35,8 @@ export declare function serverURLFromOptions(options: SDKOptions): URL | null;
35
35
  export declare const SDK_METADATA: {
36
36
  readonly language: "typescript";
37
37
  readonly openapiDocVersion: "0.1.0";
38
- readonly sdkVersion: "0.0.61";
38
+ readonly sdkVersion: "0.0.63";
39
39
  readonly genVersion: "2.915.1";
40
- readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.61 2.915.1 0.1.0 @dalmia/calibrate-mcp";
40
+ readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.63 2.915.1 0.1.0 @dalmia/calibrate-mcp";
41
41
  };
42
42
  //# sourceMappingURL=config.d.ts.map
package/esm/lib/config.js CHANGED
@@ -27,8 +27,8 @@ export function serverURLFromOptions(options) {
27
27
  export const SDK_METADATA = {
28
28
  language: "typescript",
29
29
  openapiDocVersion: "0.1.0",
30
- sdkVersion: "0.0.61",
30
+ sdkVersion: "0.0.63",
31
31
  genVersion: "2.915.1",
32
- userAgent: "speakeasy-sdk/mcp-typescript 0.0.61 2.915.1 0.1.0 @dalmia/calibrate-mcp",
32
+ userAgent: "speakeasy-sdk/mcp-typescript 0.0.63 2.915.1 0.1.0 @dalmia/calibrate-mcp",
33
33
  };
34
34
  //# sourceMappingURL=config.js.map
package/esm/mcp-server/mcp-server.js CHANGED
@@ -18,7 +18,7 @@ const routes = buildRouteMap({
18
18
  export const app = buildApplication(routes, {
19
19
  name: "mcp",
20
20
  versionInfo: {
21
- currentVersion: "0.0.61",
21
+ currentVersion: "0.0.63",
22
22
  },
23
23
  });
24
24
  run(app, process.argv.slice(2), buildContext(process));
package/esm/mcp-server/server.js CHANGED
@@ -51,7 +51,7 @@ import { tool$tracesCreate } from "./tools/tracesCreate.js";
51
51
  export function createMCPServer(deps) {
52
52
  const server = new McpServer({
53
53
  name: "CalibrateMcp",
54
- version: "0.0.61",
54
+ version: "0.0.63",
55
55
  });
56
56
  const getClient = deps.getSDK || (() => new CalibrateMcpCore({
57
57
  security: deps.security,
package/esm/models/agentcreate.js CHANGED
@@ -33,7 +33,7 @@ export const AgentCreateInteractionType$zodSchema = z.enum([
33
33
  "general",
34
34
  ]).describe("What the agent expects in the request body:\n\n- `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{\"messages\": [...]}`\n- `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{\"input\": \"...\"}`");
35
35
  export const AgentCreate$zodSchema = z.object({
36
- config: z.record(z.string(), z.any()).nullable().optional().describe("Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n \"system_prompt\": \"You are a helpful support agent.\",\n \"llm\": {\"model\": \"openai/gpt-4.1\"},\n \"stt\": {\"provider\": \"deepgram\"},\n \"tts\": {\"provider\": \"elevenlabs\"},\n \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n \"agent_url\": \"https://api.example.com/agent\",\n \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n \"benchmark_provider\": \"openrouter\"\n}\n```\n\nEvery request Calibrate makes to your endpoint carries the header\n`X-Calibrate-Eval: 1`. Read it to tell a test run from a real user, for example\nto tag the trace you send back or to skip sending one.\n\nFor `type=agent`, omitted keys inherit managed defaults. Omit `config` entirely to use all defaults. For `type=connection`, `config` is stored as-is and must contain `agent_url`"),
36
+ config: z.record(z.string(), z.any()).nullable().optional().describe("Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n \"system_prompt\": \"You are a helpful support agent.\",\n \"llm\": {\"model\": \"openai/gpt-4.1\"},\n \"stt\": {\"provider\": \"deepgram\"},\n \"tts\": {\"provider\": \"elevenlabs\"},\n \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n \"agent_url\": \"https://api.example.com/agent\",\n \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n \"benchmark_provider\": \"openrouter\"\n}\n```\n\n**Either type**:\n- `traces.scoring.enabled`: whether the traces you send for this agent are scored by its linked evaluators. On unless you set it to `false`. Turning it back on needs at least one linked evaluator that can score traces\n\n```json\n{\"traces\": {\"scoring\": {\"enabled\": false}}}\n```\n\nEvery request Calibrate makes to your endpoint carries the header\n`X-Calibrate-Eval: 1`. Read it to tell a test run from a real user, for example\nto tag the trace you send back or to skip sending one.\n\nFor `type=agent`, omitted keys inherit managed defaults. Omit `config` entirely to use all defaults. For `type=connection`, `config` is stored as-is and must contain `agent_url`"),
37
37
  interaction_type: AgentCreateInteractionType$zodSchema.default("conversation")
38
38
  .describe("What the agent expects in the request body:\n\n- `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{\"messages\": [...]}`\n- `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{\"input\": \"...\"}`"),
39
39
  name: z.string().describe("Agent name, unique within the workspace"),
package/esm/models/agentcreate.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agentcreate.js","sourceRoot":"","sources":["../../src/models/agentcreate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,KAAK,EAAE,OAAO;IACd,UAAU,EAAE,YAAY;CAChB,CAAC;AASX,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC,IAAI,CAAC;IAC9C,OAAO;IACP,YAAY;CACb,CAAC,CAAC,QAAQ,CACT,+FAA+F,CAChG,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,YAAY,EAAE,cAAc;IAC5B,OAAO,EAAE,SAAS;CACV,CAAC;AAaX,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,CAAC,IAAI,CAAC;IACzD,cAAc;IACd,SAAS;CACV,CAAC,CAAC,QAAQ,CACT,0TAA0T,CAC3T,CAAC;AASF,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,kxDAAkxD,CACnxD;IACD,gBAAgB,EAAE,oCAAoC,CAAC,OAAO,CAAC,cAAc,CAAC;SAC3E,QAAQ,CACP,0TAA0T,CAC3T;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IACpE,IAAI,EAAE,yBAAyB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,QAAQ,CACvD,+FAA+F,CAChG;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"agentcreate.js","sourceRoot":"","sources":["../../src/models/agentcreate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,KAAK,EAAE,OAAO;IACd,UAAU,EAAE,YAAY;CAChB,CAAC;AASX,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC,IAAI,CAAC;IAC9C,OAAO;IACP,YAAY;CACb,CAAC,CAAC,QAAQ,CACT,+FAA+F,CAChG,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,YAAY,EAAE,cAAc;IAC5B,OAAO,EAAE,SAAS;CACV,CAAC;AAaX,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,CAAC,IAAI,CAAC;IACzD,cAAc;IACd,SAAS;CACV,CAAC,CAAC,QAAQ,CACT,0TAA0T,CAC3T,CAAC;AASF,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,okEAAokE,CACrkE;IACD,gBAAgB,EAAE,oCAAoC,CAAC,OAAO,CAAC,cAAc,CAAC;SAC3E,QAAQ,CACP,0TAA0T,CAC3T;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IACpE,IAAI,EAAE,yBAAyB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,QAAQ,CACvD,+FAA+F,CAChG;CACF,CAAC,CAAC"}
package/esm/models/agenttestrunlistitem.js CHANGED
@@ -28,7 +28,7 @@ export const AgentTestRunListItem$zodSchema = z
28
28
  created_at: z.string().describe("When the run was created (ISO 8601 UTC)"),
29
29
  error: z.boolean().default(false).describe("True if the run failed"),
30
30
  evaluators: z.array(RunListEvaluator$zodSchema).optional().describe("The evaluators that judged this run, deduplicated and in display order. A `Tool call` entry is appended when any test in the run was a tool-call test. That entry has no `uuid`, because it is not an evaluator in the library. Empty when the run had no evaluators"),
31
- failed: z.int().nullable().optional().describe("Number of test cases that failed"),
31
+ failed: z.int().nullable().optional().describe("Number of test cases that did not pass, which includes the ones that produced no answer"),
32
32
  is_public: z.boolean().default(false).describe("Whether the run is shared publicly"),
33
33
  latency_ms: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated latency in milliseconds, as `{p50, p95, p99, count}`"),
34
34
  model_results: z.array(ModelRunSummary$zodSchema).nullable().optional()
package/esm/models/agenttestrunlistitem.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agenttestrunlistitem.js","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,4BAA4B,GAC7B,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,WAAW,EAAE,eAAe;IAC5B,YAAY,EAAE,eAAe;CACrB,CAAC;AAYX,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,CAAC,IAAI,CAAC;IACvD,eAAe;IACf,eAAe;CAChB,CAAC,CAAC,QAAQ,CACT,8HAA8H,CAC/H,CAAC;AA0BF,MAAM,CAAC,MAAM,8BAA8B,GAAoC,CAAC;KAC7E,MAAM,CAAC;IACN,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC1C,gLAAgL,CACjL;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IAC1E,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjE,sQAAsQ,CACvQ;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,iEAAiE,CAClE;IACD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACpE,QAAQ,CACP,8FAA8F,CAC/F;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,qIAAqI,CACtI;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACjE,QAAQ,CACP,mFAAmF,CACpF;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,yFAAyF,CAC1F;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,IAAI,EAAE,kCAAkC,CAAC,QAAQ,CAC/C,8HAA8H,CAC/H;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,8CAA8C,CAC/C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;CAC7C,CAAC,CAAC"}
1
+ {"version":3,"file":"agenttestrunlistitem.js","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,4BAA4B,GAC7B,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,WAAW,EAAE,eAAe;IAC5B,YAAY,EAAE,eAAe;CACrB,CAAC;AAYX,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,CAAC,IAAI,CAAC;IACvD,eAAe;IACf,eAAe;CAChB,CAAC,CAAC,QAAQ,CACT,8HAA8H,CAC/H,CAAC;AA0BF,MAAM,CAAC,MAAM,8BAA8B,GAAoC,CAAC;KAC7E,MAAM,CAAC;IACN,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC1C,gLAAgL,CACjL;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IAC1E,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjE,sQAAsQ,CACvQ;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,yFAAyF,CAC1F;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,iEAAiE,CAClE;IACD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACpE,QAAQ,CACP,8FAA8F,CAC/F;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,qIAAqI,CACtI;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACjE,QAAQ,CACP,mFAAmF,CACpF;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,yFAAyF,CAC1F;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,IAAI,EAAE,kCAAkC,CAAC,QAAQ,CAC/C,8HAA8H,CAC/H;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,8CAA8C,CAC/C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;CAC7C,CAAC,CAAC"}
package/esm/models/agentupdate.js CHANGED
@@ -3,7 +3,7 @@
3
3
  */
4
4
  import * as z from "zod";
5
5
  export const AgentUpdate$zodSchema = z.object({
6
- config: z.record(z.string(), z.any()).nullable().optional().describe("Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n \"system_prompt\": \"You are a helpful support agent.\",\n \"llm\": {\"model\": \"openai/gpt-4.1\"},\n \"stt\": {\"provider\": \"deepgram\"},\n \"tts\": {\"provider\": \"elevenlabs\"},\n \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n \"agent_url\": \"https://api.example.com/agent\",\n \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n \"benchmark_provider\": \"openrouter\"\n}\n```\n\nEvery request Calibrate makes to your endpoint carries the header\n`X-Calibrate-Eval: 1`. Read it to tell a test run from a real user, for example\nto tag the trace you send back or to skip sending one.\n\nReplaces the stored config. Omit to leave unchanged\n\nFor `type=connection`, changing `agent_url` or `agent_headers` resets the connection and benchmark verification flags"),
6
+ config: z.record(z.string(), z.any()).nullable().optional().describe("Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n \"system_prompt\": \"You are a helpful support agent.\",\n \"llm\": {\"model\": \"openai/gpt-4.1\"},\n \"stt\": {\"provider\": \"deepgram\"},\n \"tts\": {\"provider\": \"elevenlabs\"},\n \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n \"agent_url\": \"https://api.example.com/agent\",\n \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n \"benchmark_provider\": \"openrouter\"\n}\n```\n\n**Either type**:\n- `traces.scoring.enabled`: whether the traces you send for this agent are scored by its linked evaluators. On unless you set it to `false`. Turning it back on needs at least one linked evaluator that can score traces\n\n```json\n{\"traces\": {\"scoring\": {\"enabled\": false}}}\n```\n\nEvery request Calibrate makes to your endpoint carries the header\n`X-Calibrate-Eval: 1`. Read it to tell a test run from a real user, for example\nto tag the trace you send back or to skip sending one.\n\nReplaces the stored config. Omit to leave unchanged\n\nFor `type=connection`, changing `agent_url` or `agent_headers` resets the connection and benchmark verification flags"),
7
7
  name: z.string().nullable().optional().describe("New agent name. Omit to leave the name unchanged"),
8
8
  });
9
9
  //# sourceMappingURL=agentupdate.js.map
package/esm/models/agentupdate.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agentupdate.js","sourceRoot":"","sources":["../../src/models/agentupdate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAOzB,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,6wDAA6wD,CAC9wD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,kDAAkD,CACnD;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"agentupdate.js","sourceRoot":"","sources":["../../src/models/agentupdate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAOzB,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,+jEAA+jE,CAChkE;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,kDAAkD,CACnD;CACF,CAAC,CAAC"}
package/esm/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.js CHANGED
@@ -24,7 +24,7 @@ export const GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetType$zodSchema = z
24
24
  export const GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetRequest$zodSchema = z.object({
25
25
  agent_uuid: z.string().describe("Agent whose test runs to list"),
26
26
  around: z.string().describe("ID of a run to jump to, returning the page that contains it instead of the page at `offset`").nullable().optional(),
27
- has_failures: z.boolean().describe("Filter by whether the run has any failing test case or model. `true` returns only runs with failures (or errors), `false` only clean runs. Omit for both").nullable().optional(),
27
+ has_failures: z.boolean().describe("Filter by whether a test in the run did not pass. `true` returns only runs with a failing test, `false` only runs that got through every test and passed them all. A run that broke, was stopped, or gave up part way with no failing test is in neither: filter by `status` for those. Omit for both").nullable().optional(),
28
28
  limit: z.int().describe("Maximum number of items to return. Omit for no limit (all items)").nullable().optional(),
29
29
  offset: z.int().default(0).describe("Number of items to skip before returning results"),
30
30
  status: TaskStatus$zodSchema.nullable().optional().describe("Filter by run status. Omit for all statuses"),
package/esm/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"getagenttestrunsagenttestsagentagentuuidrunsgetop.js","sourceRoot":"","sources":["../../src/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAEL,+CAA+C,GAChD,MAAM,4CAA4C,CAAC;AACpD,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAEnE;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,mDAAmD,GAAG;IACjE,WAAW,EAAE,eAAe;IAC5B,YAAY,EAAE,eAAe;CACrB,CAAC;AAYX,MAAM,CAAC,MAAM,6DAA6D,GAAG,CAAC;KAC3E,IAAI,CAAC;IACJ,eAAe;IACf,eAAe;CAChB,CAAC,CAAC,QAAQ,CACT,0IAA0I,CAC3I,CAAC;AAaJ,MAAM,CAAC,MAAM,gEAAgE,GACP,CAAC,CAAC,MAAM,CAAC;IAC3E,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;IAChE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACzB,6FAA6F,CAC9F,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACvB,YAAY,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAChC,0JAA0J,CAC3J,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACvB,KAAK,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,CACrB,kEAAkE,CACnE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACvB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CACjC,kDAAkD,CACnD;IACD,MAAM,EAAE,oBAAoB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACzD,6CAA6C,CAC9C;IACD,IAAI,EAAE,6DAA6D;SAChE,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7B,0IAA0I,CAC3I;IACH,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC1C,CAAC,CAAC;AAML,MAAM,CAAC,MAAM,iEAAiE,GACP,CAAC,CAAC,KAAK,CAAC;IAC3E,+CAA+C;IAC/C,6BAA6B;CAC9B,CAAC,CAAC"}
1
+ {"version":3,"file":"getagenttestrunsagenttestsagentagentuuidrunsgetop.js","sourceRoot":"","sources":["../../src/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAEL,+CAA+C,GAChD,MAAM,4CAA4C,CAAC;AACpD,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAEnE;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,mDAAmD,GAAG;IACjE,WAAW,EAAE,eAAe;IAC5B,YAAY,EAAE,eAAe;CACrB,CAAC;AAYX,MAAM,CAAC,MAAM,6DAA6D,GAAG,CAAC;KAC3E,IAAI,CAAC;IACJ,eAAe;IACf,eAAe;CAChB,CAAC,CAAC,QAAQ,CACT,0IAA0I,CAC3I,CAAC;AAaJ,MAAM,CAAC,MAAM,gEAAgE,GACP,CAAC,CAAC,MAAM,CAAC;IAC3E,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;IAChE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACzB,6FAA6F,CAC9F,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACvB,YAAY,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAChC,uSAAuS,CACxS,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACvB,KAAK,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,CACrB,kEAAkE,CACnE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACvB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CACjC,kDAAkD,CACnD;IACD,MAAM,EAAE,oBAAoB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACzD,6CAA6C,CAC9C;IACD,IAAI,EAAE,6DAA6D;SAChE,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7B,0IAA0I,CAC3I;IACH,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC1C,CAAC,CAAC;AAML,MAAM,CAAC,MAAM,iEAAiE,GACP,CAAC,CAAC,KAAK,CAAC;IAC3E,+CAA+C;IAC/C,6BAA6B;CAC9B,CAAC,CAAC"}
package/esm/models/modelresult.js CHANGED
@@ -7,7 +7,7 @@ export const ModelResult$zodSchema = z.object({
7
7
  cost: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated cost as `{mean, min, max, count}` (USD)"),
8
8
  evaluator_summary: z.array(z.record(z.string(), z.any())).nullable()
9
9
  .optional().describe("Aggregate summary for each evaluator for this model"),
10
- failed: z.int().nullable().optional().describe("Number of test cases that failed"),
10
+ failed: z.int().nullable().optional().describe("Number of test cases that did not pass, which includes the ones that produced no answer"),
11
11
  latency_ms: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated latency in milliseconds, as `{p50, p95, p99, count}`"),
12
12
  message: z.string().describe("Status or result message for this model"),
13
13
  model: z.string().describe("Model name these results are for"),
package/esm/models/modelresult.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"modelresult.js","sourceRoot":"","sources":["../../src/models/modelresult.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAkB/E,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;SACjE,QAAQ,EAAE,CAAC,QAAQ,CAAC,qDAAqD,CAAC;IAC7E,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,iEAAiE,CAClE;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IACvE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;IAC9D,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,kGAAkG,CACnG;IACD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,oCAAoC,CACrC;IACD,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CAAC,2CAA2C,CAAC;IACxD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,iCAAiC,CAClC;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"modelresult.js","sourceRoot":"","sources":["../../src/models/modelresult.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAkB/E,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;SACjE,QAAQ,EAAE,CAAC,QAAQ,CAAC,qDAAqD,CAAC;IAC7E,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,yFAAyF,CAC1F;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,iEAAiE,CAClE;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IACvE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;IAC9D,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,kGAAkG,CACnG;IACD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,oCAAoC,CACrC;IACD,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CAAC,2CAA2C,CAAC;IACxD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,iCAAiC,CAClC;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;CACF,CAAC,CAAC"}
package/esm/models/modelrunsummary.d.ts CHANGED
@@ -13,6 +13,7 @@ export type ModelRunSummary = {
13
13
  total_tests?: number | null | undefined;
14
14
  passed?: number | null | undefined;
15
15
  failed?: number | null | undefined;
16
+ unanswered_tests?: number | null | undefined;
16
17
  };
17
18
  export declare const ModelRunSummary$zodSchema: z.ZodType<ModelRunSummary>;
18
19
  //# sourceMappingURL=modelrunsummary.d.ts.map
package/esm/models/modelrunsummary.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"modelrunsummary.d.ts","sourceRoot":"","sources":["../../src/models/modelrunsummary.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IACrC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACpC,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,CAAC,CAAC,OAAO,CAAC,eAAe,CAmBhE,CAAC"}
1
+ {"version":3,"file":"modelrunsummary.d.ts","sourceRoot":"","sources":["../../src/models/modelrunsummary.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IACrC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,CAAC,CAAC,OAAO,CAAC,eAAe,CAsBhE,CAAC"}
package/esm/models/modelrunsummary.js CHANGED
@@ -3,11 +3,12 @@
3
3
  */
4
4
  import * as z from "zod";
5
5
  export const ModelRunSummary$zodSchema = z.object({
6
- failed: z.int().nullable().optional().describe("Number of test cases that failed for this model"),
6
+ failed: z.int().nullable().optional().describe("Number of test cases that did not pass for this model, which includes the ones that produced no answer"),
7
7
  message: z.string().default("").describe("Status or result message for this model"),
8
8
  model: z.string().describe("Model name these results are for"),
9
9
  passed: z.int().nullable().optional().describe("Number of test cases that passed for this model"),
10
10
  success: z.boolean().nullable().optional().describe("Whether this model's run succeeded"),
11
11
  total_tests: z.int().nullable().optional().describe("Total test cases for this model"),
12
+ unanswered_tests: z.int().nullable().optional().describe("Number of this model's test cases that produced no answer, already counted in `failed`"),
12
13
  }).describe("Flat summary for one model in a benchmark run-LIST item. The full results\nfor each case of a model live on the benchmark detail endpoint\n(`GET /agent-tests/benchmark/{task_id}`), not here.");
13
14
  //# sourceMappingURL=modelrunsummary.js.map
package/esm/models/modelrunsummary.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"modelrunsummary.js","sourceRoot":"","sources":["../../src/models/modelrunsummary.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAkBzB,MAAM,CAAC,MAAM,yBAAyB,GAA+B,CAAC,CAAC,MAAM,CAAC;IAC5E,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,iDAAiD,CAClD;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,QAAQ,CACtC,yCAAyC,CAC1C;IACD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;IAC9D,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,iDAAiD,CAClD;IACD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,oCAAoC,CACrC;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,iCAAiC,CAClC;CACF,CAAC,CAAC,QAAQ,CACT,gMAAgM,CACjM,CAAC"}
1
+ {"version":3,"file":"modelrunsummary.js","sourceRoot":"","sources":["../../src/models/modelrunsummary.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAmBzB,MAAM,CAAC,MAAM,yBAAyB,GAA+B,CAAC,CAAC,MAAM,CAAC;IAC5E,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,wGAAwG,CACzG;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,QAAQ,CACtC,yCAAyC,CAC1C;IACD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;IAC9D,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,iDAAiD,CAClD;IACD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,oCAAoC,CACrC;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,iCAAiC,CAClC;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,wFAAwF,CACzF;CACF,CAAC,CAAC,QAAQ,CACT,gMAAgM,CACjM,CAAC"}
package/esm/models/testrunstatusresponse.js CHANGED
@@ -13,7 +13,7 @@ export const TestRunStatusResponse$zodSchema = z.object({
13
13
  .optional().describe("Totals for each evaluator over the whole run, matching the shape a benchmark reports for each model. Only evaluators that returned a verdict appear"),
14
14
  evaluators: z.array(TestRunEvaluator$zodSchema).nullable().optional()
15
15
  .describe("The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`"),
16
- failed: z.int().nullable().optional().describe("Number of test cases that failed"),
16
+ failed: z.int().nullable().optional().describe("Number of test cases that did not pass, which includes the ones that produced no answer"),
17
17
  is_public: z.boolean().default(false).describe("Whether the run is shared publicly"),
18
18
  latency_ms: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated response latency in milliseconds, as `{p50, p95, p99, count}`"),
19
19
  name: z.string().describe("Name of the run. A run nobody has renamed shows its number instead, such as `Run 1` for a test run or `Benchmark 1` for a benchmark"),
package/esm/models/testrunstatusresponse.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testrunstatusresponse.js","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAwB/B,MAAM,CAAC,MAAM,+BAA+B,GAC1C,CAAC,CAAC,MAAM,CAAC;IACP,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC1C,gLAAgL,CACjL;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC9C,kFAAkF,CACnF;IACD,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;SACjE,QAAQ,EAAE,CAAC,QAAQ,CAClB,qJAAqJ,CACtJ;IACH,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,0EAA0E,CAC3E;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,qIAAqI,CACtI;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,4BAA4B,CAC7B;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,yFAAyF,CAC1F;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,kDAAkD,CACnD;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"testrunstatusresponse.js","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAwB/B,MAAM,CAAC,MAAM,+BAA+B,GAC1C,CAAC,CAAC,MAAM,CAAC;IACP,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC1C,gLAAgL,CACjL;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC9C,kFAAkF,CACnF;IACD,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;SACjE,QAAQ,EAAE,CAAC,QAAQ,CAClB,qJAAqJ,CACtJ;IACH,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,yFAAyF,CAC1F;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,0EAA0E,CAC3E;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,qIAAqI,CACtI;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,4BAA4B,CAC7B;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,yFAAyF,CAC1F;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,kDAAkD,CACnD;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;CACF,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dalmia/calibrate-mcp",
3
- "version": "0.0.61",
3
+ "version": "0.0.63",
4
4
  "author": "Speakeasy",
5
5
  "type": "module",
6
6
  "sideEffects": false,
package/src/landing-page.ts CHANGED
@@ -930,7 +930,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
930
930
  <h1>Instructions</h1>
931
931
  <p>One-click installation for Claude Desktop users</p>
932
932
  <div class="instruction-item">
933
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.61/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
933
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.63/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
934
934
  📥 Download MCP Bundle
935
935
  </a>
936
936
  </div>
package/src/lib/config.ts CHANGED
@@ -65,8 +65,8 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
65
65
  export const SDK_METADATA = {
66
66
  language: "typescript",
67
67
  openapiDocVersion: "0.1.0",
68
- sdkVersion: "0.0.61",
68
+ sdkVersion: "0.0.63",
69
69
  genVersion: "2.915.1",
70
70
  userAgent:
71
- "speakeasy-sdk/mcp-typescript 0.0.61 2.915.1 0.1.0 @dalmia/calibrate-mcp",
71
+ "speakeasy-sdk/mcp-typescript 0.0.63 2.915.1 0.1.0 @dalmia/calibrate-mcp",
72
72
  } as const;
package/src/mcp-server/mcp-server.ts CHANGED
@@ -21,7 +21,7 @@ const routes = buildRouteMap({
21
21
  export const app = buildApplication(routes, {
22
22
  name: "mcp",
23
23
  versionInfo: {
24
- currentVersion: "0.0.61",
24
+ currentVersion: "0.0.63",
25
25
  },
26
26
  });
27
27
 
package/src/mcp-server/server.ts CHANGED
@@ -73,7 +73,7 @@ export function createMCPServer(deps: {
73
73
  }) {
74
74
  const server = new McpServer({
75
75
  name: "CalibrateMcp",
76
- version: "0.0.61",
76
+ version: "0.0.63",
77
77
  });
78
78
 
79
79
  const getClient = deps.getSDK || (() =>
package/src/models/agentcreate.ts CHANGED
@@ -70,7 +70,7 @@ export type AgentCreate = {
70
70
 
71
71
  export const AgentCreate$zodSchema: z.ZodType<AgentCreate> = z.object({
72
72
  config: z.record(z.string(), z.any()).nullable().optional().describe(
73
- "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n \"system_prompt\": \"You are a helpful support agent.\",\n \"llm\": {\"model\": \"openai/gpt-4.1\"},\n \"stt\": {\"provider\": \"deepgram\"},\n \"tts\": {\"provider\": \"elevenlabs\"},\n \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n \"agent_url\": \"https://api.example.com/agent\",\n \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n \"benchmark_provider\": \"openrouter\"\n}\n```\n\nEvery request Calibrate makes to your endpoint carries the header\n`X-Calibrate-Eval: 1`. Read it to tell a test run from a real user, for example\nto tag the trace you send back or to skip sending one.\n\nFor `type=agent`, omitted keys inherit managed defaults. Omit `config` entirely to use all defaults. For `type=connection`, `config` is stored as-is and must contain `agent_url`",
73
+ "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n \"system_prompt\": \"You are a helpful support agent.\",\n \"llm\": {\"model\": \"openai/gpt-4.1\"},\n \"stt\": {\"provider\": \"deepgram\"},\n \"tts\": {\"provider\": \"elevenlabs\"},\n \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n \"agent_url\": \"https://api.example.com/agent\",\n \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n \"benchmark_provider\": \"openrouter\"\n}\n```\n\n**Either type**:\n- `traces.scoring.enabled`: whether the traces you send for this agent are scored by its linked evaluators. On unless you set it to `false`. Turning it back on needs at least one linked evaluator that can score traces\n\n```json\n{\"traces\": {\"scoring\": {\"enabled\": false}}}\n```\n\nEvery request Calibrate makes to your endpoint carries the header\n`X-Calibrate-Eval: 1`. Read it to tell a test run from a real user, for example\nto tag the trace you send back or to skip sending one.\n\nFor `type=agent`, omitted keys inherit managed defaults. Omit `config` entirely to use all defaults. For `type=connection`, `config` is stored as-is and must contain `agent_url`",
74
74
  ),
75
75
  interaction_type: AgentCreateInteractionType$zodSchema.default("conversation")
76
76
  .describe(
package/src/models/agenttestrunlistitem.ts CHANGED
@@ -85,7 +85,7 @@ export const AgentTestRunListItem$zodSchema: z.ZodType<AgentTestRunListItem> = z
85
85
  "The evaluators that judged this run, deduplicated and in display order. A `Tool call` entry is appended when any test in the run was a tool-call test. That entry has no `uuid`, because it is not an evaluator in the library. Empty when the run had no evaluators",
86
86
  ),
87
87
  failed: z.int().nullable().optional().describe(
88
- "Number of test cases that failed",
88
+ "Number of test cases that did not pass, which includes the ones that produced no answer",
89
89
  ),
90
90
  is_public: z.boolean().default(false).describe(
91
91
  "Whether the run is shared publicly",
package/src/models/agentupdate.ts CHANGED
@@ -11,7 +11,7 @@ export type AgentUpdate = {
11
11
 
12
12
  export const AgentUpdate$zodSchema: z.ZodType<AgentUpdate> = z.object({
13
13
  config: z.record(z.string(), z.any()).nullable().optional().describe(
14
- "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n \"system_prompt\": \"You are a helpful support agent.\",\n \"llm\": {\"model\": \"openai/gpt-4.1\"},\n \"stt\": {\"provider\": \"deepgram\"},\n \"tts\": {\"provider\": \"elevenlabs\"},\n \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n \"agent_url\": \"https://api.example.com/agent\",\n \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n \"benchmark_provider\": \"openrouter\"\n}\n```\n\nEvery request Calibrate makes to your endpoint carries the header\n`X-Calibrate-Eval: 1`. Read it to tell a test run from a real user, for example\nto tag the trace you send back or to skip sending one.\n\nReplaces the stored config. Omit to leave unchanged\n\nFor `type=connection`, changing `agent_url` or `agent_headers` resets the connection and benchmark verification flags",
14
+ "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n \"system_prompt\": \"You are a helpful support agent.\",\n \"llm\": {\"model\": \"openai/gpt-4.1\"},\n \"stt\": {\"provider\": \"deepgram\"},\n \"tts\": {\"provider\": \"elevenlabs\"},\n \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n \"agent_url\": \"https://api.example.com/agent\",\n \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n \"benchmark_provider\": \"openrouter\"\n}\n```\n\n**Either type**:\n- `traces.scoring.enabled`: whether the traces you send for this agent are scored by its linked evaluators. On unless you set it to `false`. Turning it back on needs at least one linked evaluator that can score traces\n\n```json\n{\"traces\": {\"scoring\": {\"enabled\": false}}}\n```\n\nEvery request Calibrate makes to your endpoint carries the header\n`X-Calibrate-Eval: 1`. Read it to tell a test run from a real user, for example\nto tag the trace you send back or to skip sending one.\n\nReplaces the stored config. Omit to leave unchanged\n\nFor `type=connection`, changing `agent_url` or `agent_headers` resets the connection and benchmark verification flags",
15
15
  ),
16
16
  name: z.string().nullable().optional().describe(
17
17
  "New agent name. Omit to leave the name unchanged",
package/src/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.ts CHANGED
@@ -62,7 +62,7 @@ export const GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetRequest$zodSchema:
62
62
  "ID of a run to jump to, returning the page that contains it instead of the page at `offset`",
63
63
  ).nullable().optional(),
64
64
  has_failures: z.boolean().describe(
65
- "Filter by whether the run has any failing test case or model. `true` returns only runs with failures (or errors), `false` only clean runs. Omit for both",
65
+ "Filter by whether a test in the run did not pass. `true` returns only runs with a failing test, `false` only runs that got through every test and passed them all. A run that broke, was stopped, or gave up part way with no failing test is in neither: filter by `status` for those. Omit for both",
66
66
  ).nullable().optional(),
67
67
  limit: z.int().describe(
68
68
  "Maximum number of items to return. Omit for no limit (all items)",
package/src/models/modelresult.ts CHANGED
@@ -28,7 +28,7 @@ export const ModelResult$zodSchema: z.ZodType<ModelResult> = z.object({
28
28
  evaluator_summary: z.array(z.record(z.string(), z.any())).nullable()
29
29
  .optional().describe("Aggregate summary for each evaluator for this model"),
30
30
  failed: z.int().nullable().optional().describe(
31
- "Number of test cases that failed",
31
+ "Number of test cases that did not pass, which includes the ones that produced no answer",
32
32
  ),
33
33
  latency_ms: z.record(z.string(), z.any()).nullable().optional().describe(
34
34
  "Aggregated latency in milliseconds, as `{p50, p95, p99, count}`",
package/src/models/modelrunsummary.ts CHANGED
@@ -18,11 +18,12 @@ export type ModelRunSummary = {
18
18
  total_tests?: number | null | undefined;
19
19
  passed?: number | null | undefined;
20
20
  failed?: number | null | undefined;
21
+ unanswered_tests?: number | null | undefined;
21
22
  };
22
23
 
23
24
  export const ModelRunSummary$zodSchema: z.ZodType<ModelRunSummary> = z.object({
24
25
  failed: z.int().nullable().optional().describe(
25
- "Number of test cases that failed for this model",
26
+ "Number of test cases that did not pass for this model, which includes the ones that produced no answer",
26
27
  ),
27
28
  message: z.string().default("").describe(
28
29
  "Status or result message for this model",
@@ -37,6 +38,9 @@ export const ModelRunSummary$zodSchema: z.ZodType<ModelRunSummary> = z.object({
37
38
  total_tests: z.int().nullable().optional().describe(
38
39
  "Total test cases for this model",
39
40
  ),
41
+ unanswered_tests: z.int().nullable().optional().describe(
42
+ "Number of this model's test cases that produced no answer, already counted in `failed`",
43
+ ),
40
44
  }).describe(
41
45
  "Flat summary for one model in a benchmark run-LIST item. The full results\nfor each case of a model live on the benchmark detail endpoint\n(`GET /agent-tests/benchmark/{task_id}`), not here.",
42
46
  );
package/src/models/testrunstatusresponse.ts CHANGED
@@ -52,7 +52,7 @@ export const TestRunStatusResponse$zodSchema: z.ZodType<TestRunStatusResponse> =
52
52
  "The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`",
53
53
  ),
54
54
  failed: z.int().nullable().optional().describe(
55
- "Number of test cases that failed",
55
+ "Number of test cases that did not pass, which includes the ones that produced no answer",
56
56
  ),
57
57
  is_public: z.boolean().default(false).describe(
58
58
  "Whether the run is shared publicly",