@dalmia/calibrate-mcp 0.0.61 → 0.0.63

package/README.md

@@ -61,4 +61,4 @@ Point Cursor at the local build by using `node` with the path to `bin/mcp-server
 ## Resources
 
 - `npx @dalmia/calibrate-mcp start --help` — all flags and transports
-- [Calibrate docs](https://calibrate.artpark.ai/docs) — API reference
+- [Calibrate docs](https://docs.calibrate.artpark.ai) — API reference

package/bin/mcp-server.js

@@ -52802,9 +52802,9 @@ var init_config = __esm(() => {
   SDK_METADATA = {
     language: "typescript",
     openapiDocVersion: "0.1.0",
-    sdkVersion: "0.0.61",
+    sdkVersion: "0.0.63",
     genVersion: "2.915.1",
-    userAgent: "speakeasy-sdk/mcp-typescript 0.0.61 2.915.1 0.1.0 @dalmia/calibrate-mcp"
+    userAgent: "speakeasy-sdk/mcp-typescript 0.0.63 2.915.1 0.1.0 @dalmia/calibrate-mcp"
   };
 });
 
@@ -54263,7 +54263,7 @@ var init_agentcreate = __esm(() => {
     "general"
   ]).describe('What the agent expects in the request body:\n\n- `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{"messages": [...]}`\n- `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{"input": "..."}`');
   AgentCreate$zodSchema = object({
-    config: record(string2(), any()).nullable().optional().describe('Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent\'s instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n  "system_prompt": "You are a helpful support agent.",\n  "llm": {"model": "openai/gpt-4.1"},\n  "stt": {"provider": "deepgram"},\n  "tts": {"provider": "elevenlabs"},\n  "settings": {"agent_speaks_first": true, "max_assistant_turns": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n  "agent_url": "https://api.example.com/agent",\n  "agent_headers": {"Authorization": "Bearer <token>"},\n  "benchmark_provider": "openrouter"\n}\n```\n\nEvery request Calibrate makes to your endpoint carries the header\n`X-Calibrate-Eval: 1`. Read it to tell a test run from a real user, for example\nto tag the trace you send back or to skip sending one.\n\nFor `type=agent`, omitted keys inherit managed defaults. Omit `config` entirely to use all defaults. For `type=connection`, `config` is stored as-is and must contain `agent_url`'),
+    config: record(string2(), any()).nullable().optional().describe('Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent\'s instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n  "system_prompt": "You are a helpful support agent.",\n  "llm": {"model": "openai/gpt-4.1"},\n  "stt": {"provider": "deepgram"},\n  "tts": {"provider": "elevenlabs"},\n  "settings": {"agent_speaks_first": true, "max_assistant_turns": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n  "agent_url": "https://api.example.com/agent",\n  "agent_headers": {"Authorization": "Bearer <token>"},\n  "benchmark_provider": "openrouter"\n}\n```\n\n**Either type**:\n- `traces.scoring.enabled`: whether the traces you send for this agent are scored by its linked evaluators. On unless you set it to `false`. Turning it back on needs at least one linked evaluator that can score traces\n\n```json\n{"traces": {"scoring": {"enabled": false}}}\n```\n\nEvery request Calibrate makes to your endpoint carries the header\n`X-Calibrate-Eval: 1`. Read it to tell a test run from a real user, for example\nto tag the trace you send back or to skip sending one.\n\nFor `type=agent`, omitted keys inherit managed defaults. Omit `config` entirely to use all defaults. For `type=connection`, `config` is stored as-is and must contain `agent_url`'),
     interaction_type: AgentCreateInteractionType$zodSchema.default("conversation").describe('What the agent expects in the request body:\n\n- `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{"messages": [...]}`\n- `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{"input": "..."}`'),
     name: string2().describe("Agent name, unique within the workspace"),
     type: AgentCreateType$zodSchema.default("agent").describe("- `agent`: built inside Calibrate\n- `connection`: your existing agent connected to Calibrate")
@@ -55334,7 +55334,7 @@ var AgentUpdate$zodSchema;
 var init_agentupdate = __esm(() => {
   init_zod();
   AgentUpdate$zodSchema = object({
-    config: record(string2(), any()).nullable().optional().describe('Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent\'s instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n  "system_prompt": "You are a helpful support agent.",\n  "llm": {"model": "openai/gpt-4.1"},\n  "stt": {"provider": "deepgram"},\n  "tts": {"provider": "elevenlabs"},\n  "settings": {"agent_speaks_first": true, "max_assistant_turns": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n  "agent_url": "https://api.example.com/agent",\n  "agent_headers": {"Authorization": "Bearer <token>"},\n  "benchmark_provider": "openrouter"\n}\n```\n\nEvery request Calibrate makes to your endpoint carries the header\n`X-Calibrate-Eval: 1`. Read it to tell a test run from a real user, for example\nto tag the trace you send back or to skip sending one.\n\nReplaces the stored config. Omit to leave unchanged\n\nFor `type=connection`, changing `agent_url` or `agent_headers` resets the connection and benchmark verification flags'),
+    config: record(string2(), any()).nullable().optional().describe('Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent\'s instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n  "system_prompt": "You are a helpful support agent.",\n  "llm": {"model": "openai/gpt-4.1"},\n  "stt": {"provider": "deepgram"},\n  "tts": {"provider": "elevenlabs"},\n  "settings": {"agent_speaks_first": true, "max_assistant_turns": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n  "agent_url": "https://api.example.com/agent",\n  "agent_headers": {"Authorization": "Bearer <token>"},\n  "benchmark_provider": "openrouter"\n}\n```\n\n**Either type**:\n- `traces.scoring.enabled`: whether the traces you send for this agent are scored by its linked evaluators. On unless you set it to `false`. Turning it back on needs at least one linked evaluator that can score traces\n\n```json\n{"traces": {"scoring": {"enabled": false}}}\n```\n\nEvery request Calibrate makes to your endpoint carries the header\n`X-Calibrate-Eval: 1`. Read it to tell a test run from a real user, for example\nto tag the trace you send back or to skip sending one.\n\nReplaces the stored config. Omit to leave unchanged\n\nFor `type=connection`, changing `agent_url` or `agent_headers` resets the connection and benchmark verification flags'),
     name: string2().nullable().optional().describe("New agent name. Omit to leave the name unchanged")
   });
 });
@@ -56023,7 +56023,7 @@ var init_modelresult = __esm(() => {
   ModelResult$zodSchema = object({
     cost: record(string2(), any()).nullable().optional().describe("Aggregated cost as `{mean, min, max, count}` (USD)"),
     evaluator_summary: array(record(string2(), any())).nullable().optional().describe("Aggregate summary for each evaluator for this model"),
-    failed: int().nullable().optional().describe("Number of test cases that failed"),
+    failed: int().nullable().optional().describe("Number of test cases that did not pass, which includes the ones that produced no answer"),
     latency_ms: record(string2(), any()).nullable().optional().describe("Aggregated latency in milliseconds, as `{p50, p95, p99, count}`"),
     message: string2().describe("Status or result message for this model"),
     model: string2().describe("Model name these results are for"),
@@ -56253,7 +56253,7 @@ var init_testrunstatusresponse = __esm(() => {
     error: string2().nullable().optional().describe("Why the run could not be carried out, when it failed before producing any result"),
     evaluator_summary: array(record(string2(), any())).nullable().optional().describe("Totals for each evaluator over the whole run, matching the shape a benchmark reports for each model. Only evaluators that returned a verdict appear"),
     evaluators: array(TestRunEvaluator$zodSchema).nullable().optional().describe("The evaluators used in this run. Each verdict in `judge_results` links to one of these by `evaluator_uuid`"),
-    failed: int().nullable().optional().describe("Number of test cases that failed"),
+    failed: int().nullable().optional().describe("Number of test cases that did not pass, which includes the ones that produced no answer"),
     is_public: boolean2().default(false).describe("Whether the run is shared publicly"),
     latency_ms: record(string2(), any()).nullable().optional().describe("Aggregated response latency in milliseconds, as `{p50, p95, p99, count}`"),
     name: string2().describe("Name of the run. A run nobody has renamed shows its number instead, such as `Run 1` for a test run or `Benchmark 1` for a benchmark"),
@@ -56915,12 +56915,13 @@ var ModelRunSummary$zodSchema;
 var init_modelrunsummary = __esm(() => {
   init_zod();
   ModelRunSummary$zodSchema = object({
-    failed: int().nullable().optional().describe("Number of test cases that failed for this model"),
+    failed: int().nullable().optional().describe("Number of test cases that did not pass for this model, which includes the ones that produced no answer"),
     message: string2().default("").describe("Status or result message for this model"),
     model: string2().describe("Model name these results are for"),
     passed: int().nullable().optional().describe("Number of test cases that passed for this model"),
     success: boolean2().nullable().optional().describe("Whether this model's run succeeded"),
-    total_tests: int().nullable().optional().describe("Total test cases for this model")
+    total_tests: int().nullable().optional().describe("Total test cases for this model"),
+    unanswered_tests: int().nullable().optional().describe("Number of this model's test cases that produced no answer, already counted in `failed`")
   }).describe("Flat summary for one model in a benchmark run-LIST item. The full results\nfor each case of a model live on the benchmark detail endpoint\n(`GET /agent-tests/benchmark/{task_id}`), not here.");
 });
 
@@ -56967,7 +56968,7 @@ var init_agenttestrunlistitem = __esm(() => {
     created_at: string2().describe("When the run was created (ISO 8601 UTC)"),
     error: boolean2().default(false).describe("True if the run failed"),
     evaluators: array(RunListEvaluator$zodSchema).optional().describe("The evaluators that judged this run, deduplicated and in display order. A `Tool call` entry is appended when any test in the run was a tool-call test. That entry has no `uuid`, because it is not an evaluator in the library. Empty when the run had no evaluators"),
-    failed: int().nullable().optional().describe("Number of test cases that failed"),
+    failed: int().nullable().optional().describe("Number of test cases that did not pass, which includes the ones that produced no answer"),
     is_public: boolean2().default(false).describe("Whether the run is shared publicly"),
     latency_ms: record(string2(), any()).nullable().optional().describe("Aggregated latency in milliseconds, as `{p50, p95, p99, count}`"),
     model_results: array(ModelRunSummary$zodSchema).nullable().optional().describe("Flat summary for each model in a benchmark run (fetch the benchmark detail for full results)"),
@@ -57013,7 +57014,7 @@ var init_getagenttestrunsagenttestsagentagentuuidrunsgetop = __esm(() => {
   GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetRequest$zodSchema = object({
     agent_uuid: string2().describe("Agent whose test runs to list"),
     around: string2().describe("ID of a run to jump to, returning the page that contains it instead of the page at `offset`").nullable().optional(),
-    has_failures: boolean2().describe("Filter by whether the run has any failing test case or model. `true` returns only runs with failures (or errors), `false` only clean runs. Omit for both").nullable().optional(),
+    has_failures: boolean2().describe("Filter by whether a test in the run did not pass. `true` returns only runs with a failing test, `false` only runs that got through every test and passed them all. A run that broke, was stopped, or gave up part way with no failing test is in neither: filter by `status` for those. Omit for both").nullable().optional(),
     limit: int().describe("Maximum number of items to return. Omit for no limit (all items)").nullable().optional(),
     offset: int().default(0).describe("Number of items to skip before returning results"),
     status: TaskStatus$zodSchema.nullable().optional().describe("Filter by run status. Omit for all statuses"),
@@ -61605,7 +61606,7 @@ hits its trace limit.
 function createMCPServer(deps) {
   const server = new McpServer({
     name: "CalibrateMcp",
-    version: "0.0.61"
+    version: "0.0.63"
   });
   const getClient = deps.getSDK || (() => new CalibrateMcpCore({
     security: deps.security,
@@ -62894,7 +62895,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
         <h1>Instructions</h1>
         <p>One-click installation for Claude Desktop users</p>
         <div class="instruction-item">
-          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.61/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
+          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.63/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
             \uD83D\uDCE5 Download MCP Bundle
           </a>
         </div>
@@ -65778,7 +65779,7 @@ var routes = buildRouteMap({
 var app = buildApplication(routes, {
   name: "mcp",
   versionInfo: {
-    currentVersion: "0.0.61"
+    currentVersion: "0.0.63"
   }
 });
 run(app, process3.argv.slice(2), buildContext(process3));
@@ -65786,5 +65787,5 @@ export {
   app
 };
 
-//# debugId=92996C3E08F6D0B164756E2164756E21
+//# debugId=DF7D04973BECCF3264756E2164756E21
 //# sourceMappingURL=mcp-server.js.map