@dalmia/calibrate-mcp 0.0.46 → 0.0.48

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/bin/mcp-server.js +2306 -1227
  2. package/bin/mcp-server.js.map +48 -46
  3. package/esm/landing-page.js +1 -1
  4. package/esm/lib/config.d.ts +2 -2
  5. package/esm/lib/config.js +2 -2
  6. package/esm/mcp-server/mcp-server.js +1 -1
  7. package/esm/mcp-server/server.js +1 -1
  8. package/esm/models/agentcreate.js +1 -1
  9. package/esm/models/agentcreate.js.map +1 -1
  10. package/esm/models/agenttestrunlistitem.d.ts +1 -0
  11. package/esm/models/agenttestrunlistitem.d.ts.map +1 -1
  12. package/esm/models/agenttestrunlistitem.js +1 -0
  13. package/esm/models/agenttestrunlistitem.js.map +1 -1
  14. package/esm/models/agentupdate.js +1 -1
  15. package/esm/models/agentupdate.js.map +1 -1
  16. package/esm/models/testcaseresult.d.ts +1 -0
  17. package/esm/models/testcaseresult.d.ts.map +1 -1
  18. package/esm/models/testcaseresult.js +1 -0
  19. package/esm/models/testcaseresult.js.map +1 -1
  20. package/esm/models/testruncasesummary.js +1 -1
  21. package/esm/models/testruncasesummary.js.map +1 -1
  22. package/esm/models/testrunstatusresponse.d.ts +2 -0
  23. package/esm/models/testrunstatusresponse.d.ts.map +1 -1
  24. package/esm/models/testrunstatusresponse.js +2 -0
  25. package/esm/models/testrunstatusresponse.js.map +1 -1
  26. package/esm/models/traceingest.d.ts +1 -0
  27. package/esm/models/traceingest.d.ts.map +1 -1
  28. package/esm/models/traceingest.js +1 -0
  29. package/esm/models/traceingest.js.map +1 -1
  30. package/package.json +1 -1
  31. package/src/landing-page.ts +1 -1
  32. package/src/lib/config.ts +2 -2
  33. package/src/mcp-server/mcp-server.ts +1 -1
  34. package/src/mcp-server/server.ts +1 -1
  35. package/src/models/agentcreate.ts +1 -1
  36. package/src/models/agenttestrunlistitem.ts +4 -0
  37. package/src/models/agentupdate.ts +1 -1
  38. package/src/models/testcaseresult.ts +4 -0
  39. package/src/models/testruncasesummary.ts +1 -1
  40. package/src/models/testrunstatusresponse.ts +8 -0
  41. package/src/models/traceingest.ts +4 -0
package/esm/landing-page.js CHANGED
@@ -915,7 +915,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
915
915
  <h1>Instructions</h1>
916
916
  <p>One-click installation for Claude Desktop users</p>
917
917
  <div class="instruction-item">
918
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.46/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
918
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.48/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
919
919
  📥 Download MCP Bundle
920
920
  </a>
921
921
  </div>
package/esm/lib/config.d.ts CHANGED
@@ -35,8 +35,8 @@ export declare function serverURLFromOptions(options: SDKOptions): URL | null;
35
35
  export declare const SDK_METADATA: {
36
36
  readonly language: "typescript";
37
37
  readonly openapiDocVersion: "0.1.0";
38
- readonly sdkVersion: "0.0.46";
38
+ readonly sdkVersion: "0.0.48";
39
39
  readonly genVersion: "2.915.1";
40
- readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.46 2.915.1 0.1.0 @dalmia/calibrate-mcp";
40
+ readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.48 2.915.1 0.1.0 @dalmia/calibrate-mcp";
41
41
  };
42
42
  //# sourceMappingURL=config.d.ts.map
package/esm/lib/config.js CHANGED
@@ -27,8 +27,8 @@ export function serverURLFromOptions(options) {
27
27
  export const SDK_METADATA = {
28
28
  language: "typescript",
29
29
  openapiDocVersion: "0.1.0",
30
- sdkVersion: "0.0.46",
30
+ sdkVersion: "0.0.48",
31
31
  genVersion: "2.915.1",
32
- userAgent: "speakeasy-sdk/mcp-typescript 0.0.46 2.915.1 0.1.0 @dalmia/calibrate-mcp",
32
+ userAgent: "speakeasy-sdk/mcp-typescript 0.0.48 2.915.1 0.1.0 @dalmia/calibrate-mcp",
33
33
  };
34
34
  //# sourceMappingURL=config.js.map
package/esm/mcp-server/mcp-server.js CHANGED
@@ -18,7 +18,7 @@ const routes = buildRouteMap({
18
18
  export const app = buildApplication(routes, {
19
19
  name: "mcp",
20
20
  versionInfo: {
21
- currentVersion: "0.0.46",
21
+ currentVersion: "0.0.48",
22
22
  },
23
23
  });
24
24
  run(app, process.argv.slice(2), buildContext(process));
package/esm/mcp-server/server.js CHANGED
@@ -50,7 +50,7 @@ import { tool$tracesCreate } from "./tools/tracesCreate.js";
50
50
  export function createMCPServer(deps) {
51
51
  const server = new McpServer({
52
52
  name: "CalibrateMcp",
53
- version: "0.0.46",
53
+ version: "0.0.48",
54
54
  });
55
55
  const getClient = deps.getSDK || (() => new CalibrateMcpCore({
56
56
  security: deps.security,
package/esm/models/agentcreate.js CHANGED
@@ -33,7 +33,7 @@ export const AgentCreateInteractionType$zodSchema = z.enum([
33
33
  "general",
34
34
  ]).describe("What the agent expects in the request body:\n\n- `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{\"messages\": [...]}`\n- `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{\"input\": \"...\"}`");
35
35
  export const AgentCreate$zodSchema = z.object({
36
- config: z.record(z.string(), z.any()).nullable().optional().describe("Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n \"system_prompt\": \"You are a helpful support agent.\",\n \"llm\": {\"model\": \"openai/gpt-4.1\"},\n \"stt\": {\"provider\": \"deepgram\"},\n \"tts\": {\"provider\": \"elevenlabs\"},\n \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n \"agent_url\": \"https://api.example.com/agent\",\n \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n \"benchmark_provider\": \"openrouter\"\n}\n```\n\nFor `type=agent`, omitted keys inherit managed defaults. Omit `config` entirely to use all defaults. For `type=connection`, `config` is stored as-is and must contain `agent_url`"),
36
+ config: z.record(z.string(), z.any()).nullable().optional().describe("Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n \"system_prompt\": \"You are a helpful support agent.\",\n \"llm\": {\"model\": \"openai/gpt-4.1\"},\n \"stt\": {\"provider\": \"deepgram\"},\n \"tts\": {\"provider\": \"elevenlabs\"},\n \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n \"agent_url\": \"https://api.example.com/agent\",\n \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n \"benchmark_provider\": \"openrouter\"\n}\n```\n\nEvery request Calibrate makes to your endpoint carries the header\n`X-Calibrate-Eval: 1`. Read it to tell a test run from a real user, for example\nto tag the trace you send back or to skip sending one.\n\nFor `type=agent`, omitted keys inherit managed defaults. Omit `config` entirely to use all defaults. For `type=connection`, `config` is stored as-is and must contain `agent_url`"),
37
37
  interaction_type: AgentCreateInteractionType$zodSchema.default("conversation")
38
38
  .describe("What the agent expects in the request body:\n\n- `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{\"messages\": [...]}`\n- `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{\"input\": \"...\"}`"),
39
39
  name: z.string().describe("Agent name, unique within the workspace"),
package/esm/models/agentcreate.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agentcreate.js","sourceRoot":"","sources":["../../src/models/agentcreate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,KAAK,EAAE,OAAO;IACd,UAAU,EAAE,YAAY;CAChB,CAAC;AASX,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC,IAAI,CAAC;IAC9C,OAAO;IACP,YAAY;CACb,CAAC,CAAC,QAAQ,CACT,+FAA+F,CAChG,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,YAAY,EAAE,cAAc;IAC5B,OAAO,EAAE,SAAS;CACV,CAAC;AAaX,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,CAAC,IAAI,CAAC;IACzD,cAAc;IACd,SAAS;CACV,CAAC,CAAC,QAAQ,CACT,0TAA0T,CAC3T,CAAC;AASF,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,okDAAokD,CACrkD;IACD,gBAAgB,EAAE,oCAAoC,CAAC,OAAO,CAAC,cAAc,CAAC;SAC3E,QAAQ,CACP,0TAA0T,CAC3T;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IACpE,IAAI,EAAE,yBAAyB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,QAAQ,CACvD,+FAA+F,CAChG;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"agentcreate.js","sourceRoot":"","sources":["../../src/models/agentcreate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,KAAK,EAAE,OAAO;IACd,UAAU,EAAE,YAAY;CAChB,CAAC;AASX,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC,IAAI,CAAC;IAC9C,OAAO;IACP,YAAY;CACb,CAAC,CAAC,QAAQ,CACT,+FAA+F,CAChG,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,YAAY,EAAE,cAAc;IAC5B,OAAO,EAAE,SAAS;CACV,CAAC;AAaX,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,CAAC,IAAI,CAAC;IACzD,cAAc;IACd,SAAS;CACV,CAAC,CAAC,QAAQ,CACT,0TAA0T,CAC3T,CAAC;AASF,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,kxDAAkxD,CACnxD;IACD,gBAAgB,EAAE,oCAAoC,CAAC,OAAO,CAAC,cAAc,CAAC;SAC3E,QAAQ,CACP,0TAA0T,CAC3T;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IACpE,IAAI,EAAE,yBAAyB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,QAAQ,CACvD,+FAA+F,CAChG;CACF,CAAC,CAAC"}
package/esm/models/agenttestrunlistitem.d.ts CHANGED
@@ -49,6 +49,7 @@ export type AgentTestRunListItem = {
49
49
  [k: string]: any;
50
50
  } | null | undefined;
51
51
  model_results?: Array<ModelRunSummary> | null | undefined;
52
+ unanswered_tests?: number | null | undefined;
52
53
  error?: boolean | undefined;
53
54
  is_public?: boolean | undefined;
54
55
  share_token?: string | null | undefined;
package/esm/models/agenttestrunlistitem.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agenttestrunlistitem.d.ts","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,eAAe,EAEhB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EACL,kBAAkB,EAEnB,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB;;;CAG3B,CAAC;AACX;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,UAAU,CAC/C,OAAO,wBAAwB,CAChC,CAAC;AAEF,eAAO,MAAM,kCAAkC;;;EAK9C,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,wBAAwB,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,SAAS,CAAC;IACjD,OAAO,CAAC,EAAE,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,aAAa,CAAC,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC1D,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,8BAA8B,EAAE,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAkDvE,CAAC"}
1
+ {"version":3,"file":"agenttestrunlistitem.d.ts","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,eAAe,EAEhB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EACL,kBAAkB,EAEnB,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB;;;CAG3B,CAAC;AACX;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,UAAU,CAC/C,OAAO,wBAAwB,CAChC,CAAC;AAEF,eAAO,MAAM,kCAAkC;;;EAK9C,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,wBAAwB,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,SAAS,CAAC;IACjD,OAAO,CAAC,EAAE,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,aAAa,CAAC,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC1D,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7C,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,8BAA8B,EAAE,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAqDvE,CAAC"}
package/esm/models/agenttestrunlistitem.js CHANGED
@@ -41,6 +41,7 @@ export const AgentTestRunListItem$zodSchema = z
41
41
  total_tests: z.int().nullable().optional().describe("Total number of test cases"),
42
42
  total_tokens: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated token usage as `{mean, min, max, count}`"),
43
43
  type: AgentTestRunListItemType$zodSchema.describe("What kind of run this is:\n- `llm-unit-test`: a single run of the agent's tests\n- `llm-benchmark`: a multi-model comparison"),
44
+ unanswered_tests: z.int().nullable().optional().describe("Number of test cases that produced no answer because the agent or the judge could not be reached, which makes the pass rate an unfair measure of the agent"),
44
45
  updated_at: z.string().describe("When the run was last updated (ISO 8601 UTC)"),
45
46
  uuid: z.string().describe("Test run job ID"),
46
47
  });
package/esm/models/agenttestrunlistitem.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agenttestrunlistitem.js","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,4BAA4B,GAC7B,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,WAAW,EAAE,eAAe;IAC5B,YAAY,EAAE,eAAe;CACrB,CAAC;AAYX,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,CAAC,IAAI,CAAC;IACvD,eAAe;IACf,eAAe;CAChB,CAAC,CAAC,QAAQ,CACT,8HAA8H,CAC/H,CAAC;AAuBF,MAAM,CAAC,MAAM,8BAA8B,GAAoC,CAAC;KAC7E,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IAC1E,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjE,sQAAsQ,CACvQ;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,iEAAiE,CAClE;IACD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACpE,QAAQ,CACP,8FAA8F,CAC/F;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,gFAAgF,CACjF;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACjE,QAAQ,CACP,mFAAmF,CACpF;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,IAAI,EAAE,kCAAkC,CAAC,QAAQ,CAC/C,8HAA8H,CAC/H;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,8CAA8C,CAC/C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;CAC7C,CAAC,CAAC"}
1
+ {"version":3,"file":"agenttestrunlistitem.js","sourceRoot":"","sources":["../../src/models/agenttestrunlistitem.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,4BAA4B,GAC7B,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,WAAW,EAAE,eAAe;IAC5B,YAAY,EAAE,eAAe;CACrB,CAAC;AAYX,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,CAAC,IAAI,CAAC;IACvD,eAAe;IACf,eAAe;CAChB,CAAC,CAAC,QAAQ,CACT,8HAA8H,CAC/H,CAAC;AAwBF,MAAM,CAAC,MAAM,8BAA8B,GAAoC,CAAC;KAC7E,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IAC1E,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjE,sQAAsQ,CACvQ;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,iEAAiE,CAClE;IACD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACpE,QAAQ,CACP,8FAA8F,CAC/F;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,gFAAgF,CACjF;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACjE,QAAQ,CACP,mFAAmF,CACpF;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,IAAI,EAAE,kCAAkC,CAAC,QAAQ,CAC/C,8HAA8H,CAC/H;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,8CAA8C,CAC/C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;CAC7C,CAAC,CAAC"}
package/esm/models/agentupdate.js CHANGED
@@ -3,7 +3,7 @@
3
3
  */
4
4
  import * as z from "zod";
5
5
  export const AgentUpdate$zodSchema = z.object({
6
- config: z.record(z.string(), z.any()).nullable().optional().describe("Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n \"system_prompt\": \"You are a helpful support agent.\",\n \"llm\": {\"model\": \"openai/gpt-4.1\"},\n \"stt\": {\"provider\": \"deepgram\"},\n \"tts\": {\"provider\": \"elevenlabs\"},\n \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n \"agent_url\": \"https://api.example.com/agent\",\n \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n \"benchmark_provider\": \"openrouter\"\n}\n```\n\nReplaces the stored config. Omit to leave unchanged\n\nFor `type=connection`, changing `agent_url` or `agent_headers` resets the connection and benchmark verification flags"),
6
+ config: z.record(z.string(), z.any()).nullable().optional().describe("Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n \"system_prompt\": \"You are a helpful support agent.\",\n \"llm\": {\"model\": \"openai/gpt-4.1\"},\n \"stt\": {\"provider\": \"deepgram\"},\n \"tts\": {\"provider\": \"elevenlabs\"},\n \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n \"agent_url\": \"https://api.example.com/agent\",\n \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n \"benchmark_provider\": \"openrouter\"\n}\n```\n\nEvery request Calibrate makes to your endpoint carries the header\n`X-Calibrate-Eval: 1`. Read it to tell a test run from a real user, for example\nto tag the trace you send back or to skip sending one.\n\nReplaces the stored config. Omit to leave unchanged\n\nFor `type=connection`, changing `agent_url` or `agent_headers` resets the connection and benchmark verification flags"),
7
7
  name: z.string().nullable().optional().describe("New agent name. Omit to leave the name unchanged"),
8
8
  });
9
9
  //# sourceMappingURL=agentupdate.js.map
package/esm/models/agentupdate.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agentupdate.js","sourceRoot":"","sources":["../../src/models/agentupdate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAOzB,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,+jDAA+jD,CAChkD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,kDAAkD,CACnD;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"agentupdate.js","sourceRoot":"","sources":["../../src/models/agentupdate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAOzB,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,6wDAA6wD,CAC9wD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,kDAAkD,CACnD;CACF,CAAC,CAAC"}
package/esm/models/testcaseresult.d.ts CHANGED
@@ -16,6 +16,7 @@ export type TestCaseResult = {
16
16
  judge_results?: Array<JudgeResult> | null | undefined;
17
17
  latency_ms?: number | null | undefined;
18
18
  cost?: number | null | undefined;
19
+ unanswered?: boolean | undefined;
19
20
  };
20
21
  export declare const TestCaseResult$zodSchema: z.ZodType<TestCaseResult>;
21
22
  //# sourceMappingURL=testcaseresult.d.ts.map
package/esm/models/testcaseresult.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testcaseresult.d.ts","sourceRoot":"","sources":["../../src/models/testcaseresult.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AAEnE,MAAM,MAAM,cAAc,GAAG;IAC3B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACzC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,MAAM,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACtC,MAAM,CAAC,EAAE,UAAU,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,SAAS,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACpD,MAAM,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,aAAa,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,eAAO,MAAM,wBAAwB,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAyB7D,CAAC"}
1
+ {"version":3,"file":"testcaseresult.d.ts","sourceRoot":"","sources":["../../src/models/testcaseresult.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AAEnE,MAAM,MAAM,cAAc,GAAG;IAC3B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACzC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,MAAM,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACtC,MAAM,CAAC,EAAE,UAAU,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,SAAS,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACpD,MAAM,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,aAAa,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,UAAU,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,eAAO,MAAM,wBAAwB,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CA4B7D,CAAC"}
package/esm/models/testcaseresult.js CHANGED
@@ -15,5 +15,6 @@ export const TestCaseResult$zodSchema = z.object({
15
15
  reasoning: z.string().nullable().optional().describe("The judge's reasoning, or the tool-call diff for a tool-call test"),
16
16
  test_case: z.record(z.string(), z.any()).nullable().optional().describe("The test case definition that was run"),
17
17
  test_case_id: z.string().nullable().optional().describe("ID of the test case within the run"),
18
+ unanswered: z.boolean().default(false).describe("Whether this case produced no answer because the agent or the judge could not be reached, in which case `reasoning` carries the error and `passed` is not a verdict on the agent"),
18
19
  });
19
20
  //# sourceMappingURL=testcaseresult.js.map
package/esm/models/testcaseresult.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testcaseresult.js","sourceRoot":"","sources":["../../src/models/testcaseresult.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAenE,MAAM,CAAC,MAAM,wBAAwB,GAA8B,CAAC,CAAC,MAAM,CAAC;IAC1E,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC1E,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,uHAAuH,CACxH;IACD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC1E,gCAAgC,CACjC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnD,qDAAqD,CACtD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACnE,MAAM,EAAE,oBAAoB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACzD,kCAAkC,CACnC;IACD,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC7E,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClD,mEAAmE,CACpE;IACD,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACrE,uCAAuC,CACxC;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACrD,oCAAoC,CACrC;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"testcaseresult.js","sourceRoot":"","sources":["../../src/models/testcaseresult.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAgBnE,MAAM,CAAC,MAAM,wBAAwB,GAA8B,CAAC,CAAC,MAAM,CAAC;IAC1E,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC1E,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,uHAAuH,CACxH;IACD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC1E,gCAAgC,CACjC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnD,qDAAqD,CACtD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACnE,MAAM,EAAE,oBAAoB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACzD,kCAAkC,CACnC;IACD,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC7E,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClD,mEAAmE,CACpE;IACD,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACrE,uCAAuC,CACxC;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACrD,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC7C,kLAAkL,CACnL;CACF,CAAC,CAAC"}
package/esm/models/testruncasesummary.js CHANGED
@@ -5,6 +5,6 @@ import * as z from "zod";
5
5
  export const TestRunCaseSummary$zodSchema = z
6
6
  .object({
7
7
  name: z.string().nullable().optional().describe("Name of the test case"),
8
- passed: z.boolean().nullable().optional().describe("Whether the case passed (null if it errored or is still running)"),
8
+ passed: z.boolean().nullable().optional().describe("Whether the case passed (null while the case is still running)"),
9
9
  }).describe("Flat summary for one test case in the run-LIST endpoints. Carries only\nenough to render a run's pass/fail breakdown and a case name. The full detail\nfor each case (agent output, judge verdicts, reasoning, latency, cost, the\ntest-case definition) lives on the run-DETAIL endpoint\n(`GET /agent-tests/run/{task_id}`).");
10
10
  //# sourceMappingURL=testruncasesummary.js.map
package/esm/models/testruncasesummary.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testruncasesummary.js","sourceRoot":"","sources":["../../src/models/testruncasesummary.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAgBzB,MAAM,CAAC,MAAM,4BAA4B,GAAkC,CAAC;KACzE,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACxE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChD,kEAAkE,CACnE;CACF,CAAC,CAAC,QAAQ,CACT,gUAAgU,CACjU,CAAC"}
1
+ {"version":3,"file":"testruncasesummary.js","sourceRoot":"","sources":["../../src/models/testruncasesummary.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAgBzB,MAAM,CAAC,MAAM,4BAA4B,GAAkC,CAAC;KACzE,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACxE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChD,gEAAgE,CACjE;CACF,CAAC,CAAC,QAAQ,CACT,gUAAgU,CACjU,CAAC"}
package/esm/models/testrunstatusresponse.d.ts CHANGED
@@ -20,6 +20,8 @@ export type TestRunStatusResponse = {
20
20
  } | null | undefined;
21
21
  evaluators?: Array<TestRunEvaluator> | null | undefined;
22
22
  results?: Array<TestCaseResult> | null | undefined;
23
+ unanswered_tests?: number | null | undefined;
24
+ stopped_early?: boolean | undefined;
23
25
  error?: boolean | undefined;
24
26
  is_public?: boolean | undefined;
25
27
  share_token?: string | null | undefined;
package/esm/models/testrunstatusresponse.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testrunstatusresponse.d.ts","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,cAAc,EAA4B,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,OAAO,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnD,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAuCzE,CAAC"}
1
+ {"version":3,"file":"testrunstatusresponse.d.ts","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,cAAc,EAA4B,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,OAAO,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnD,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7C,aAAa,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACpC,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CA6CzE,CAAC"}
package/esm/models/testrunstatusresponse.js CHANGED
@@ -17,9 +17,11 @@ export const TestRunStatusResponse$zodSchema = z.object({
17
17
  results: z.array(TestCaseResult$zodSchema).nullable().optional().describe("Results for each test case"),
18
18
  share_token: z.string().nullable().optional().describe("Token for building the public share URL"),
19
19
  status: TaskStatus$zodSchema,
20
+ stopped_early: z.boolean().default(false).describe("Whether the run stopped before starting every test case, after too many failed in a row"),
20
21
  task_id: z.string().describe("Test run job ID"),
21
22
  test_uuids: z.array(z.string()).nullable().optional().describe("IDs of the tests this run executed, in run order"),
22
23
  total_tests: z.int().nullable().optional().describe("Total number of test cases"),
23
24
  total_tokens: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated token usage as `{mean, min, max, count}`"),
25
+ unanswered_tests: z.int().nullable().optional().describe("Number of test cases that produced no answer because the agent or the judge could not be reached, which makes the pass rate an unfair measure of the agent"),
24
26
  });
25
27
  //# sourceMappingURL=testrunstatusresponse.js.map
package/esm/models/testrunstatusresponse.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testrunstatusresponse.js","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAmB/B,MAAM,CAAC,MAAM,+BAA+B,GAC1C,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,0EAA0E,CAC3E;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,4BAA4B,CAC7B;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,kDAAkD,CACnD;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"testrunstatusresponse.js","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAqB/B,MAAM,CAAC,MAAM,+BAA+B,GAC1C,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,0EAA0E,CAC3E;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,4BAA4B,CAC7B;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,yFAAyF,CAC1F;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,kDAAkD,CACnD;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;IACD,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtD,4JAA4J,CAC7J;CACF,CAAC,CAAC"}
package/esm/models/traceingest.d.ts CHANGED
@@ -14,6 +14,7 @@ export type TraceIngest = {
14
14
  input: string | Array<TraceTurn>;
15
15
  output: TraceOutput;
16
16
  metadata?: Array<TraceMetadataEntry> | null | undefined;
17
+ labels?: Array<string> | undefined;
17
18
  };
18
19
  export declare const TraceIngest$zodSchema: z.ZodType<TraceIngest>;
19
20
  //# sourceMappingURL=traceingest.d.ts.map
package/esm/models/traceingest.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"traceingest.d.ts","sourceRoot":"","sources":["../../src/models/traceingest.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EACL,kBAAkB,EAEnB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,SAAS,EAAuB,MAAM,gBAAgB,CAAC;AAEhE;;GAEG;AACH,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;AAE9C,eAAO,MAAM,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAK5C,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC5C,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;IACjC,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,CAAC,EAAE,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;CACzD,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAqBvD,CAAC"}
1
+ {"version":3,"file":"traceingest.d.ts","sourceRoot":"","sources":["../../src/models/traceingest.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EACL,kBAAkB,EAEnB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,SAAS,EAAuB,MAAM,gBAAgB,CAAC;AAEhE;;GAEG;AACH,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;AAE9C,eAAO,MAAM,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAK5C,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC5C,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;IACjC,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,CAAC,EAAE,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,MAAM,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;CACpC,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAwBvD,CAAC"}
package/esm/models/traceingest.js CHANGED
@@ -16,6 +16,7 @@ export const TraceIngest$zodSchema = z.object({
16
16
  z.string(),
17
17
  z.array(TraceTurn$zodSchema),
18
18
  ]).describe("What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format"),
19
+ labels: z.array(z.string()).optional().describe("Your own tags for this turn, such as an environment or a release. Matched exactly when filtering, so keep the spelling stable. Omit if you have none"),
19
20
  message_id: z.string().nullable().optional().describe("Your own ID for the last user message in `input`, stored for reference only. Omit if you have none"),
20
21
  metadata: z.array(TraceMetadataEntry$zodSchema).nullable().optional()
21
22
  .describe("Key-value pairs stored with the trace. Prefer OTel `gen_ai.*` key names where they fit. Omit if you have none"),
package/esm/models/traceingest.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"traceingest.js","sourceRoot":"","sources":["../../src/models/traceingest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAEL,4BAA4B,GAC7B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAa,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAOhE,MAAM,CAAC,MAAM,eAAe,GAAqB,CAAC,CAAC,KAAK,CAAC;IACvD,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC;CAC7B,CAAC,CAAC,QAAQ,CACT,+MAA+M,CAChN,CAAC;AAWF,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC3B,4EAA4E,CAC7E;IACD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxD,yGAAyG,CAC1G;IACD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC;QACb,CAAC,CAAC,MAAM,EAAE;QACV,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC;KAC7B,CAAC,CAAC,QAAQ,CACT,+MAA+M,CAChN;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnD,oGAAoG,CACrG;IACD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,+GAA+G,CAChH;IACH,MAAM,EAAE,qBAAqB;CAC9B,CAAC,CAAC"}
1
+ {"version":3,"file":"traceingest.js","sourceRoot":"","sources":["../../src/models/traceingest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAEL,4BAA4B,GAC7B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAa,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAOhE,MAAM,CAAC,MAAM,eAAe,GAAqB,CAAC,CAAC,KAAK,CAAC;IACvD,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC;CAC7B,CAAC,CAAC,QAAQ,CACT,+MAA+M,CAChN,CAAC;AAYF,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC3B,4EAA4E,CAC7E;IACD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxD,yGAAyG,CAC1G;IACD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC;QACb,CAAC,CAAC,MAAM,EAAE;QACV,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC;KAC7B,CAAC,CAAC,QAAQ,CACT,+MAA+M,CAChN;IACD,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,sJAAsJ,CACvJ;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnD,oGAAoG,CACrG;IACD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,+GAA+G,CAChH;IACH,MAAM,EAAE,qBAAqB;CAC9B,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dalmia/calibrate-mcp",
3
- "version": "0.0.46",
3
+ "version": "0.0.48",
4
4
  "author": "Speakeasy",
5
5
  "type": "module",
6
6
  "sideEffects": false,
package/src/landing-page.ts CHANGED
@@ -930,7 +930,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
930
930
  <h1>Instructions</h1>
931
931
  <p>One-click installation for Claude Desktop users</p>
932
932
  <div class="instruction-item">
933
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.46/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
933
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.48/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
934
934
  📥 Download MCP Bundle
935
935
  </a>
936
936
  </div>
package/src/lib/config.ts CHANGED
@@ -65,8 +65,8 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
65
65
  export const SDK_METADATA = {
66
66
  language: "typescript",
67
67
  openapiDocVersion: "0.1.0",
68
- sdkVersion: "0.0.46",
68
+ sdkVersion: "0.0.48",
69
69
  genVersion: "2.915.1",
70
70
  userAgent:
71
- "speakeasy-sdk/mcp-typescript 0.0.46 2.915.1 0.1.0 @dalmia/calibrate-mcp",
71
+ "speakeasy-sdk/mcp-typescript 0.0.48 2.915.1 0.1.0 @dalmia/calibrate-mcp",
72
72
  } as const;
package/src/mcp-server/mcp-server.ts CHANGED
@@ -21,7 +21,7 @@ const routes = buildRouteMap({
21
21
  export const app = buildApplication(routes, {
22
22
  name: "mcp",
23
23
  versionInfo: {
24
- currentVersion: "0.0.46",
24
+ currentVersion: "0.0.48",
25
25
  },
26
26
  });
27
27
 
package/src/mcp-server/server.ts CHANGED
@@ -72,7 +72,7 @@ export function createMCPServer(deps: {
72
72
  }) {
73
73
  const server = new McpServer({
74
74
  name: "CalibrateMcp",
75
- version: "0.0.46",
75
+ version: "0.0.48",
76
76
  });
77
77
 
78
78
  const getClient = deps.getSDK || (() =>
package/src/models/agentcreate.ts CHANGED
@@ -70,7 +70,7 @@ export type AgentCreate = {
70
70
 
71
71
  export const AgentCreate$zodSchema: z.ZodType<AgentCreate> = z.object({
72
72
  config: z.record(z.string(), z.any()).nullable().optional().describe(
73
- "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n \"system_prompt\": \"You are a helpful support agent.\",\n \"llm\": {\"model\": \"openai/gpt-4.1\"},\n \"stt\": {\"provider\": \"deepgram\"},\n \"tts\": {\"provider\": \"elevenlabs\"},\n \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n \"agent_url\": \"https://api.example.com/agent\",\n \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n \"benchmark_provider\": \"openrouter\"\n}\n```\n\nFor `type=agent`, omitted keys inherit managed defaults. Omit `config` entirely to use all defaults. For `type=connection`, `config` is stored as-is and must contain `agent_url`",
73
+ "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n \"system_prompt\": \"You are a helpful support agent.\",\n \"llm\": {\"model\": \"openai/gpt-4.1\"},\n \"stt\": {\"provider\": \"deepgram\"},\n \"tts\": {\"provider\": \"elevenlabs\"},\n \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n \"agent_url\": \"https://api.example.com/agent\",\n \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n \"benchmark_provider\": \"openrouter\"\n}\n```\n\nEvery request Calibrate makes to your endpoint carries the header\n`X-Calibrate-Eval: 1`. Read it to tell a test run from a real user, for example\nto tag the trace you send back or to skip sending one.\n\nFor `type=agent`, omitted keys inherit managed defaults. Omit `config` entirely to use all defaults. For `type=connection`, `config` is stored as-is and must contain `agent_url`",
74
74
  ),
75
75
  interaction_type: AgentCreateInteractionType$zodSchema.default("conversation")
76
76
  .describe(
package/src/models/agenttestrunlistitem.ts CHANGED
@@ -63,6 +63,7 @@ export type AgentTestRunListItem = {
63
63
  cost?: { [k: string]: any } | null | undefined;
64
64
  total_tokens?: { [k: string]: any } | null | undefined;
65
65
  model_results?: Array<ModelRunSummary> | null | undefined;
66
+ unanswered_tests?: number | null | undefined;
66
67
  error?: boolean | undefined;
67
68
  is_public?: boolean | undefined;
68
69
  share_token?: string | null | undefined;
@@ -114,6 +115,9 @@ export const AgentTestRunListItem$zodSchema: z.ZodType<AgentTestRunListItem> = z
114
115
  type: AgentTestRunListItemType$zodSchema.describe(
115
116
  "What kind of run this is:\n- `llm-unit-test`: a single run of the agent's tests\n- `llm-benchmark`: a multi-model comparison",
116
117
  ),
118
+ unanswered_tests: z.int().nullable().optional().describe(
119
+ "Number of test cases that produced no answer because the agent or the judge could not be reached, which makes the pass rate an unfair measure of the agent",
120
+ ),
117
121
  updated_at: z.string().describe(
118
122
  "When the run was last updated (ISO 8601 UTC)",
119
123
  ),
package/src/models/agentupdate.ts CHANGED
@@ -11,7 +11,7 @@ export type AgentUpdate = {
11
11
 
12
12
  export const AgentUpdate$zodSchema: z.ZodType<AgentUpdate> = z.object({
13
13
  config: z.record(z.string(), z.any()).nullable().optional().describe(
14
- "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n \"system_prompt\": \"You are a helpful support agent.\",\n \"llm\": {\"model\": \"openai/gpt-4.1\"},\n \"stt\": {\"provider\": \"deepgram\"},\n \"tts\": {\"provider\": \"elevenlabs\"},\n \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n \"agent_url\": \"https://api.example.com/agent\",\n \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n \"benchmark_provider\": \"openrouter\"\n}\n```\n\nReplaces the stored config. Omit to leave unchanged\n\nFor `type=connection`, changing `agent_url` or `agent_headers` resets the connection and benchmark verification flags",
14
+ "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n \"system_prompt\": \"You are a helpful support agent.\",\n \"llm\": {\"model\": \"openai/gpt-4.1\"},\n \"stt\": {\"provider\": \"deepgram\"},\n \"tts\": {\"provider\": \"elevenlabs\"},\n \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n \"agent_url\": \"https://api.example.com/agent\",\n \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n \"benchmark_provider\": \"openrouter\"\n}\n```\n\nEvery request Calibrate makes to your endpoint carries the header\n`X-Calibrate-Eval: 1`. Read it to tell a test run from a real user, for example\nto tag the trace you send back or to skip sending one.\n\nReplaces the stored config. Omit to leave unchanged\n\nFor `type=connection`, changing `agent_url` or `agent_headers` resets the connection and benchmark verification flags",
15
15
  ),
16
16
  name: z.string().nullable().optional().describe(
17
17
  "New agent name. Omit to leave the name unchanged",
package/src/models/testcaseresult.ts CHANGED
@@ -17,6 +17,7 @@ export type TestCaseResult = {
17
17
  judge_results?: Array<JudgeResult> | null | undefined;
18
18
  latency_ms?: number | null | undefined;
19
19
  cost?: number | null | undefined;
20
+ unanswered?: boolean | undefined;
20
21
  };
21
22
 
22
23
  export const TestCaseResult$zodSchema: z.ZodType<TestCaseResult> = z.object({
@@ -44,4 +45,7 @@ export const TestCaseResult$zodSchema: z.ZodType<TestCaseResult> = z.object({
44
45
  test_case_id: z.string().nullable().optional().describe(
45
46
  "ID of the test case within the run",
46
47
  ),
48
+ unanswered: z.boolean().default(false).describe(
49
+ "Whether this case produced no answer because the agent or the judge could not be reached, in which case `reasoning` carries the error and `passed` is not a verdict on the agent",
50
+ ),
47
51
  });
package/src/models/testruncasesummary.ts CHANGED
@@ -22,7 +22,7 @@ export const TestRunCaseSummary$zodSchema: z.ZodType<TestRunCaseSummary> = z
22
22
  .object({
23
23
  name: z.string().nullable().optional().describe("Name of the test case"),
24
24
  passed: z.boolean().nullable().optional().describe(
25
- "Whether the case passed (null if it errored or is still running)",
25
+ "Whether the case passed (null while the case is still running)",
26
26
  ),
27
27
  }).describe(
28
28
  "Flat summary for one test case in the run-LIST endpoints. Carries only\nenough to render a run's pass/fail breakdown and a case name. The full detail\nfor each case (agent output, judge verdicts, reasoning, latency, cost, the\ntest-case definition) lives on the run-DETAIL endpoint\n(`GET /agent-tests/run/{task_id}`).",
package/src/models/testrunstatusresponse.ts CHANGED
@@ -22,6 +22,8 @@ export type TestRunStatusResponse = {
22
22
  total_tokens?: { [k: string]: any } | null | undefined;
23
23
  evaluators?: Array<TestRunEvaluator> | null | undefined;
24
24
  results?: Array<TestCaseResult> | null | undefined;
25
+ unanswered_tests?: number | null | undefined;
26
+ stopped_early?: boolean | undefined;
25
27
  error?: boolean | undefined;
26
28
  is_public?: boolean | undefined;
27
29
  share_token?: string | null | undefined;
@@ -56,6 +58,9 @@ export const TestRunStatusResponse$zodSchema: z.ZodType<TestRunStatusResponse> =
56
58
  "Token for building the public share URL",
57
59
  ),
58
60
  status: TaskStatus$zodSchema,
61
+ stopped_early: z.boolean().default(false).describe(
62
+ "Whether the run stopped before starting every test case, after too many failed in a row",
63
+ ),
59
64
  task_id: z.string().describe("Test run job ID"),
60
65
  test_uuids: z.array(z.string()).nullable().optional().describe(
61
66
  "IDs of the tests this run executed, in run order",
@@ -66,4 +71,7 @@ export const TestRunStatusResponse$zodSchema: z.ZodType<TestRunStatusResponse> =
66
71
  total_tokens: z.record(z.string(), z.any()).nullable().optional().describe(
67
72
  "Aggregated token usage as `{mean, min, max, count}`",
68
73
  ),
74
+ unanswered_tests: z.int().nullable().optional().describe(
75
+ "Number of test cases that produced no answer because the agent or the judge could not be reached, which makes the pass rate an unfair measure of the agent",
76
+ ),
69
77
  });
package/src/models/traceingest.ts CHANGED
@@ -29,6 +29,7 @@ export type TraceIngest = {
29
29
  input: string | Array<TraceTurn>;
30
30
  output: TraceOutput;
31
31
  metadata?: Array<TraceMetadataEntry> | null | undefined;
32
+ labels?: Array<string> | undefined;
32
33
  };
33
34
 
34
35
  export const TraceIngest$zodSchema: z.ZodType<TraceIngest> = z.object({
@@ -44,6 +45,9 @@ export const TraceIngest$zodSchema: z.ZodType<TraceIngest> = z.object({
44
45
  ]).describe(
45
46
  "What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format",
46
47
  ),
48
+ labels: z.array(z.string()).optional().describe(
49
+ "Your own tags for this turn, such as an environment or a release. Matched exactly when filtering, so keep the spelling stable. Omit if you have none",
50
+ ),
47
51
  message_id: z.string().nullable().optional().describe(
48
52
  "Your own ID for the last user message in `input`, stored for reference only. Omit if you have none",
49
53
  ),