@dalmia/calibrate-mcp 0.0.31 → 0.0.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/bin/mcp-server.js +647 -386
  2. package/bin/mcp-server.js.map +16 -16
  3. package/esm/landing-page.js +1 -1
  4. package/esm/lib/config.d.ts +2 -2
  5. package/esm/lib/config.js +2 -2
  6. package/esm/mcp-server/mcp-server.js +1 -1
  7. package/esm/mcp-server/server.js +1 -1
  8. package/esm/models/agentsummary.d.ts +1 -0
  9. package/esm/models/agentsummary.d.ts.map +1 -1
  10. package/esm/models/agentsummary.js +1 -0
  11. package/esm/models/agentsummary.js.map +1 -1
  12. package/esm/models/annotationtaskresponse.d.ts +1 -0
  13. package/esm/models/annotationtaskresponse.d.ts.map +1 -1
  14. package/esm/models/annotationtaskresponse.js +1 -0
  15. package/esm/models/annotationtaskresponse.js.map +1 -1
  16. package/esm/models/benchmarkstatusresponse.d.ts +1 -0
  17. package/esm/models/benchmarkstatusresponse.d.ts.map +1 -1
  18. package/esm/models/benchmarkstatusresponse.js +1 -0
  19. package/esm/models/benchmarkstatusresponse.js.map +1 -1
  20. package/esm/models/bulktestitem.d.ts +3 -0
  21. package/esm/models/bulktestitem.d.ts.map +1 -1
  22. package/esm/models/bulktestitem.js +1 -0
  23. package/esm/models/bulktestitem.js.map +1 -1
  24. package/esm/models/taskagreementresponse.js +1 -1
  25. package/esm/models/taskagreementresponse.js.map +1 -1
  26. package/esm/models/testcaseresult.d.ts +3 -0
  27. package/esm/models/testcaseresult.d.ts.map +1 -1
  28. package/esm/models/testcaseresult.js +1 -0
  29. package/esm/models/testcaseresult.js.map +1 -1
  30. package/esm/models/testrunstatusresponse.d.ts +1 -0
  31. package/esm/models/testrunstatusresponse.d.ts.map +1 -1
  32. package/esm/models/testrunstatusresponse.js +1 -0
  33. package/esm/models/testrunstatusresponse.js.map +1 -1
  34. package/package.json +1 -1
  35. package/src/landing-page.ts +1 -1
  36. package/src/lib/config.ts +2 -2
  37. package/src/mcp-server/mcp-server.ts +1 -1
  38. package/src/mcp-server/server.ts +1 -1
  39. package/src/models/agentsummary.ts +4 -0
  40. package/src/models/annotationtaskresponse.ts +4 -0
  41. package/src/models/benchmarkstatusresponse.ts +4 -0
  42. package/src/models/bulktestitem.ts +4 -0
  43. package/src/models/taskagreementresponse.ts +1 -1
  44. package/src/models/testcaseresult.ts +4 -0
  45. package/src/models/testrunstatusresponse.ts +4 -0
package/esm/landing-page.js CHANGED
@@ -915,7 +915,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
915
915
  <h1>Instructions</h1>
916
916
  <p>One-click installation for Claude Desktop users</p>
917
917
  <div class="instruction-item">
918
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.31/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
918
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.32/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
919
919
  📥 Download MCP Bundle
920
920
  </a>
921
921
  </div>
package/esm/lib/config.d.ts CHANGED
@@ -35,8 +35,8 @@ export declare function serverURLFromOptions(options: SDKOptions): URL | null;
35
35
  export declare const SDK_METADATA: {
36
36
  readonly language: "typescript";
37
37
  readonly openapiDocVersion: "0.1.0";
38
- readonly sdkVersion: "0.0.31";
38
+ readonly sdkVersion: "0.0.32";
39
39
  readonly genVersion: "2.915.1";
40
- readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.31 2.915.1 0.1.0 @dalmia/calibrate-mcp";
40
+ readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.32 2.915.1 0.1.0 @dalmia/calibrate-mcp";
41
41
  };
42
42
  //# sourceMappingURL=config.d.ts.map
package/esm/lib/config.js CHANGED
@@ -27,8 +27,8 @@ export function serverURLFromOptions(options) {
27
27
  export const SDK_METADATA = {
28
28
  language: "typescript",
29
29
  openapiDocVersion: "0.1.0",
30
- sdkVersion: "0.0.31",
30
+ sdkVersion: "0.0.32",
31
31
  genVersion: "2.915.1",
32
- userAgent: "speakeasy-sdk/mcp-typescript 0.0.31 2.915.1 0.1.0 @dalmia/calibrate-mcp",
32
+ userAgent: "speakeasy-sdk/mcp-typescript 0.0.32 2.915.1 0.1.0 @dalmia/calibrate-mcp",
33
33
  };
34
34
  //# sourceMappingURL=config.js.map
package/esm/mcp-server/mcp-server.js CHANGED
@@ -18,7 +18,7 @@ const routes = buildRouteMap({
18
18
  export const app = buildApplication(routes, {
19
19
  name: "mcp",
20
20
  versionInfo: {
21
- currentVersion: "0.0.31",
21
+ currentVersion: "0.0.32",
22
22
  },
23
23
  });
24
24
  run(app, process.argv.slice(2), buildContext(process));
package/esm/mcp-server/server.js CHANGED
@@ -44,7 +44,7 @@ import { tool$testsUpdate } from "./tools/testsUpdate.js";
44
44
  export function createMCPServer(deps) {
45
45
  const server = new McpServer({
46
46
  name: "CalibrateMcp",
47
- version: "0.0.31",
47
+ version: "0.0.32",
48
48
  });
49
49
  const getClient = deps.getSDK || (() => new CalibrateMcpCore({
50
50
  security: deps.security,
package/esm/models/agentsummary.d.ts CHANGED
@@ -27,6 +27,7 @@ export type AgentSummary = {
27
27
  type: AgentSummaryType;
28
28
  updated_at: string;
29
29
  connection_verified?: boolean | null | undefined;
30
+ has_default_inputs: boolean;
30
31
  };
31
32
  export declare const AgentSummary$zodSchema: z.ZodType<AgentSummary>;
32
33
  //# sourceMappingURL=agentsummary.d.ts.map
package/esm/models/agentsummary.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agentsummary.d.ts","sourceRoot":"","sources":["../../src/models/agentsummary.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB;;;CAGnB,CAAC;AACX;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAEnE,eAAO,MAAM,0BAA0B;;;EAKtC,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,gBAAgB,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;CAClD,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAYzD,CAAC"}
1
+ {"version":3,"file":"agentsummary.d.ts","sourceRoot":"","sources":["../../src/models/agentsummary.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB;;;CAGnB,CAAC;AACX;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAEnE,eAAO,MAAM,0BAA0B;;;EAKtC,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,gBAAgB,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,kBAAkB,EAAE,OAAO,CAAC;CAC7B,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAezD,CAAC"}
package/esm/models/agentsummary.js CHANGED
@@ -18,6 +18,7 @@ export const AgentSummaryType$zodSchema = z.enum([
18
18
  ]).describe("- `agent`: built inside Calibrate\n- `connection`: your existing agent connected to Calibrate");
19
19
  export const AgentSummary$zodSchema = z.object({
20
20
  connection_verified: z.boolean().nullable().optional().describe("Whether the agent's connection has been verified, for a `type=connection` agent"),
21
+ has_default_inputs: z.boolean().describe("Whether the agent has custom request fields configured"),
21
22
  name: z.string().describe("Name of the agent"),
22
23
  type: AgentSummaryType$zodSchema.describe("- `agent`: built inside Calibrate\n- `connection`: your existing agent connected to Calibrate"),
23
24
  updated_at: z.string().describe("When the agent was last updated (ISO 8601 UTC)"),
package/esm/models/agentsummary.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agentsummary.js","sourceRoot":"","sources":["../../src/models/agentsummary.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,KAAK,EAAE,OAAO;IACd,UAAU,EAAE,YAAY;CAChB,CAAC;AASX,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,IAAI,CAAC;IAC/C,OAAO;IACP,YAAY;CACb,CAAC,CAAC,QAAQ,CACT,+FAA+F,CAChG,CAAC;AAUF,MAAM,CAAC,MAAM,sBAAsB,GAA4B,CAAC,CAAC,MAAM,CAAC;IACtE,mBAAmB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7D,iFAAiF,CAClF;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IAC9C,IAAI,EAAE,0BAA0B,CAAC,QAAQ,CACvC,+FAA+F,CAChG;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,gDAAgD,CACjD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;CAC7C,CAAC,CAAC"}
1
+ {"version":3,"file":"agentsummary.js","sourceRoot":"","sources":["../../src/models/agentsummary.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,KAAK,EAAE,OAAO;IACd,UAAU,EAAE,YAAY;CAChB,CAAC;AASX,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,IAAI,CAAC;IAC/C,OAAO;IACP,YAAY;CACb,CAAC,CAAC,QAAQ,CACT,+FAA+F,CAChG,CAAC;AAWF,MAAM,CAAC,MAAM,sBAAsB,GAA4B,CAAC,CAAC,MAAM,CAAC;IACtE,mBAAmB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7D,iFAAiF,CAClF;IACD,kBAAkB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CACtC,wDAAwD,CACzD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IAC9C,IAAI,EAAE,0BAA0B,CAAC,QAAQ,CACvC,+FAA+F,CAChG;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,gDAAgD,CACjD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;CAC7C,CAAC,CAAC"}
package/esm/models/annotationtaskresponse.d.ts CHANGED
@@ -44,6 +44,7 @@ export type AnnotationTaskResponse = {
44
44
  [k: string]: any;
45
45
  }> | undefined;
46
46
  item_count?: number | null | undefined;
47
+ has_agreement?: boolean | undefined;
47
48
  items?: Array<{
48
49
  [k: string]: any;
49
50
  }> | undefined;
package/esm/models/annotationtaskresponse.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"annotationtaskresponse.d.ts","sourceRoot":"","sources":["../../src/models/annotationtaskresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C;;;;;;;;GAQG;AACH,eAAO,MAAM,0BAA0B;;;;;;CAM7B,CAAC;AACX;;;;;;;;GAQG;AACH,MAAM,MAAM,0BAA0B,GAAG,UAAU,CACjD,OAAO,0BAA0B,CAClC,CAAC;AAEF,eAAO,MAAM,oCAAoC;;;;;;EAQhD,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,0BAA0B,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,SAAS,CAAC;IACrD,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,KAAK,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,SAAS,CAAC;IAChD,IAAI,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,SAAS,CAAC;CAChD,CAAC;AAEF,eAAO,MAAM,gCAAgC,EAAE,CAAC,CAAC,OAAO,CACtD,sBAAsB,CA0BtB,CAAC"}
1
+ {"version":3,"file":"annotationtaskresponse.d.ts","sourceRoot":"","sources":["../../src/models/annotationtaskresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C;;;;;;;;GAQG;AACH,eAAO,MAAM,0BAA0B;;;;;;CAM7B,CAAC;AACX;;;;;;;;GAQG;AACH,MAAM,MAAM,0BAA0B,GAAG,UAAU,CACjD,OAAO,0BAA0B,CAClC,CAAC;AAEF,eAAO,MAAM,oCAAoC;;;;;;EAQhD,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,0BAA0B,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,SAAS,CAAC;IACrD,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,aAAa,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACpC,KAAK,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,SAAS,CAAC;IAChD,IAAI,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,SAAS,CAAC;CAChD,CAAC;AAEF,eAAO,MAAM,gCAAgC,EAAE,CAAC,CAAC,OAAO,CACtD,sBAAsB,CA6BtB,CAAC"}
package/esm/models/annotationtaskresponse.js CHANGED
@@ -29,6 +29,7 @@ export const AnnotationTaskResponse$zodSchema = z.object({
29
29
  created_at: z.string().describe("When the task was created (ISO 8601 UTC)"),
30
30
  description: z.string().nullable().optional().describe("The task's description"),
31
31
  evaluators: z.array(z.record(z.string(), z.any())).optional().describe("The evaluators linked to this task, in display order. Fetching one task by ID enriches each with its live version's rubric. Listing tasks returns lightweight metadata (uuid, name, type, position) only"),
32
+ has_agreement: z.boolean().default(false).describe("Whether the task has at least one comparable human-vs-human or human-vs-evaluator pair, computed over all time"),
32
33
  item_count: z.int().nullable().optional().describe("Number of items in the task"),
33
34
  items: z.array(z.record(z.string(), z.any())).optional().describe("The task's items, each with its agreement stats. You get these when you fetch one task by ID, not when you list tasks"),
34
35
  jobs: z.array(z.record(z.string(), z.any())).optional().describe("The task's labelling jobs. You get these when you fetch one task by ID, not when you list tasks"),
package/esm/models/annotationtaskresponse.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"annotationtaskresponse.js","sourceRoot":"","sources":["../../src/models/annotationtaskresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;IAC5B,GAAG,EAAE,KAAK;CACF,CAAC;AAcX,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,CAAC,IAAI,CAAC;IACzD,KAAK;IACL,KAAK;IACL,aAAa;IACb,cAAc;IACd,KAAK;CACN,CAAC,CAAC,QAAQ,CACT,qQAAqQ,CACtQ,CAAC;AAeF,MAAM,CAAC,MAAM,gCAAgC,GAEzC,CAAC,CAAC,MAAM,CAAC;IACX,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IAC3E,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,wBAAwB,CACzB;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpE,0MAA0M,CAC3M;IACD,UAAU,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChD,6BAA6B,CAC9B;IACD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC/D,uHAAuH,CACxH;IACD,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC9D,iGAAiG,CAClG;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IAC7C,IAAI,EAAE,oCAAoC,CAAC,QAAQ,CACjD,qQAAqQ,CACtQ;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,+CAA+C,CAChD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,wBAAwB,CAAC;CACpD,CAAC,CAAC"}
1
+ {"version":3,"file":"annotationtaskresponse.js","sourceRoot":"","sources":["../../src/models/annotationtaskresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,aAAa;IACzB,YAAY,EAAE,cAAc;IAC5B,GAAG,EAAE,KAAK;CACF,CAAC;AAcX,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,CAAC,IAAI,CAAC;IACzD,KAAK;IACL,KAAK;IACL,aAAa;IACb,cAAc;IACd,KAAK;CACN,CAAC,CAAC,QAAQ,CACT,qQAAqQ,CACtQ,CAAC;AAgBF,MAAM,CAAC,MAAM,gCAAgC,GAEzC,CAAC,CAAC,MAAM,CAAC;IACX,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IAC3E,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,wBAAwB,CACzB;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpE,0MAA0M,CAC3M;IACD,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAChD,gHAAgH,CACjH;IACD,UAAU,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChD,6BAA6B,CAC9B;IACD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC/D,uHAAuH,CACxH;IACD,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC9D,iGAAiG,CAClG;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IAC7C,IAAI,EAAE,oCAAoC,CAAC,QAAQ,CACjD,qQAAqQ,CACtQ;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,+CAA+C,CAChD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,wBAAwB,CAAC;CACpD,CAAC,CAAC"}
package/esm/models/benchmarkstatusresponse.d.ts CHANGED
@@ -5,6 +5,7 @@ import { TestRunEvaluator } from "./testrunevaluator.js";
5
5
  export type BenchmarkStatusResponse = {
6
6
  task_id: string;
7
7
  status: TaskStatus;
8
+ test_uuids?: Array<string> | null | undefined;
8
9
  evaluators?: Array<TestRunEvaluator> | null | undefined;
9
10
  model_results?: Array<ModelResult> | null | undefined;
10
11
  leaderboard_summary?: Array<{
package/esm/models/benchmarkstatusresponse.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"benchmarkstatusresponse.d.ts","sourceRoot":"","sources":["../../src/models/benchmarkstatusresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,uBAAuB,GAAG;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,aAAa,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,mBAAmB,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACrE,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,iCAAiC,EAAE,CAAC,CAAC,OAAO,CACvD,uBAAuB,CAsBvB,CAAC"}
1
+ {"version":3,"file":"benchmarkstatusresponse.d.ts","sourceRoot":"","sources":["../../src/models/benchmarkstatusresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,uBAAuB,GAAG;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,aAAa,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,mBAAmB,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACrE,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,iCAAiC,EAAE,CAAC,CAAC,OAAO,CACvD,uBAAuB,CAyBvB,CAAC"}
package/esm/models/benchmarkstatusresponse.js CHANGED
@@ -16,5 +16,6 @@ export const BenchmarkStatusResponse$zodSchema = z.object({
16
16
  share_token: z.string().nullable().optional().describe("Token for building the public share URL"),
17
17
  status: TaskStatus$zodSchema,
18
18
  task_id: z.string().describe("Benchmark run job ID"),
19
+ test_uuids: z.array(z.string()).nullable().optional().describe("IDs of the tests this benchmark executed, in run order"),
19
20
  });
20
21
  //# sourceMappingURL=benchmarkstatusresponse.js.map
package/esm/models/benchmarkstatusresponse.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"benchmarkstatusresponse.js","sourceRoot":"","sources":["../../src/models/benchmarkstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAa/B,MAAM,CAAC,MAAM,iCAAiC,GAE1C,CAAC,CAAC,MAAM,CAAC;IACX,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,mBAAmB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;SACnE,QAAQ,EAAE,CAAC,QAAQ,CAClB,oMAAoM,CACrM;IACH,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC1E,wBAAwB,CACzB;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;CACrD,CAAC,CAAC"}
1
+ {"version":3,"file":"benchmarkstatusresponse.js","sourceRoot":"","sources":["../../src/models/benchmarkstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAc/B,MAAM,CAAC,MAAM,iCAAiC,GAE1C,CAAC,CAAC,MAAM,CAAC;IACX,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,mBAAmB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;SACnE,QAAQ,EAAE,CAAC,QAAQ,CAClB,oMAAoM,CACrM;IACH,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC1E,wBAAwB,CACzB;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;IACpD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,wDAAwD,CACzD;CACF,CAAC,CAAC"}
package/esm/models/bulktestitem.d.ts CHANGED
@@ -7,6 +7,9 @@ export type BulkTestItem = {
7
7
  conversation_history: Array<ChatMessage>;
8
8
  evaluators?: Array<RoutersTestsEvaluatorRef> | null | undefined;
9
9
  tool_calls?: Array<ExpectedToolCall> | null | undefined;
10
+ inputs?: {
11
+ [k: string]: any;
12
+ } | null | undefined;
10
13
  };
11
14
  export declare const BulkTestItem$zodSchema: z.ZodType<BulkTestItem>;
12
15
  //# sourceMappingURL=bulktestitem.d.ts.map
package/esm/models/bulktestitem.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"bulktestitem.d.ts","sourceRoot":"","sources":["../../src/models/bulktestitem.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,wBAAwB,EAEzB,MAAM,+BAA+B,CAAC;AAEvC,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IACzC,UAAU,CAAC,EAAE,KAAK,CAAC,wBAAwB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAChE,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;CACzD,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAazD,CAAC"}
1
+ {"version":3,"file":"bulktestitem.d.ts","sourceRoot":"","sources":["../../src/models/bulktestitem.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,wBAAwB,EAEzB,MAAM,+BAA+B,CAAC;AAEvC,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IACzC,UAAU,CAAC,EAAE,KAAK,CAAC,wBAAwB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAChE,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,MAAM,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;CAClD,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAgBzD,CAAC"}
package/esm/models/bulktestitem.js CHANGED
@@ -9,6 +9,7 @@ export const BulkTestItem$zodSchema = z.object({
9
9
  conversation_history: z.array(ChatMessage$zodSchema).describe("Ordered messages ending at the user turn the agent should answer"),
10
10
  evaluators: z.array(RoutersTestsEvaluatorRef$zodSchema).nullable().optional()
11
11
  .describe("Evaluators to link. Used by `response` and `conversation` tests"),
12
+ inputs: z.record(z.string(), z.any()).nullable().optional().describe("Extra request fields for this test, overriding the agent's `default_inputs` per key"),
12
13
  name: z.string().describe("Name of the test, unique within the workspace and within the batch"),
13
14
  tool_calls: z.array(ExpectedToolCall$zodSchema).nullable().optional()
14
15
  .describe("Expected tool calls. **Required for `tool_call` batches**"),
package/esm/models/bulktestitem.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"bulktestitem.js","sourceRoot":"","sources":["../../src/models/bulktestitem.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAEL,kCAAkC,GACnC,MAAM,+BAA+B,CAAC;AASvC,MAAM,CAAC,MAAM,sBAAsB,GAA4B,CAAC,CAAC,MAAM,CAAC;IACtE,oBAAoB,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,QAAQ,CAC3D,kEAAkE,CACnE;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAC1E,QAAQ,CACP,iEAAiE,CAClE;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,oEAAoE,CACrE;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CAAC,2DAA2D,CAAC;CACzE,CAAC,CAAC"}
1
+ {"version":3,"file":"bulktestitem.js","sourceRoot":"","sources":["../../src/models/bulktestitem.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAEL,kCAAkC,GACnC,MAAM,+BAA+B,CAAC;AAUvC,MAAM,CAAC,MAAM,sBAAsB,GAA4B,CAAC,CAAC,MAAM,CAAC;IACtE,oBAAoB,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,QAAQ,CAC3D,kEAAkE,CACnE;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAC1E,QAAQ,CACP,iEAAiE,CAClE;IACH,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,qFAAqF,CACtF;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,oEAAoE,CACrE;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CAAC,2DAA2D,CAAC;CACzE,CAAC,CAAC"}
package/esm/models/taskagreementresponse.js CHANGED
@@ -7,7 +7,7 @@ import { HumanAgreementBlock$zodSchema, HumanAgreementBlock$zodSchemaOutbound, }
7
7
  export const TaskAgreementResponse$zodSchema = collectExtraKeys$(z.object({
8
8
  bucket: z.string().describe("Time bucket used for the trend series"),
9
9
  days: z.int().describe("Length of the trailing window in days"),
10
- evaluators: z.array(z.record(z.string(), z.any())).describe("Agreement between each evaluator and the annotators, one entry per linked evaluator"),
10
+ evaluators: z.array(z.record(z.string(), z.any())).describe("Agreement between each evaluator and the annotators, plus that evaluator's own results, one entry per linked evaluator"),
11
11
  human_human: HumanAgreementBlock$zodSchema,
12
12
  task_id: z.string().describe("ID of the task"),
13
13
  }).catchall(z.any()), "AdditionalProperties", true);
package/esm/models/taskagreementresponse.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"taskagreementresponse.js","sourceRoot":"","sources":["../../src/models/taskagreementresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,gBAAgB,IAAI,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC1E,OAAO,EAEL,6BAA6B,EAC7B,qCAAqC,GACtC,MAAM,0BAA0B,CAAC;AAWlC,MAAM,CAAC,MAAM,+BAA+B,GAC1C,iBAAiB,CACf,CAAC,CAAC,MAAM,CAAC;IACP,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;IACpE,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;IAC/D,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CACzD,qFAAqF,CACtF;IACD,WAAW,EAAE,6BAA6B;IAC1C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;CAC/C,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,EACpB,sBAAsB,EACtB,IAAI,CACL,CAAC;AAEJ,MAAM,CAAC,MAAM,uCAAuC,GAAmB,CAAC,CAAC,MAAM,CAC7E;IACE,oBAAoB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC9D,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE;IACb,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;IAClD,WAAW,EAAE,qCAAqC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;CACpB,CACF,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE;IAChB,MAAM,EAAE,oBAAoB,EAAE,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC;IAC5C,OAAO;QACL,GAAG,IAAI;QACP,GAAG,oBAAoB;KACxB,CAAC;AACJ,CAAC,CAAC,CAAC"}
1
+ {"version":3,"file":"taskagreementresponse.js","sourceRoot":"","sources":["../../src/models/taskagreementresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,gBAAgB,IAAI,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC1E,OAAO,EAEL,6BAA6B,EAC7B,qCAAqC,GACtC,MAAM,0BAA0B,CAAC;AAWlC,MAAM,CAAC,MAAM,+BAA+B,GAC1C,iBAAiB,CACf,CAAC,CAAC,MAAM,CAAC;IACP,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;IACpE,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;IAC/D,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CACzD,wHAAwH,CACzH;IACD,WAAW,EAAE,6BAA6B;IAC1C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;CAC/C,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,EACpB,sBAAsB,EACtB,IAAI,CACL,CAAC;AAEJ,MAAM,CAAC,MAAM,uCAAuC,GAAmB,CAAC,CAAC,MAAM,CAC7E;IACE,oBAAoB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC9D,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE;IACb,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;IAClD,WAAW,EAAE,qCAAqC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;CACpB,CACF,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE;IAChB,MAAM,EAAE,oBAAoB,EAAE,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC;IAC5C,OAAO;QACL,GAAG,IAAI;QACP,GAAG,oBAAoB;KACxB,CAAC;AACJ,CAAC,CAAC,CAAC"}
package/esm/models/testcaseresult.d.ts CHANGED
@@ -10,6 +10,9 @@ export type TestCaseResult = {
10
10
  test_case?: {
11
11
  [k: string]: any;
12
12
  } | null | undefined;
13
+ inputs?: {
14
+ [k: string]: any;
15
+ } | null | undefined;
13
16
  judge_results?: Array<JudgeResult> | null | undefined;
14
17
  latency_ms?: number | null | undefined;
15
18
  cost?: number | null | undefined;
package/esm/models/testcaseresult.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testcaseresult.d.ts","sourceRoot":"","sources":["../../src/models/testcaseresult.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AAEnE,MAAM,MAAM,cAAc,GAAG;IAC3B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACzC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,MAAM,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACtC,MAAM,CAAC,EAAE,UAAU,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,SAAS,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACpD,aAAa,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,eAAO,MAAM,wBAAwB,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAsB7D,CAAC"}
1
+ {"version":3,"file":"testcaseresult.d.ts","sourceRoot":"","sources":["../../src/models/testcaseresult.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AAEnE,MAAM,MAAM,cAAc,GAAG;IAC3B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACzC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,MAAM,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACtC,MAAM,CAAC,EAAE,UAAU,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,SAAS,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACpD,MAAM,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,aAAa,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,eAAO,MAAM,wBAAwB,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAyB7D,CAAC"}
package/esm/models/testcaseresult.js CHANGED
@@ -6,6 +6,7 @@ import { JudgeResult$zodSchema } from "./judgeresult.js";
6
6
  import { TestOutput$zodSchema } from "./testoutput.js";
7
7
  export const TestCaseResult$zodSchema = z.object({
8
8
  cost: z.number().nullable().optional().describe("Cost of this case (USD)"),
9
+ inputs: z.record(z.string(), z.any()).nullable().optional().describe("Extra request fields sent to the agent for this case, the agent's `default_inputs` with this case's overrides applied"),
9
10
  judge_results: z.array(JudgeResult$zodSchema).nullable().optional().describe("One verdict for each evaluator"),
10
11
  latency_ms: z.number().nullable().optional().describe("How long the agent took to respond, in milliseconds"),
11
12
  name: z.string().nullable().optional().describe("Name of the test"),
package/esm/models/testcaseresult.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testcaseresult.js","sourceRoot":"","sources":["../../src/models/testcaseresult.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAcnE,MAAM,CAAC,MAAM,wBAAwB,GAA8B,CAAC,CAAC,MAAM,CAAC;IAC1E,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC1E,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC1E,gCAAgC,CACjC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnD,qDAAqD,CACtD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACnE,MAAM,EAAE,oBAAoB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACzD,kCAAkC,CACnC;IACD,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC7E,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClD,mEAAmE,CACpE;IACD,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACrE,uCAAuC,CACxC;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACrD,oCAAoC,CACrC;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"testcaseresult.js","sourceRoot":"","sources":["../../src/models/testcaseresult.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAenE,MAAM,CAAC,MAAM,wBAAwB,GAA8B,CAAC,CAAC,MAAM,CAAC;IAC1E,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC1E,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,uHAAuH,CACxH;IACD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC1E,gCAAgC,CACjC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnD,qDAAqD,CACtD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACnE,MAAM,EAAE,oBAAoB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACzD,kCAAkC,CACnC;IACD,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC7E,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClD,mEAAmE,CACpE;IACD,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACrE,uCAAuC,CACxC;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACrD,oCAAoC,CACrC;CACF,CAAC,CAAC"}
package/esm/models/testrunstatusresponse.d.ts CHANGED
@@ -5,6 +5,7 @@ import { TestRunEvaluator } from "./testrunevaluator.js";
5
5
  export type TestRunStatusResponse = {
6
6
  task_id: string;
7
7
  status: TaskStatus;
8
+ test_uuids?: Array<string> | null | undefined;
8
9
  total_tests?: number | null | undefined;
9
10
  passed?: number | null | undefined;
10
11
  failed?: number | null | undefined;
package/esm/models/testrunstatusresponse.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testrunstatusresponse.d.ts","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,cAAc,EAA4B,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,UAAU,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,OAAO,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnD,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAoCzE,CAAC"}
1
+ {"version":3,"file":"testrunstatusresponse.d.ts","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAwB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,cAAc,EAA4B,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EACL,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,qBAAqB,GAAG;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,OAAO,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACnD,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAuCzE,CAAC"}
package/esm/models/testrunstatusresponse.js CHANGED
@@ -18,6 +18,7 @@ export const TestRunStatusResponse$zodSchema = z.object({
18
18
  share_token: z.string().nullable().optional().describe("Token for building the public share URL"),
19
19
  status: TaskStatus$zodSchema,
20
20
  task_id: z.string().describe("Test run job ID"),
21
+ test_uuids: z.array(z.string()).nullable().optional().describe("IDs of the tests this run executed, in run order"),
21
22
  total_tests: z.int().nullable().optional().describe("Total number of test cases"),
22
23
  total_tokens: z.record(z.string(), z.any()).nullable().optional().describe("Aggregated token usage as `{mean, min, max, count}`"),
23
24
  });
package/esm/models/testrunstatusresponse.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"testrunstatusresponse.js","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAkB/B,MAAM,CAAC,MAAM,+BAA+B,GAC1C,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,0EAA0E,CAC3E;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,4BAA4B,CAC7B;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/C,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"testrunstatusresponse.js","sourceRoot":"","sources":["../../src/models/testrunstatusresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAc,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAkB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAEL,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAmB/B,MAAM,CAAC,MAAM,+BAA+B,GAC1C,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChE,oDAAoD,CACrD;IACD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACpE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,4GAA4G,CAC7G;IACH,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAC5C,oCAAoC,CACrC;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,0EAA0E,CAC3E;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5C,kCAAkC,CACnC;IACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,4BAA4B,CAC7B;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,yCAAyC,CAC1C;IACD,MAAM,EAAE,oBAAoB;IAC5B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC5D,kDAAkD,CACnD;IACD,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjD,4BAA4B,CAC7B;IACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxE,qDAAqD,CACtD;CACF,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dalmia/calibrate-mcp",
3
- "version": "0.0.31",
3
+ "version": "0.0.32",
4
4
  "author": "Speakeasy",
5
5
  "type": "module",
6
6
  "sideEffects": false,
package/src/landing-page.ts CHANGED
@@ -930,7 +930,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
930
930
  <h1>Instructions</h1>
931
931
  <p>One-click installation for Claude Desktop users</p>
932
932
  <div class="instruction-item">
933
- <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.31/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
933
+ <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.32/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
934
934
  📥 Download MCP Bundle
935
935
  </a>
936
936
  </div>
package/src/lib/config.ts CHANGED
@@ -65,8 +65,8 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
65
65
  export const SDK_METADATA = {
66
66
  language: "typescript",
67
67
  openapiDocVersion: "0.1.0",
68
- sdkVersion: "0.0.31",
68
+ sdkVersion: "0.0.32",
69
69
  genVersion: "2.915.1",
70
70
  userAgent:
71
- "speakeasy-sdk/mcp-typescript 0.0.31 2.915.1 0.1.0 @dalmia/calibrate-mcp",
71
+ "speakeasy-sdk/mcp-typescript 0.0.32 2.915.1 0.1.0 @dalmia/calibrate-mcp",
72
72
  } as const;
package/src/mcp-server/mcp-server.ts CHANGED
@@ -21,7 +21,7 @@ const routes = buildRouteMap({
21
21
  export const app = buildApplication(routes, {
22
22
  name: "mcp",
23
23
  versionInfo: {
24
- currentVersion: "0.0.31",
24
+ currentVersion: "0.0.32",
25
25
  },
26
26
  });
27
27
 
package/src/mcp-server/server.ts CHANGED
@@ -66,7 +66,7 @@ export function createMCPServer(deps: {
66
66
  }) {
67
67
  const server = new McpServer({
68
68
  name: "CalibrateMcp",
69
- version: "0.0.31",
69
+ version: "0.0.32",
70
70
  });
71
71
 
72
72
  const getClient = deps.getSDK || (() =>
package/src/models/agentsummary.ts CHANGED
@@ -36,12 +36,16 @@ export type AgentSummary = {
36
36
  type: AgentSummaryType;
37
37
  updated_at: string;
38
38
  connection_verified?: boolean | null | undefined;
39
+ has_default_inputs: boolean;
39
40
  };
40
41
 
41
42
  export const AgentSummary$zodSchema: z.ZodType<AgentSummary> = z.object({
42
43
  connection_verified: z.boolean().nullable().optional().describe(
43
44
  "Whether the agent's connection has been verified, for a `type=connection` agent",
44
45
  ),
46
+ has_default_inputs: z.boolean().describe(
47
+ "Whether the agent has custom request fields configured",
48
+ ),
45
49
  name: z.string().describe("Name of the agent"),
46
50
  type: AgentSummaryType$zodSchema.describe(
47
51
  "- `agent`: built inside Calibrate\n- `connection`: your existing agent connected to Calibrate",
package/src/models/annotationtaskresponse.ts CHANGED
@@ -53,6 +53,7 @@ export type AnnotationTaskResponse = {
53
53
  updated_at: string;
54
54
  evaluators?: Array<{ [k: string]: any }> | undefined;
55
55
  item_count?: number | null | undefined;
56
+ has_agreement?: boolean | undefined;
56
57
  items?: Array<{ [k: string]: any }> | undefined;
57
58
  jobs?: Array<{ [k: string]: any }> | undefined;
58
59
  };
@@ -67,6 +68,9 @@ export const AnnotationTaskResponse$zodSchema: z.ZodType<
67
68
  evaluators: z.array(z.record(z.string(), z.any())).optional().describe(
68
69
  "The evaluators linked to this task, in display order. Fetching one task by ID enriches each with its live version's rubric. Listing tasks returns lightweight metadata (uuid, name, type, position) only",
69
70
  ),
71
+ has_agreement: z.boolean().default(false).describe(
72
+ "Whether the task has at least one comparable human-vs-human or human-vs-evaluator pair, computed over all time",
73
+ ),
70
74
  item_count: z.int().nullable().optional().describe(
71
75
  "Number of items in the task",
72
76
  ),
package/src/models/benchmarkstatusresponse.ts CHANGED
@@ -13,6 +13,7 @@ import {
13
13
  export type BenchmarkStatusResponse = {
14
14
  task_id: string;
15
15
  status: TaskStatus;
16
+ test_uuids?: Array<string> | null | undefined;
16
17
  evaluators?: Array<TestRunEvaluator> | null | undefined;
17
18
  model_results?: Array<ModelResult> | null | undefined;
18
19
  leaderboard_summary?: Array<{ [k: string]: any }> | null | undefined;
@@ -44,4 +45,7 @@ export const BenchmarkStatusResponse$zodSchema: z.ZodType<
44
45
  ),
45
46
  status: TaskStatus$zodSchema,
46
47
  task_id: z.string().describe("Benchmark run job ID"),
48
+ test_uuids: z.array(z.string()).nullable().optional().describe(
49
+ "IDs of the tests this benchmark executed, in run order",
50
+ ),
47
51
  });
package/src/models/bulktestitem.ts CHANGED
@@ -18,6 +18,7 @@ export type BulkTestItem = {
18
18
  conversation_history: Array<ChatMessage>;
19
19
  evaluators?: Array<RoutersTestsEvaluatorRef> | null | undefined;
20
20
  tool_calls?: Array<ExpectedToolCall> | null | undefined;
21
+ inputs?: { [k: string]: any } | null | undefined;
21
22
  };
22
23
 
23
24
  export const BulkTestItem$zodSchema: z.ZodType<BulkTestItem> = z.object({
@@ -28,6 +29,9 @@ export const BulkTestItem$zodSchema: z.ZodType<BulkTestItem> = z.object({
28
29
  .describe(
29
30
  "Evaluators to link. Used by `response` and `conversation` tests",
30
31
  ),
32
+ inputs: z.record(z.string(), z.any()).nullable().optional().describe(
33
+ "Extra request fields for this test, overriding the agent's `default_inputs` per key",
34
+ ),
31
35
  name: z.string().describe(
32
36
  "Name of the test, unique within the workspace and within the batch",
33
37
  ),
package/src/models/taskagreementresponse.ts CHANGED
@@ -25,7 +25,7 @@ export const TaskAgreementResponse$zodSchema: z.ZodType<TaskAgreementResponse> =
25
25
  bucket: z.string().describe("Time bucket used for the trend series"),
26
26
  days: z.int().describe("Length of the trailing window in days"),
27
27
  evaluators: z.array(z.record(z.string(), z.any())).describe(
28
- "Agreement between each evaluator and the annotators, one entry per linked evaluator",
28
+ "Agreement between each evaluator and the annotators, plus that evaluator's own results, one entry per linked evaluator",
29
29
  ),
30
30
  human_human: HumanAgreementBlock$zodSchema,
31
31
  task_id: z.string().describe("ID of the task"),
package/src/models/testcaseresult.ts CHANGED
@@ -13,6 +13,7 @@ export type TestCaseResult = {
13
13
  reasoning?: string | null | undefined;
14
14
  output?: TestOutput | null | undefined;
15
15
  test_case?: { [k: string]: any } | null | undefined;
16
+ inputs?: { [k: string]: any } | null | undefined;
16
17
  judge_results?: Array<JudgeResult> | null | undefined;
17
18
  latency_ms?: number | null | undefined;
18
19
  cost?: number | null | undefined;
@@ -20,6 +21,9 @@ export type TestCaseResult = {
20
21
 
21
22
  export const TestCaseResult$zodSchema: z.ZodType<TestCaseResult> = z.object({
22
23
  cost: z.number().nullable().optional().describe("Cost of this case (USD)"),
24
+ inputs: z.record(z.string(), z.any()).nullable().optional().describe(
25
+ "Extra request fields sent to the agent for this case, the agent's `default_inputs` with this case's overrides applied",
26
+ ),
23
27
  judge_results: z.array(JudgeResult$zodSchema).nullable().optional().describe(
24
28
  "One verdict for each evaluator",
25
29
  ),
package/src/models/testrunstatusresponse.ts CHANGED
@@ -13,6 +13,7 @@ import {
13
13
  export type TestRunStatusResponse = {
14
14
  task_id: string;
15
15
  status: TaskStatus;
16
+ test_uuids?: Array<string> | null | undefined;
16
17
  total_tests?: number | null | undefined;
17
18
  passed?: number | null | undefined;
18
19
  failed?: number | null | undefined;
@@ -56,6 +57,9 @@ export const TestRunStatusResponse$zodSchema: z.ZodType<TestRunStatusResponse> =
56
57
  ),
57
58
  status: TaskStatus$zodSchema,
58
59
  task_id: z.string().describe("Test run job ID"),
60
+ test_uuids: z.array(z.string()).nullable().optional().describe(
61
+ "IDs of the tests this run executed, in run order",
62
+ ),
59
63
  total_tests: z.int().nullable().optional().describe(
60
64
  "Total number of test cases",
61
65
  ),