@dalmia/calibrate-mcp 0.0.31 → 0.0.33

package/src/models/annotationtaskresponse.ts

@@ -53,6 +53,7 @@ export type AnnotationTaskResponse = {
   updated_at: string;
   evaluators?: Array<{ [k: string]: any }> | undefined;
   item_count?: number | null | undefined;
+  has_agreement?: boolean | undefined;
   items?: Array<{ [k: string]: any }> | undefined;
   jobs?: Array<{ [k: string]: any }> | undefined;
 };
@@ -67,6 +68,9 @@ export const AnnotationTaskResponse$zodSchema: z.ZodType<
   evaluators: z.array(z.record(z.string(), z.any())).optional().describe(
     "The evaluators linked to this task, in display order. Fetching one task by ID enriches each with its live version's rubric. Listing tasks returns lightweight metadata (uuid, name, type, position) only",
   ),
+  has_agreement: z.boolean().default(false).describe(
+    "Whether the task has at least one comparable human-vs-human or human-vs-evaluator pair, computed over all time",
+  ),
   item_count: z.int().nullable().optional().describe(
     "Number of items in the task",
   ),

package/src/models/benchmarkstatusresponse.ts

@@ -13,6 +13,7 @@ import {
 export type BenchmarkStatusResponse = {
   task_id: string;
   status: TaskStatus;
+  test_uuids?: Array<string> | null | undefined;
   evaluators?: Array<TestRunEvaluator> | null | undefined;
   model_results?: Array<ModelResult> | null | undefined;
   leaderboard_summary?: Array<{ [k: string]: any }> | null | undefined;
@@ -44,4 +45,7 @@ export const BenchmarkStatusResponse$zodSchema: z.ZodType<
   ),
   status: TaskStatus$zodSchema,
   task_id: z.string().describe("Benchmark run job ID"),
+  test_uuids: z.array(z.string()).nullable().optional().describe(
+    "IDs of the tests this benchmark executed, in run order",
+  ),
 });

package/src/models/bulktestitem.ts

@@ -18,6 +18,7 @@ export type BulkTestItem = {
   conversation_history: Array<ChatMessage>;
   evaluators?: Array<RoutersTestsEvaluatorRef> | null | undefined;
   tool_calls?: Array<ExpectedToolCall> | null | undefined;
+  inputs?: { [k: string]: any } | null | undefined;
 };
 
 export const BulkTestItem$zodSchema: z.ZodType<BulkTestItem> = z.object({
@@ -28,6 +29,9 @@ export const BulkTestItem$zodSchema: z.ZodType<BulkTestItem> = z.object({
     .describe(
       "Evaluators to link. Used by `response` and `conversation` tests",
     ),
+  inputs: z.record(z.string(), z.any()).nullable().optional().describe(
+    "Extra request fields for this test, overriding the agent's `default_inputs` per key",
+  ),
   name: z.string().describe(
     "Name of the test, unique within the workspace and within the batch",
   ),

package/src/models/getagentendpointagentsagentuuidgetop.ts

@@ -3,14 +3,11 @@
  */
 
 import * as z from "zod";
+import { AgentResponse, AgentResponse$zodSchema } from "./agentresponse.js";
 import {
   HTTPValidationError,
   HTTPValidationError$zodSchema,
 } from "./httpvalidationerror.js";
-import {
-  RoutersAgentsAgentResponse,
-  RoutersAgentsAgentResponse$zodSchema,
-} from "./routersagentsagentresponse.js";
 
 export type GetAgentEndpointAgentsAgentUuidGetRequest = {
   agent_uuid: string;
@@ -25,12 +22,12 @@ export const GetAgentEndpointAgentsAgentUuidGetRequest$zodSchema: z.ZodType<
 });
 
 export type GetAgentEndpointAgentsAgentUuidGetResponse =
-  | RoutersAgentsAgentResponse
+  | AgentResponse
   | HTTPValidationError;
 
 export const GetAgentEndpointAgentsAgentUuidGetResponse$zodSchema: z.ZodType<
   GetAgentEndpointAgentsAgentUuidGetResponse
 > = z.union([
-  RoutersAgentsAgentResponse$zodSchema,
+  AgentResponse$zodSchema,
   HTTPValidationError$zodSchema,
 ]);

package/src/models/taskagreementresponse.ts

@@ -25,7 +25,7 @@ export const TaskAgreementResponse$zodSchema: z.ZodType<TaskAgreementResponse> =
       bucket: z.string().describe("Time bucket used for the trend series"),
       days: z.int().describe("Length of the trailing window in days"),
       evaluators: z.array(z.record(z.string(), z.any())).describe(
-        "Agreement between each evaluator and the annotators, one entry per linked evaluator",
+        "Agreement between each evaluator and the annotators, plus that evaluator's own results, one entry per linked evaluator",
       ),
       human_human: HumanAgreementBlock$zodSchema,
       task_id: z.string().describe("ID of the task"),

package/src/models/testcaseresult.ts

@@ -13,6 +13,7 @@ export type TestCaseResult = {
   reasoning?: string | null | undefined;
   output?: TestOutput | null | undefined;
   test_case?: { [k: string]: any } | null | undefined;
+  inputs?: { [k: string]: any } | null | undefined;
   judge_results?: Array<JudgeResult> | null | undefined;
   latency_ms?: number | null | undefined;
   cost?: number | null | undefined;
@@ -20,6 +21,9 @@ export type TestCaseResult = {
 
 export const TestCaseResult$zodSchema: z.ZodType<TestCaseResult> = z.object({
   cost: z.number().nullable().optional().describe("Cost of this case (USD)"),
+  inputs: z.record(z.string(), z.any()).nullable().optional().describe(
+    "Extra request fields sent to the agent for this case, the agent's `default_inputs` with this case's overrides applied",
+  ),
   judge_results: z.array(JudgeResult$zodSchema).nullable().optional().describe(
     "One verdict for each evaluator",
   ),

package/src/models/testrunstatusresponse.ts

@@ -13,6 +13,7 @@ import {
 export type TestRunStatusResponse = {
   task_id: string;
   status: TaskStatus;
+  test_uuids?: Array<string> | null | undefined;
   total_tests?: number | null | undefined;
   passed?: number | null | undefined;
   failed?: number | null | undefined;
@@ -56,6 +57,9 @@ export const TestRunStatusResponse$zodSchema: z.ZodType<TestRunStatusResponse> =
     ),
     status: TaskStatus$zodSchema,
     task_id: z.string().describe("Test run job ID"),
+    test_uuids: z.array(z.string()).nullable().optional().describe(
+      "IDs of the tests this run executed, in run order",
+    ),
     total_tests: z.int().nullable().optional().describe(
       "Total number of test cases",
     ),

package/src/models/updateagentendpointagentsagentuuidputop.ts

@@ -3,15 +3,12 @@
  */
 
 import * as z from "zod";
+import { AgentResponse, AgentResponse$zodSchema } from "./agentresponse.js";
 import { AgentUpdate, AgentUpdate$zodSchema } from "./agentupdate.js";
 import {
   HTTPValidationError,
   HTTPValidationError$zodSchema,
 } from "./httpvalidationerror.js";
-import {
-  RoutersAgentsAgentResponse,
-  RoutersAgentsAgentResponse$zodSchema,
-} from "./routersagentsagentresponse.js";
 
 export type UpdateAgentEndpointAgentsAgentUuidPutRequest = {
   agent_uuid: string;
@@ -28,12 +25,12 @@ export const UpdateAgentEndpointAgentsAgentUuidPutRequest$zodSchema: z.ZodType<
 });
 
 export type UpdateAgentEndpointAgentsAgentUuidPutResponse =
-  | RoutersAgentsAgentResponse
+  | AgentResponse
   | HTTPValidationError;
 
 export const UpdateAgentEndpointAgentsAgentUuidPutResponse$zodSchema: z.ZodType<
   UpdateAgentEndpointAgentsAgentUuidPutResponse
 > = z.union([
-  RoutersAgentsAgentResponse$zodSchema,
+  AgentResponse$zodSchema,
   HTTPValidationError$zodSchema,
 ]);

package/esm/models/routersagentsagentresponse.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"routersagentsagentresponse.d.ts","sourceRoot":"","sources":["../../src/models/routersagentsagentresponse.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B;;;CAGjC,CAAC;AACX;;;;;GAKG;AACH,MAAM,MAAM,8BAA8B,GAAG,UAAU,CACrD,OAAO,8BAA8B,CACtC,CAAC;AAEF,eAAO,MAAM,wCAAwC;;;EAKpD,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,8BAA8B,CAAC;IACrC,MAAM,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,eAAO,MAAM,oCAAoC,EAAE,CAAC,CAAC,OAAO,CAC1D,0BAA0B,CAc1B,CAAC"}

package/esm/models/routersagentsagentresponse.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"routersagentsagentresponse.js","sourceRoot":"","sources":["../../src/models/routersagentsagentresponse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAG;IAC5C,KAAK,EAAE,OAAO;IACd,UAAU,EAAE,YAAY;CAChB,CAAC;AAWX,MAAM,CAAC,MAAM,wCAAwC,GAAG,CAAC,CAAC,IAAI,CAAC;IAC7D,OAAO;IACP,YAAY;CACb,CAAC,CAAC,QAAQ,CACT,+FAA+F,CAChG,CAAC;AAWF,MAAM,CAAC,MAAM,oCAAoC,GAE7C,CAAC,CAAC,MAAM,CAAC;IACX,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,qBAAqB,CACtB;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2CAA2C,CAAC;IAC5E,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IAC9C,IAAI,EAAE,wCAAwC,CAAC,QAAQ,CACrD,+FAA+F,CAChG;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC7B,gDAAgD,CACjD;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;CAC7C,CAAC,CAAC"}