@dalmia/calibrate-mcp 0.0.40 → 0.0.42

package/esm/models/testupdate.js

@@ -7,17 +7,19 @@ export const TestUpdateType = {
     Response: "response",
     ToolCall: "tool_call",
     Conversation: "conversation",
+    General: "general",
 };
 export const TestUpdateType$zodSchema = z.enum([
     "response",
     "tool_call",
     "conversation",
+    "general",
 ]);
 export const TestUpdate$zodSchema = z.object({
-    config: z.record(z.string(), z.any()).nullable().optional().describe("The calibrate test config. Three top-level keys.\n\n- `history`: the required conversation up to the agent's turn. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `evaluation`: the required `{type, ...}`, where `type` matches the test's `type` below.\n- `settings`: an optional object, e.g. `{\"language\": \"en\"}`.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nReplaces the stored config. Omit to leave unchanged"),
+    config: z.record(z.string(), z.any()).nullable().optional().describe("The calibrate test config.\n\n- `history`: the conversation up to the agent's turn, required for `response` and `conversation` tests and for a `tool_call` test aimed at a conversational agent. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `input`: a standalone prompt with no conversation around it, required for `general` tests and for a `tool_call` test aimed at a `general` agent. A string, not a conversation.\n- `evaluation`: the required `{type, ...}`, where `type` matches the test's `type` below.\n- `settings`: an optional object, e.g. `{\"language\": \"en\"}`.\n\nA `tool_call` test carries exactly one of `history` or `input`, and which one it carries decides the agent it can be linked to.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n- `general`: judge a standalone, non-conversational input/output pair, graded by the linked evaluators. `{\"type\": \"general\"}`\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example, for a conversational agent. Swap `history` for `input` to aim it at a `general` agent:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\n`general` example:\n```json\n{\n  \"input\": \"Summarize this article: ...\",\n  \"evaluation\": {\"type\": \"general\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nReplaces the stored config. Omit to leave unchanged"),
     evaluators: z.array(RoutersTestsEvaluatorRef$zodSchema).nullable().optional()
         .describe("New evaluator links for the test. Omit to leave unchanged. An empty list clears them, except on `conversation` tests, which must keep at least one"),
     name: z.string().nullable().optional().describe("New test name. Omit to leave unchanged"),
-    type: TestUpdateType$zodSchema.nullable().optional().describe("What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\n\nImmutable. Omit it, or send the current value"),
+    type: TestUpdateType$zodSchema.nullable().optional().describe("What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n\n\nImmutable. Omit it, or send the current value"),
 });
 //# sourceMappingURL=testupdate.js.map

package/esm/models/testupdate.js.map

@@ -1 +1 @@
-{"version":3,"file":"testupdate.js","sourceRoot":"","sources":["../../src/models/testupdate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,kCAAkC,GACnC,MAAM,+BAA+B,CAAC;AAEvC,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,QAAQ,EAAE,UAAU;IACpB,QAAQ,EAAE,WAAW;IACrB,YAAY,EAAE,cAAc;CACpB,CAAC;AAGX,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC,IAAI,CAAC;IAC7C,UAAU;IACV,WAAW;IACX,cAAc;CACf,CAAC,CAAC;AASH,MAAM,CAAC,MAAM,oBAAoB,GAA0B,CAAC,CAAC,MAAM,CAAC;IAClE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,+5DAA+5D,CACh6D;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAC1E,QAAQ,CACP,oJAAoJ,CACrJ;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,wCAAwC,CACzC;IACD,IAAI,EAAE,wBAAwB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC3D,qNAAqN,CACtN;CACF,CAAC,CAAC"}
+{"version":3,"file":"testupdate.js","sourceRoot":"","sources":["../../src/models/testupdate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAEL,kCAAkC,GACnC,MAAM,+BAA+B,CAAC;AAEvC,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,QAAQ,EAAE,UAAU;IACpB,QAAQ,EAAE,WAAW;IACrB,YAAY,EAAE,cAAc;IAC5B,OAAO,EAAE,SAAS;CACV,CAAC;AAGX,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC,IAAI,CAAC;IAC7C,UAAU;IACV,WAAW;IACX,cAAc;IACd,SAAS;CACV,CAAC,CAAC;AASH,MAAM,CAAC,MAAM,oBAAoB,GAA0B,CAAC,CAAC,MAAM,CAAC;IAClE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClE,yqFAAyqF,CAC1qF;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAC1E,QAAQ,CACP,oJAAoJ,CACrJ;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,wCAAwC,CACzC;IACD,IAAI,EAAE,wBAAwB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC3D,+VAA+V,CAChW;CACF,CAAC,CAAC"}

package/esm/models/traceingest.d.ts

@@ -2,11 +2,16 @@ import * as z from "zod";
 import { TraceMetadataEntry } from "./tracemetadataentry.js";
 import { TraceOutput } from "./traceoutput.js";
 import { TraceTurn } from "./traceturn.js";
+/**
+ * What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format
+ */
+export type Input = string | Array<TraceTurn>;
+export declare const Input$zodSchema: z.ZodType<Input>;
 export type TraceIngest = {
     agent_id: string;
     message_id?: string | null | undefined;
     conversation_id?: string | null | undefined;
-    input: Array<TraceTurn>;
+    input: string | Array<TraceTurn>;
     output: TraceOutput;
     metadata?: Array<TraceMetadataEntry> | null | undefined;
 };

package/esm/models/traceingest.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"traceingest.d.ts","sourceRoot":"","sources":["../../src/models/traceingest.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EACL,kBAAkB,EAEnB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,SAAS,EAAuB,MAAM,gBAAgB,CAAC;AAEhE,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC5C,KAAK,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IACxB,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,CAAC,EAAE,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;CACzD,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAkBvD,CAAC"}
+{"version":3,"file":"traceingest.d.ts","sourceRoot":"","sources":["../../src/models/traceingest.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EACL,kBAAkB,EAEnB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,SAAS,EAAuB,MAAM,gBAAgB,CAAC;AAEhE;;GAEG;AACH,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;AAE9C,eAAO,MAAM,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAK5C,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC5C,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;IACjC,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,CAAC,EAAE,KAAK,CAAC,kBAAkB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;CACzD,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAqBvD,CAAC"}

package/esm/models/traceingest.js

@@ -5,10 +5,17 @@ import * as z from "zod";
 import { TraceMetadataEntry$zodSchema, } from "./tracemetadataentry.js";
 import { TraceOutput$zodSchema } from "./traceoutput.js";
 import { TraceTurn$zodSchema } from "./traceturn.js";
+export const Input$zodSchema = z.union([
+    z.string(),
+    z.array(TraceTurn$zodSchema),
+]).describe("What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format");
 export const TraceIngest$zodSchema = z.object({
     agent_id: z.string().describe("ID of the agent that produced the turn. Must be an agent in your workspace"),
     conversation_id: z.string().nullable().optional().describe("Your own ID for the conversation this turn belongs to, stored for reference only. Omit if you have none"),
-    input: z.array(TraceTurn$zodSchema).describe("Conversation history up to the reported output, oldest turn first, in OpenAI chat format"),
+    input: z.union([
+        z.string(),
+        z.array(TraceTurn$zodSchema),
+    ]).describe("What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format"),
     message_id: z.string().nullable().optional().describe("Your own ID for the last user message in `input`, stored for reference only. Omit if you have none"),
     metadata: z.array(TraceMetadataEntry$zodSchema).nullable().optional()
         .describe("Key-value pairs stored with the trace. Prefer OTel `gen_ai.*` key names where they fit. Omit if you have none"),

package/esm/models/traceingest.js.map

@@ -1 +1 @@
-{"version":3,"file":"traceingest.js","sourceRoot":"","sources":["../../src/models/traceingest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAEL,4BAA4B,GAC7B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAa,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAWhE,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC3B,4EAA4E,CAC7E;IACD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxD,yGAAyG,CAC1G;IACD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,QAAQ,CAC1C,0FAA0F,CAC3F;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnD,oGAAoG,CACrG;IACD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,+GAA+G,CAChH;IACH,MAAM,EAAE,qBAAqB;CAC9B,CAAC,CAAC"}
+{"version":3,"file":"traceingest.js","sourceRoot":"","sources":["../../src/models/traceingest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAEL,4BAA4B,GAC7B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAa,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAOhE,MAAM,CAAC,MAAM,eAAe,GAAqB,CAAC,CAAC,KAAK,CAAC;IACvD,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC;CAC7B,CAAC,CAAC,QAAQ,CACT,+MAA+M,CAChN,CAAC;AAWF,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAC3B,4EAA4E,CAC7E;IACD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACxD,yGAAyG,CAC1G;IACD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC;QACb,CAAC,CAAC,MAAM,EAAE;QACV,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC;KAC7B,CAAC,CAAC,QAAQ,CACT,+MAA+M,CAChN;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnD,oGAAoG,CACrG;IACD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAClE,QAAQ,CACP,+GAA+G,CAChH;IACH,MAAM,EAAE,qBAAqB;CAC9B,CAAC,CAAC"}

package/esm/models/tracetoolcall.d.ts

@@ -4,6 +4,7 @@ export type TraceToolCall = {
     arguments?: {
         [k: string]: any;
     } | null | undefined;
+    output?: any | undefined;
 };
 export declare const TraceToolCall$zodSchema: z.ZodType<TraceToolCall>;
 //# sourceMappingURL=tracetoolcall.d.ts.map

package/esm/models/tracetoolcall.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tracetoolcall.d.ts","sourceRoot":"","sources":["../../src/models/tracetoolcall.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;CACrD,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,CAAC,CAAC,OAAO,CAAC,aAAa,CAK3D,CAAC"}
+{"version":3,"file":"tracetoolcall.d.ts","sourceRoot":"","sources":["../../src/models/tracetoolcall.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACpD,MAAM,CAAC,EAAE,GAAG,GAAG,SAAS,CAAC;CAC1B,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,CAAC,CAAC,OAAO,CAAC,aAAa,CAQ3D,CAAC"}

package/esm/models/tracetoolcall.js

@@ -4,6 +4,7 @@
 import * as z from "zod";
 export const TraceToolCall$zodSchema = z.object({
     arguments: z.record(z.string(), z.any()).nullable().optional().describe("Argument values the agent passed to the tool. Omit when the call had none"),
+    output: z.any().optional().describe("What the tool returned for this call. Any JSON value. Omit when you do not record it"),
     tool: z.string().describe("Name of the tool the agent called"),
 });
 //# sourceMappingURL=tracetoolcall.js.map

package/esm/models/tracetoolcall.js.map

@@ -1 +1 @@
-{"version":3,"file":"tracetoolcall.js","sourceRoot":"","sources":["../../src/models/tracetoolcall.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAOzB,MAAM,CAAC,MAAM,uBAAuB,GAA6B,CAAC,CAAC,MAAM,CAAC;IACxE,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACrE,2EAA2E,CAC5E;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;CAC/D,CAAC,CAAC"}
+{"version":3,"file":"tracetoolcall.js","sourceRoot":"","sources":["../../src/models/tracetoolcall.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAQzB,MAAM,CAAC,MAAM,uBAAuB,GAA6B,CAAC,CAAC,MAAM,CAAC;IACxE,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACrE,2EAA2E,CAC5E;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjC,sFAAsF,CACvF;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;CAC/D,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dalmia/calibrate-mcp",
-  "version": "0.0.40",
+  "version": "0.0.42",
   "author": "Speakeasy",
   "type": "module",
   "sideEffects": false,

package/src/funcs/agentTestsListRunsForAgent.ts

@@ -97,6 +97,7 @@ async function $do(
     pathParams$,
   );
   const query$ = encodeFormQuery({
+    "around": payload$.around,
     "has_failures": payload$.has_failures,
     "limit": payload$.limit,
     "offset": payload$.offset,

package/src/landing-page.ts

@@ -930,7 +930,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
         <h1>Instructions</h1>
         <p>One-click installation for Claude Desktop users</p>
         <div class="instruction-item">
-          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.40/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
+          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.42/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
             📥 Download MCP Bundle
           </a>
         </div>

package/src/lib/config.ts

@@ -65,8 +65,8 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
 export const SDK_METADATA = {
   language: "typescript",
   openapiDocVersion: "0.1.0",
-  sdkVersion: "0.0.40",
+  sdkVersion: "0.0.42",
   genVersion: "2.915.1",
   userAgent:
-    "speakeasy-sdk/mcp-typescript 0.0.40 2.915.1 0.1.0 @dalmia/calibrate-mcp",
+    "speakeasy-sdk/mcp-typescript 0.0.42 2.915.1 0.1.0 @dalmia/calibrate-mcp",
 } as const;

package/src/mcp-server/mcp-server.ts

@@ -21,7 +21,7 @@ const routes = buildRouteMap({
 export const app = buildApplication(routes, {
   name: "mcp",
   versionInfo: {
-    currentVersion: "0.0.40",
+    currentVersion: "0.0.42",
   },
 });
 

package/src/mcp-server/server.ts

@@ -71,7 +71,7 @@ export function createMCPServer(deps: {
 }) {
   const server = new McpServer({
     name: "CalibrateMcp",
-    version: "0.0.40",
+    version: "0.0.42",
   });
 
   const getClient = deps.getSDK || (() =>

package/src/models/agentcreate.ts

@@ -30,9 +30,41 @@ export const AgentCreateType$zodSchema = z.enum([
   "- `agent`: built inside Calibrate\n- `connection`: your existing agent connected to Calibrate",
 );
 
+/**
+ * What the agent expects in the request body:
+ *
+ * @remarks
+ *
+ * - `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{"messages": [...]}`
+ * - `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{"input": "..."}`
+ */
+export const AgentCreateInteractionType = {
+  Conversation: "conversation",
+  General: "general",
+} as const;
+/**
+ * What the agent expects in the request body:
+ *
+ * @remarks
+ *
+ * - `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{"messages": [...]}`
+ * - `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{"input": "..."}`
+ */
+export type AgentCreateInteractionType = ClosedEnum<
+  typeof AgentCreateInteractionType
+>;
+
+export const AgentCreateInteractionType$zodSchema = z.enum([
+  "conversation",
+  "general",
+]).describe(
+  "What the agent expects in the request body:\n\n- `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{\"messages\": [...]}`\n- `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{\"input\": \"...\"}`",
+);
+
 export type AgentCreate = {
   name: string;
   type?: AgentCreateType | undefined;
+  interaction_type?: AgentCreateInteractionType | undefined;
   config?: { [k: string]: any } | null | undefined;
 };
 
@@ -40,6 +72,10 @@ export const AgentCreate$zodSchema: z.ZodType<AgentCreate> = z.object({
   config: z.record(z.string(), z.any()).nullable().optional().describe(
     "Agent behavioral config. The keys depend on `type`.\n\n**`type=agent`**, built inside Calibrate:\n- `system_prompt`: the agent's instructions\n- `llm.model`: `provider/model`, e.g. `openai/gpt-4.1` or `google/gemini-2.5-flash`\n- `stt.provider`: `deepgram`, `openai`, `cartesia`, `elevenlabs`, `google`, `sarvam`, or `smallest`\n- `tts.provider`: `cartesia`, `openai`, `google`, `elevenlabs`, `sarvam`, or `smallest`\n- `settings.agent_speaks_first`, `settings.max_assistant_turns`\n- `system_tools.end_call`: let the agent end the call\n- `data_extraction_fields`: `[{name, type, description, required}]`\n\n```json\n{\n  \"system_prompt\": \"You are a helpful support agent.\",\n  \"llm\": {\"model\": \"openai/gpt-4.1\"},\n  \"stt\": {\"provider\": \"deepgram\"},\n  \"tts\": {\"provider\": \"elevenlabs\"},\n  \"settings\": {\"agent_speaks_first\": true, \"max_assistant_turns\": 50}\n}\n```\n\n**`type=connection`**, your own HTTP endpoint:\n- `agent_url`: public HTTP(S) endpoint your agent is called at\n- `agent_headers`: headers sent on each request, e.g. auth\n- `benchmark_provider`: `openrouter` by default. Other values: `openai`, `google`, `anthropic`, `meta-llama`, `mistralai`, `deepseek`, `x-ai`, `cohere`, `qwen`, or `ai21`\n\n```json\n{\n  \"agent_url\": \"https://api.example.com/agent\",\n  \"agent_headers\": {\"Authorization\": \"Bearer <token>\"},\n  \"benchmark_provider\": \"openrouter\"\n}\n```\n\nFor `type=agent`, omitted keys inherit managed defaults. Omit `config` entirely to use all defaults. For `type=connection`, `config` is stored as-is and must contain `agent_url`",
   ),
+  interaction_type: AgentCreateInteractionType$zodSchema.default("conversation")
+    .describe(
+      "What the agent expects in the request body:\n\n- `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{\"messages\": [...]}`\n- `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{\"input\": \"...\"}`",
+    ),
   name: z.string().describe("Agent name, unique within the workspace"),
   type: AgentCreateType$zodSchema.default("agent").describe(
     "- `agent`: built inside Calibrate\n- `connection`: your existing agent connected to Calibrate",

package/src/models/agentresponse.ts

@@ -30,10 +30,42 @@ export const AgentResponseType$zodSchema = z.enum([
   "- `agent`: built inside Calibrate\n- `connection`: your existing agent connected to Calibrate",
 );
 
+/**
+ * What the agent expects in the request body:
+ *
+ * @remarks
+ *
+ * - `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{"messages": [...]}`
+ * - `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{"input": "..."}`
+ */
+export const AgentResponseInteractionType = {
+  Conversation: "conversation",
+  General: "general",
+} as const;
+/**
+ * What the agent expects in the request body:
+ *
+ * @remarks
+ *
+ * - `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{"messages": [...]}`
+ * - `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{"input": "..."}`
+ */
+export type AgentResponseInteractionType = ClosedEnum<
+  typeof AgentResponseInteractionType
+>;
+
+export const AgentResponseInteractionType$zodSchema = z.enum([
+  "conversation",
+  "general",
+]).describe(
+  "What the agent expects in the request body:\n\n- `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{\"messages\": [...]}`\n- `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{\"input\": \"...\"}`",
+);
+
 export type AgentResponse = {
   uuid: string;
   name: string;
   type: AgentResponseType;
+  interaction_type: AgentResponseInteractionType;
   config?: { [k: string]: any } | null | undefined;
   created_at: string;
   updated_at: string;
@@ -44,6 +76,9 @@ export const AgentResponse$zodSchema: z.ZodType<AgentResponse> = z.object({
     "Agent configuration",
   ),
   created_at: z.string().describe("When the agent was created (ISO 8601 UTC)"),
+  interaction_type: AgentResponseInteractionType$zodSchema.describe(
+    "What the agent expects in the request body:\n\n- `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{\"messages\": [...]}`\n- `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{\"input\": \"...\"}`",
+  ),
   name: z.string().describe("Name of the agent"),
   type: AgentResponseType$zodSchema.describe(
     "- `agent`: built inside Calibrate\n- `connection`: your existing agent connected to Calibrate",

package/src/models/agentsummary.ts

@@ -30,10 +30,42 @@ export const AgentSummaryType$zodSchema = z.enum([
   "- `agent`: built inside Calibrate\n- `connection`: your existing agent connected to Calibrate",
 );
 
+/**
+ * What the agent expects in the request body:
+ *
+ * @remarks
+ *
+ * - `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{"messages": [...]}`
+ * - `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{"input": "..."}`
+ */
+export const AgentSummaryInteractionType = {
+  Conversation: "conversation",
+  General: "general",
+} as const;
+/**
+ * What the agent expects in the request body:
+ *
+ * @remarks
+ *
+ * - `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{"messages": [...]}`
+ * - `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{"input": "..."}`
+ */
+export type AgentSummaryInteractionType = ClosedEnum<
+  typeof AgentSummaryInteractionType
+>;
+
+export const AgentSummaryInteractionType$zodSchema = z.enum([
+  "conversation",
+  "general",
+]).describe(
+  "What the agent expects in the request body:\n\n- `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{\"messages\": [...]}`\n- `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{\"input\": \"...\"}`",
+);
+
 export type AgentSummary = {
   uuid: string;
   name: string;
   type: AgentSummaryType;
+  interaction_type: AgentSummaryInteractionType;
   created_at: string;
   updated_at: string;
   connection_verified?: boolean | null | undefined;
@@ -48,6 +80,9 @@ export const AgentSummary$zodSchema: z.ZodType<AgentSummary> = z.object({
   has_default_inputs: z.boolean().describe(
     "Whether the agent has custom request fields configured",
   ),
+  interaction_type: AgentSummaryInteractionType$zodSchema.describe(
+    "What the agent expects in the request body:\n\n- `conversation`: a normal back-and-forth agent, answers within an ongoing conversation. Receives `{\"messages\": [...]}`\n- `general`: a one-shot agent, takes a single plain input and produces a single plain output, no conversation. Receives `{\"input\": \"...\"}`",
+  ),
   name: z.string().describe("Name of the agent"),
   type: AgentSummaryType$zodSchema.describe(
     "- `agent`: built inside Calibrate\n- `connection`: your existing agent connected to Calibrate",

package/src/models/agenttestrunlistitem.ts

@@ -48,10 +48,12 @@ export type AgentTestRunListItem = {
   name: string;
   status: TaskStatus;
   type: AgentTestRunListItemType;
+  created_at: string;
   updated_at: string;
   total_tests?: number | null | undefined;
   passed?: number | null | undefined;
   failed?: number | null | undefined;
+  evaluators?: Array<string> | undefined;
   results?: Array<TestRunCaseSummary> | null | undefined;
   latency_ms?: { [k: string]: any } | null | undefined;
   cost?: { [k: string]: any } | null | undefined;
@@ -67,7 +69,11 @@ export const AgentTestRunListItem$zodSchema: z.ZodType<AgentTestRunListItem> = z
     cost: z.record(z.string(), z.any()).nullable().optional().describe(
       "Aggregated cost as `{mean, min, max, count}` (USD)",
     ),
+    created_at: z.string().describe("When the run was created (ISO 8601 UTC)"),
     error: z.boolean().default(false).describe("True if the run failed"),
+    evaluators: z.array(z.string()).optional().describe(
+      "Names of the evaluators that judged this run, deduplicated and in display order. `Tool call` is appended when any test in the run was a tool-call test. Empty when the run had no evaluators",
+    ),
     failed: z.int().nullable().optional().describe(
       "Number of test cases that failed",
     ),

package/src/models/annotationtaskresponse.ts

@@ -75,7 +75,7 @@ export const AnnotationTaskResponse$zodSchema: z.ZodType<
     "Number of items in the task",
   ),
   items: z.array(z.record(z.string(), z.any())).optional().describe(
-    "The task's items, each with its agreement stats. You get these when you fetch one task by ID, not when you list tasks",
+    "The task's items, each with its agreement stats and an `is_tool_call` flag marking the rows a person labels on the tool call rather than a text reply. You get these when you fetch one task by ID, not when you list tasks",
   ),
   jobs: z.array(z.record(z.string(), z.any())).optional().describe(
     "The task's labelling jobs. You get these when you fetch one task by ID, not when you list tasks",

package/src/models/bulktestitem.ts

@@ -15,20 +15,25 @@ import {
 
 export type BulkTestItem = {
   name: string;
-  conversation_history: Array<ChatMessage>;
+  conversation_history?: Array<ChatMessage> | null | undefined;
+  input?: string | null | undefined;
   evaluators?: Array<RoutersTestsEvaluatorRef> | null | undefined;
   tool_calls?: Array<ExpectedToolCall> | null | undefined;
   inputs?: { [k: string]: any } | null | undefined;
 };
 
 export const BulkTestItem$zodSchema: z.ZodType<BulkTestItem> = z.object({
-  conversation_history: z.array(ChatMessage$zodSchema).describe(
-    "Ordered messages ending at the user turn the agent should answer",
-  ),
+  conversation_history: z.array(ChatMessage$zodSchema).nullable().optional()
+    .describe(
+      "Ordered messages ending at the user turn the agent should answer. **Required for `response` and `conversation` batches**, and for `tool_call` batches aimed at a conversational agent",
+    ),
   evaluators: z.array(RoutersTestsEvaluatorRef$zodSchema).nullable().optional()
     .describe(
-      "Evaluators to link. Used by `response` and `conversation` tests",
+      "Evaluators to link. Used by `response`, `conversation`, and `general` tests",
     ),
+  input: z.string().nullable().optional().describe(
+    "Standalone prompt with no conversation around it. **Required for `general` batches**, and for `tool_call` batches aimed at a general agent",
+  ),
   inputs: z.record(z.string(), z.any()).nullable().optional().describe(
     "Extra request fields for this test, overriding the agent's `default_inputs` per key",
   ),

package/src/models/bulktestupload.ts

@@ -14,6 +14,7 @@ import { BulkTestItem, BulkTestItem$zodSchema } from "./bulktestitem.js";
  * - `response`: judges the generated reply
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
+ * - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
  *
  * Applied to every test in the batch
  */
@@ -21,6 +22,7 @@ export const BulkTestUploadType = {
   Response: "response",
   ToolCall: "tool_call",
   Conversation: "conversation",
+  General: "general",
 } as const;
 /**
  * What the test judges:
@@ -30,6 +32,7 @@ export const BulkTestUploadType = {
  * - `response`: judges the generated reply
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
+ * - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
  *
  * Applied to every test in the batch
  */
@@ -39,8 +42,9 @@ export const BulkTestUploadType$zodSchema = z.enum([
   "response",
   "tool_call",
   "conversation",
+  "general",
 ]).describe(
-  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\n\nApplied to every test in the batch",
+  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n\n\nApplied to every test in the batch",
 );
 
 export type BulkTestUpload = {
@@ -61,6 +65,6 @@ export const BulkTestUpload$zodSchema: z.ZodType<BulkTestUpload> = z.object({
     "Test items to create, at most 500 per request, with names unique within the batch",
   ),
   type: BulkTestUploadType$zodSchema.describe(
-    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\n\nApplied to every test in the batch",
+    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n\n\nApplied to every test in the batch",
   ),
 });

package/src/models/evaluatorcreate.ts

@@ -19,6 +19,7 @@ import {
  * - `llm`: a reply with its conversation history
  * - `llm-general`: a standalone input and output pair
  * - `conversation`: a full conversation
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
  */
 export const EvaluatorCreateEvaluatorType = {
   Tts: "tts",
@@ -26,6 +27,7 @@ export const EvaluatorCreateEvaluatorType = {
   Llm: "llm",
   LlmGeneral: "llm-general",
   Conversation: "conversation",
+  ToolCall: "tool-call",
 } as const;
 /**
  * What the evaluator judges:
@@ -37,6 +39,7 @@ export const EvaluatorCreateEvaluatorType = {
  * - `llm`: a reply with its conversation history
  * - `llm-general`: a standalone input and output pair
  * - `conversation`: a full conversation
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
  */
 export type EvaluatorCreateEvaluatorType = ClosedEnum<
   typeof EvaluatorCreateEvaluatorType
@@ -48,8 +51,9 @@ export const EvaluatorCreateEvaluatorType$zodSchema = z.enum([
   "llm",
   "llm-general",
   "conversation",
+  "tool-call",
 ]).describe(
-  "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n",
+  "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n",
 );
 
 /**
@@ -130,7 +134,7 @@ export const EvaluatorCreate$zodSchema: z.ZodType<EvaluatorCreate> = z.object({
   ),
   evaluator_type: EvaluatorCreateEvaluatorType$zodSchema.default("llm")
     .describe(
-      "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n",
+      "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n",
     ),
   name: z.string().describe("Evaluator name, unique within your workspace"),
   output_type: EvaluatorCreateOutputType$zodSchema.default("binary").describe(

package/src/models/evaluatordetailresponsecompact.ts

@@ -19,6 +19,7 @@ import {
  * - `llm`: a reply with its conversation history
  * - `llm-general`: a standalone input and output pair
  * - `conversation`: a full conversation
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
  */
 export const EvaluatorDetailResponseCompactEvaluatorType = {
   Tts: "tts",
@@ -26,6 +27,7 @@ export const EvaluatorDetailResponseCompactEvaluatorType = {
   Llm: "llm",
   LlmGeneral: "llm-general",
   Conversation: "conversation",
+  ToolCall: "tool-call",
 } as const;
 /**
  * What the evaluator judges:
@@ -37,6 +39,7 @@ export const EvaluatorDetailResponseCompactEvaluatorType = {
  * - `llm`: a reply with its conversation history
  * - `llm-general`: a standalone input and output pair
  * - `conversation`: a full conversation
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
  */
 export type EvaluatorDetailResponseCompactEvaluatorType = ClosedEnum<
   typeof EvaluatorDetailResponseCompactEvaluatorType
@@ -48,8 +51,9 @@ export const EvaluatorDetailResponseCompactEvaluatorType$zodSchema = z.enum([
   "llm",
   "llm-general",
   "conversation",
+  "tool-call",
 ]).describe(
-  "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n",
+  "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n",
 );
 
 /**
@@ -120,6 +124,7 @@ export type EvaluatorDetailResponseCompact = {
   data_type: EvaluatorDetailResponseCompactDataType;
   output_type: EvaluatorDetailResponseCompactOutputType;
   is_default: boolean;
+  is_protected: boolean;
   slug?: string | null | undefined;
   source_default_slug?: string | null | undefined;
   live_version_id?: string | null | undefined;
@@ -142,11 +147,14 @@ export const EvaluatorDetailResponseCompact$zodSchema: z.ZodType<
   ),
   evaluator_type: EvaluatorDetailResponseCompactEvaluatorType$zodSchema
     .describe(
-      "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n",
+      "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n",
     ),
   is_default: z.boolean().describe(
     "True when the evaluator is a built-in default or your workspace's editable copy of one. False for an evaluator you created yourself",
   ),
+  is_protected: z.boolean().describe(
+    "True when the evaluator is locked. A locked evaluator cannot be deleted, and only its name, description and rubric can change",
+  ),
   live_version_id: z.string().nullable().optional().describe(
     "ID of the version that is currently live",
   ),

package/src/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.ts

@@ -49,6 +49,7 @@ export type GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetRequest = {
   type?: GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetType | null | undefined;
   status?: TaskStatus | null | undefined;
   has_failures?: boolean | null | undefined;
+  around?: string | null | undefined;
   limit?: number | null | undefined;
   offset?: number | undefined;
   xAPIKey?: string | null | undefined;
@@ -57,6 +58,9 @@ export type GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetRequest = {
 export const GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetRequest$zodSchema:
   z.ZodType<GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetRequest> = z.object({
     agent_uuid: z.string().describe("Agent whose test runs to list"),
+    around: z.string().describe(
+      "ID of a run to jump to, returning the page that contains it instead of the page at `offset`",
+    ).nullable().optional(),
     has_failures: z.boolean().describe(
       "Filter by whether the run has any failing test case or model. `true` returns only runs with failures (or errors), `false` only clean runs. Omit for both",
     ).nullable().optional(),

package/src/models/listevaluatorsevaluatorsgetop.ts

@@ -22,6 +22,7 @@ export const ListEvaluatorsEvaluatorsGetEvaluatorType = {
   Llm: "llm",
   LlmGeneral: "llm-general",
   Conversation: "conversation",
+  ToolCall: "tool-call",
 } as const;
 /**
  * Filter by what the evaluator judges. Omit for all types
@@ -36,6 +37,7 @@ export const ListEvaluatorsEvaluatorsGetEvaluatorType$zodSchema = z.enum([
   "llm",
   "llm-general",
   "conversation",
+  "tool-call",
 ]).describe("Filter by what the evaluator judges. Omit for all types");
 
 /**