@dalmia/calibrate-mcp 0.0.40 → 0.0.42

package/src/models/routersevaluatorsevaluatorresponse.ts

@@ -19,6 +19,7 @@ import {
  * - `llm`: a reply with its conversation history
  * - `llm-general`: a standalone input and output pair
  * - `conversation`: a full conversation
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
  */
 export const RoutersEvaluatorsEvaluatorResponseEvaluatorType = {
   Tts: "tts",
@@ -26,6 +27,7 @@ export const RoutersEvaluatorsEvaluatorResponseEvaluatorType = {
   Llm: "llm",
   LlmGeneral: "llm-general",
   Conversation: "conversation",
+  ToolCall: "tool-call",
 } as const;
 /**
  * What the evaluator judges:
@@ -37,6 +39,7 @@ export const RoutersEvaluatorsEvaluatorResponseEvaluatorType = {
  * - `llm`: a reply with its conversation history
  * - `llm-general`: a standalone input and output pair
  * - `conversation`: a full conversation
+ * - `tool-call`: whether the agent called the right tool, labelled by a person
  */
 export type RoutersEvaluatorsEvaluatorResponseEvaluatorType = ClosedEnum<
   typeof RoutersEvaluatorsEvaluatorResponseEvaluatorType
@@ -49,9 +52,10 @@ export const RoutersEvaluatorsEvaluatorResponseEvaluatorType$zodSchema = z.enum(
     "llm",
     "llm-general",
     "conversation",
+    "tool-call",
   ],
 ).describe(
-  "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n",
+  "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n",
 );
 
 /**
@@ -122,6 +126,7 @@ export type RoutersEvaluatorsEvaluatorResponse = {
   data_type: RoutersEvaluatorsEvaluatorResponseDataType;
   output_type: RoutersEvaluatorsEvaluatorResponseOutputType;
   is_default: boolean;
+  is_protected: boolean;
   slug?: string | null | undefined;
   source_default_slug?: string | null | undefined;
   live_version_id?: string | null | undefined;
@@ -144,11 +149,14 @@ export const RoutersEvaluatorsEvaluatorResponse$zodSchema: z.ZodType<
   ),
   evaluator_type: RoutersEvaluatorsEvaluatorResponseEvaluatorType$zodSchema
     .describe(
-      "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n",
+      "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n",
     ),
   is_default: z.boolean().describe(
     "True when the evaluator is a built-in default or your workspace's editable copy of one. False for an evaluator you created yourself",
   ),
+  is_protected: z.boolean().describe(
+    "True when the evaluator is locked. A locked evaluator cannot be deleted, and only its name, description and rubric can change",
+  ),
   live_version: EvaluatorLiveVersionSummary$zodSchema.nullable().optional()
     .describe("The version that is currently live"),
   live_version_id: z.string().nullable().optional().describe(

package/src/models/testcreate.ts

@@ -17,11 +17,13 @@ import {
  * - `response`: judges the generated reply
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
+ * - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
  */
 export const TestCreateType = {
   Response: "response",
   ToolCall: "tool_call",
   Conversation: "conversation",
+  General: "general",
 } as const;
 /**
  * What the test judges:
@@ -31,6 +33,7 @@ export const TestCreateType = {
  * - `response`: judges the generated reply
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
+ * - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
  */
 export type TestCreateType = ClosedEnum<typeof TestCreateType>;
 
@@ -38,8 +41,9 @@ export const TestCreateType$zodSchema = z.enum([
   "response",
   "tool_call",
   "conversation",
+  "general",
 ]).describe(
-  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
+  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n",
 );
 
 export type TestCreate = {
@@ -51,7 +55,7 @@ export type TestCreate = {
 
 export const TestCreate$zodSchema: z.ZodType<TestCreate> = z.object({
   config: z.record(z.string(), z.any()).nullable().optional().describe(
-    "The calibrate test config. Three top-level keys.\n\n- `history`: the required conversation up to the agent's turn. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `evaluation`: the required `{type, ...}`, where `type` matches the test's `type` below.\n- `settings`: an optional object, e.g. `{\"language\": \"en\"}`.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nOmit to create the test with no config and fill it in later via update",
+    "The calibrate test config.\n\n- `history`: the conversation up to the agent's turn, required for `response` and `conversation` tests and for a `tool_call` test aimed at a conversational agent. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `input`: a standalone prompt with no conversation around it, required for `general` tests and for a `tool_call` test aimed at a `general` agent. A string, not a conversation.\n- `evaluation`: the required `{type, ...}`, where `type` matches the test's `type` below.\n- `settings`: an optional object, e.g. `{\"language\": \"en\"}`.\n\nA `tool_call` test carries exactly one of `history` or `input`, and which one it carries decides the agent it can be linked to.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n- `general`: judge a standalone, non-conversational input/output pair, graded by the linked evaluators. `{\"type\": \"general\"}`\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example, for a conversational agent. Swap `history` for `input` to aim it at a `general` agent:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\n`general` example:\n```json\n{\n  \"input\": \"Summarize this article: ...\",\n  \"evaluation\": {\"type\": \"general\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nOmit to create the test with no config and fill it in later via update",
   ),
   evaluators: z.array(RoutersTestsEvaluatorRef$zodSchema).nullable().optional()
     .describe(
@@ -59,6 +63,6 @@ export const TestCreate$zodSchema: z.ZodType<TestCreate> = z.object({
     ),
   name: z.string().describe("Name of the test, unique within the workspace"),
   type: TestCreateType$zodSchema.describe(
-    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
+    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n",
   ),
 });

package/src/models/testlistresponse.ts

@@ -14,11 +14,13 @@ import { TestListConfig, TestListConfig$zodSchema } from "./testlistconfig.js";
  * - `response`: judges the generated reply
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
+ * - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
  */
 export const TestListResponseType = {
   Response: "response",
   ToolCall: "tool_call",
   Conversation: "conversation",
+  General: "general",
 } as const;
 /**
  * What the test judges:
@@ -28,6 +30,7 @@ export const TestListResponseType = {
  * - `response`: judges the generated reply
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
+ * - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
  */
 export type TestListResponseType = ClosedEnum<typeof TestListResponseType>;
 
@@ -35,8 +38,9 @@ export const TestListResponseType$zodSchema = z.enum([
   "response",
   "tool_call",
   "conversation",
+  "general",
 ]).describe(
-  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
+  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n",
 );
 
 /**
@@ -63,7 +67,7 @@ export const TestListResponse$zodSchema: z.ZodType<TestListResponse> = z.object(
     created_at: z.string().describe("When the test was created (ISO 8601 UTC)"),
     name: z.string().describe("Name of the test"),
     type: TestListResponseType$zodSchema.describe(
-      "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
+      "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n",
     ),
     updated_at: z.string().describe(
       "When the test was last updated (ISO 8601 UTC)",

package/src/models/testresponse.ts

@@ -13,11 +13,13 @@ import { ClosedEnum } from "../types/enums.js";
  * - `response`: judges the generated reply
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
+ * - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
  */
 export const TestResponseType = {
   Response: "response",
   ToolCall: "tool_call",
   Conversation: "conversation",
+  General: "general",
 } as const;
 /**
  * What the test judges:
@@ -27,6 +29,7 @@ export const TestResponseType = {
  * - `response`: judges the generated reply
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
+ * - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
  */
 export type TestResponseType = ClosedEnum<typeof TestResponseType>;
 
@@ -34,8 +37,9 @@ export const TestResponseType$zodSchema = z.enum([
   "response",
   "tool_call",
   "conversation",
+  "general",
 ]).describe(
-  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
+  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n",
 );
 
 export type TestResponse = {
@@ -58,7 +62,7 @@ export const TestResponse$zodSchema: z.ZodType<TestResponse> = z.object({
   ),
   name: z.string().describe("Name of the test"),
   type: TestResponseType$zodSchema.describe(
-    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
+    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n",
   ),
   updated_at: z.string().describe(
     "When the test was last updated (ISO 8601 UTC)",

package/src/models/testupdate.ts

@@ -13,6 +13,7 @@ export const TestUpdateType = {
   Response: "response",
   ToolCall: "tool_call",
   Conversation: "conversation",
+  General: "general",
 } as const;
 export type TestUpdateType = ClosedEnum<typeof TestUpdateType>;
 
@@ -20,6 +21,7 @@ export const TestUpdateType$zodSchema = z.enum([
   "response",
   "tool_call",
   "conversation",
+  "general",
 ]);
 
 export type TestUpdate = {
@@ -31,7 +33,7 @@ export type TestUpdate = {
 
 export const TestUpdate$zodSchema: z.ZodType<TestUpdate> = z.object({
   config: z.record(z.string(), z.any()).nullable().optional().describe(
-    "The calibrate test config. Three top-level keys.\n\n- `history`: the required conversation up to the agent's turn. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `evaluation`: the required `{type, ...}`, where `type` matches the test's `type` below.\n- `settings`: an optional object, e.g. `{\"language\": \"en\"}`.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nReplaces the stored config. Omit to leave unchanged",
+    "The calibrate test config.\n\n- `history`: the conversation up to the agent's turn, required for `response` and `conversation` tests and for a `tool_call` test aimed at a conversational agent. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `input`: a standalone prompt with no conversation around it, required for `general` tests and for a `tool_call` test aimed at a `general` agent. A string, not a conversation.\n- `evaluation`: the required `{type, ...}`, where `type` matches the test's `type` below.\n- `settings`: an optional object, e.g. `{\"language\": \"en\"}`.\n\nA `tool_call` test carries exactly one of `history` or `input`, and which one it carries decides the agent it can be linked to.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n- `general`: judge a standalone, non-conversational input/output pair, graded by the linked evaluators. `{\"type\": \"general\"}`\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example, for a conversational agent. Swap `history` for `input` to aim it at a `general` agent:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\n`general` example:\n```json\n{\n  \"input\": \"Summarize this article: ...\",\n  \"evaluation\": {\"type\": \"general\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nReplaces the stored config. Omit to leave unchanged",
   ),
   evaluators: z.array(RoutersTestsEvaluatorRef$zodSchema).nullable().optional()
     .describe(
@@ -41,6 +43,6 @@ export const TestUpdate$zodSchema: z.ZodType<TestUpdate> = z.object({
     "New test name. Omit to leave unchanged",
   ),
   type: TestUpdateType$zodSchema.nullable().optional().describe(
-    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\n\nImmutable. Omit it, or send the current value",
+    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n\n\nImmutable. Omit it, or send the current value",
   ),
 });

package/src/models/traceingest.ts

@@ -10,11 +10,23 @@ import {
 import { TraceOutput, TraceOutput$zodSchema } from "./traceoutput.js";
 import { TraceTurn, TraceTurn$zodSchema } from "./traceturn.js";
 
+/**
+ * What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format
+ */
+export type Input = string | Array<TraceTurn>;
+
+export const Input$zodSchema: z.ZodType<Input> = z.union([
+  z.string(),
+  z.array(TraceTurn$zodSchema),
+]).describe(
+  "What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format",
+);
+
 export type TraceIngest = {
   agent_id: string;
   message_id?: string | null | undefined;
   conversation_id?: string | null | undefined;
-  input: Array<TraceTurn>;
+  input: string | Array<TraceTurn>;
   output: TraceOutput;
   metadata?: Array<TraceMetadataEntry> | null | undefined;
 };
@@ -26,8 +38,11 @@ export const TraceIngest$zodSchema: z.ZodType<TraceIngest> = z.object({
   conversation_id: z.string().nullable().optional().describe(
     "Your own ID for the conversation this turn belongs to, stored for reference only. Omit if you have none",
   ),
-  input: z.array(TraceTurn$zodSchema).describe(
-    "Conversation history up to the reported output, oldest turn first, in OpenAI chat format",
+  input: z.union([
+    z.string(),
+    z.array(TraceTurn$zodSchema),
+  ]).describe(
+    "What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format",
   ),
   message_id: z.string().nullable().optional().describe(
     "Your own ID for the last user message in `input`, stored for reference only. Omit if you have none",

package/src/models/tracetoolcall.ts

@@ -7,11 +7,15 @@ import * as z from "zod";
 export type TraceToolCall = {
   tool: string;
   arguments?: { [k: string]: any } | null | undefined;
+  output?: any | undefined;
 };
 
 export const TraceToolCall$zodSchema: z.ZodType<TraceToolCall> = z.object({
   arguments: z.record(z.string(), z.any()).nullable().optional().describe(
     "Argument values the agent passed to the tool. Omit when the call had none",
   ),
+  output: z.any().optional().describe(
+    "What the tool returned for this call. Any JSON value. Omit when you do not record it",
+  ),
   tool: z.string().describe("Name of the tool the agent called"),
 });