@dalmia/calibrate-mcp 0.0.39 → 0.0.41

package/src/models/bulktestitem.ts

@@ -15,20 +15,25 @@ import {
 
 export type BulkTestItem = {
   name: string;
-  conversation_history: Array<ChatMessage>;
+  conversation_history?: Array<ChatMessage> | null | undefined;
+  input?: string | null | undefined;
   evaluators?: Array<RoutersTestsEvaluatorRef> | null | undefined;
   tool_calls?: Array<ExpectedToolCall> | null | undefined;
   inputs?: { [k: string]: any } | null | undefined;
 };
 
 export const BulkTestItem$zodSchema: z.ZodType<BulkTestItem> = z.object({
-  conversation_history: z.array(ChatMessage$zodSchema).describe(
-    "Ordered messages ending at the user turn the agent should answer",
-  ),
+  conversation_history: z.array(ChatMessage$zodSchema).nullable().optional()
+    .describe(
+      "Ordered messages ending at the user turn the agent should answer. **Required for `response` and `conversation` batches**, and for `tool_call` batches aimed at a conversational agent",
+    ),
   evaluators: z.array(RoutersTestsEvaluatorRef$zodSchema).nullable().optional()
     .describe(
-      "Evaluators to link. Used by `response` and `conversation` tests",
+      "Evaluators to link. Used by `response`, `conversation`, and `general` tests",
     ),
+  input: z.string().nullable().optional().describe(
+    "Standalone prompt with no conversation around it. **Required for `general` batches**, and for `tool_call` batches aimed at a general agent",
+  ),
   inputs: z.record(z.string(), z.any()).nullable().optional().describe(
     "Extra request fields for this test, overriding the agent's `default_inputs` per key",
   ),

package/src/models/bulktestupload.ts

@@ -14,6 +14,7 @@ import { BulkTestItem, BulkTestItem$zodSchema } from "./bulktestitem.js";
  * - `response`: judges the generated reply
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
+ * - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
  *
  * Applied to every test in the batch
  */
@@ -21,6 +22,7 @@ export const BulkTestUploadType = {
   Response: "response",
   ToolCall: "tool_call",
   Conversation: "conversation",
+  General: "general",
 } as const;
 /**
  * What the test judges:
@@ -30,6 +32,7 @@ export const BulkTestUploadType = {
  * - `response`: judges the generated reply
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
+ * - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
  *
  * Applied to every test in the batch
  */
@@ -39,8 +42,9 @@ export const BulkTestUploadType$zodSchema = z.enum([
   "response",
   "tool_call",
   "conversation",
+  "general",
 ]).describe(
-  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\n\nApplied to every test in the batch",
+  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n\n\nApplied to every test in the batch",
 );
 
 export type BulkTestUpload = {
@@ -61,6 +65,6 @@ export const BulkTestUpload$zodSchema: z.ZodType<BulkTestUpload> = z.object({
     "Test items to create, at most 500 per request, with names unique within the batch",
   ),
   type: BulkTestUploadType$zodSchema.describe(
-    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\n\nApplied to every test in the batch",
+    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n\n\nApplied to every test in the batch",
   ),
 });

package/src/models/getagenttestrunsagenttestsagentagentuuidrunsgetop.ts

@@ -49,6 +49,7 @@ export type GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetRequest = {
   type?: GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetType | null | undefined;
   status?: TaskStatus | null | undefined;
   has_failures?: boolean | null | undefined;
+  around?: string | null | undefined;
   limit?: number | null | undefined;
   offset?: number | undefined;
   xAPIKey?: string | null | undefined;
@@ -57,6 +58,9 @@ export type GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetRequest = {
 export const GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetRequest$zodSchema:
   z.ZodType<GetAgentTestRunsAgentTestsAgentAgentUuidRunsGetRequest> = z.object({
     agent_uuid: z.string().describe("Agent whose test runs to list"),
+    around: z.string().describe(
+      "ID of a run to jump to, returning the page that contains it instead of the page at `offset`",
+    ).nullable().optional(),
     has_failures: z.boolean().describe(
       "Filter by whether the run has any failing test case or model. `true` returns only runs with failures (or errors), `false` only clean runs. Omit for both",
     ).nullable().optional(),

package/src/models/taskagreementresponse.ts

@@ -25,7 +25,7 @@ export const TaskAgreementResponse$zodSchema: z.ZodType<TaskAgreementResponse> =
       bucket: z.string().describe("Time bucket used for the trend series"),
       days: z.int().describe("Length of the trailing window in days"),
       evaluators: z.array(z.record(z.string(), z.any())).describe(
-        "Agreement between each evaluator and the annotators, plus that evaluator's own results, one entry per linked evaluator",
+        "Agreement between each evaluator and the annotators, plus that evaluator's own results and the human results, one entry per linked evaluator",
       ),
       human_human: HumanAgreementBlock$zodSchema,
       task_id: z.string().describe("ID of the task"),

package/src/models/testcreate.ts

@@ -17,11 +17,13 @@ import {
  * - `response`: judges the generated reply
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
+ * - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
  */
 export const TestCreateType = {
   Response: "response",
   ToolCall: "tool_call",
   Conversation: "conversation",
+  General: "general",
 } as const;
 /**
  * What the test judges:
@@ -31,6 +33,7 @@ export const TestCreateType = {
  * - `response`: judges the generated reply
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
+ * - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
  */
 export type TestCreateType = ClosedEnum<typeof TestCreateType>;
 
@@ -38,8 +41,9 @@ export const TestCreateType$zodSchema = z.enum([
   "response",
   "tool_call",
   "conversation",
+  "general",
 ]).describe(
-  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
+  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n",
 );
 
 export type TestCreate = {
@@ -51,7 +55,7 @@ export type TestCreate = {
 
 export const TestCreate$zodSchema: z.ZodType<TestCreate> = z.object({
   config: z.record(z.string(), z.any()).nullable().optional().describe(
-    "The calibrate test config. Three top-level keys.\n\n- `history`: the required conversation up to the agent's turn. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `evaluation`: the required `{type, ...}`, where `type` matches the test's `type` below.\n- `settings`: an optional object, e.g. `{\"language\": \"en\"}`.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nOmit to create the test with no config and fill it in later via update",
+    "The calibrate test config.\n\n- `history`: the conversation up to the agent's turn, required for `response` and `conversation` tests and for a `tool_call` test aimed at a conversational agent. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `input`: a standalone prompt with no conversation around it, required for `general` tests and for a `tool_call` test aimed at a `general` agent. A string, not a conversation.\n- `evaluation`: the required `{type, ...}`, where `type` matches the test's `type` below.\n- `settings`: an optional object, e.g. `{\"language\": \"en\"}`.\n\nA `tool_call` test carries exactly one of `history` or `input`, and which one it carries decides the agent it can be linked to.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n- `general`: judge a standalone, non-conversational input/output pair, graded by the linked evaluators. `{\"type\": \"general\"}`\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example, for a conversational agent. Swap `history` for `input` to aim it at a `general` agent:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\n`general` example:\n```json\n{\n  \"input\": \"Summarize this article: ...\",\n  \"evaluation\": {\"type\": \"general\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nOmit to create the test with no config and fill it in later via update",
   ),
   evaluators: z.array(RoutersTestsEvaluatorRef$zodSchema).nullable().optional()
     .describe(
@@ -59,6 +63,6 @@ export const TestCreate$zodSchema: z.ZodType<TestCreate> = z.object({
     ),
   name: z.string().describe("Name of the test, unique within the workspace"),
   type: TestCreateType$zodSchema.describe(
-    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
+    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n",
   ),
 });

package/src/models/testlistresponse.ts

@@ -14,11 +14,13 @@ import { TestListConfig, TestListConfig$zodSchema } from "./testlistconfig.js";
  * - `response`: judges the generated reply
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
+ * - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
  */
 export const TestListResponseType = {
   Response: "response",
   ToolCall: "tool_call",
   Conversation: "conversation",
+  General: "general",
 } as const;
 /**
  * What the test judges:
@@ -28,6 +30,7 @@ export const TestListResponseType = {
  * - `response`: judges the generated reply
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
+ * - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
  */
 export type TestListResponseType = ClosedEnum<typeof TestListResponseType>;
 
@@ -35,8 +38,9 @@ export const TestListResponseType$zodSchema = z.enum([
   "response",
   "tool_call",
   "conversation",
+  "general",
 ]).describe(
-  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
+  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n",
 );
 
 /**
@@ -63,7 +67,7 @@ export const TestListResponse$zodSchema: z.ZodType<TestListResponse> = z.object(
     created_at: z.string().describe("When the test was created (ISO 8601 UTC)"),
     name: z.string().describe("Name of the test"),
     type: TestListResponseType$zodSchema.describe(
-      "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
+      "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n",
     ),
     updated_at: z.string().describe(
       "When the test was last updated (ISO 8601 UTC)",

package/src/models/testresponse.ts

@@ -13,11 +13,13 @@ import { ClosedEnum } from "../types/enums.js";
  * - `response`: judges the generated reply
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
+ * - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
  */
 export const TestResponseType = {
   Response: "response",
   ToolCall: "tool_call",
   Conversation: "conversation",
+  General: "general",
 } as const;
 /**
  * What the test judges:
@@ -27,6 +29,7 @@ export const TestResponseType = {
  * - `response`: judges the generated reply
  * - `tool_call`: diffs the generated tool calls
  * - `conversation`: judges the full conversation
+ * - `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)
  */
 export type TestResponseType = ClosedEnum<typeof TestResponseType>;
 
@@ -34,8 +37,9 @@ export const TestResponseType$zodSchema = z.enum([
   "response",
   "tool_call",
   "conversation",
+  "general",
 ]).describe(
-  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
+  "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n",
 );
 
 export type TestResponse = {
@@ -58,7 +62,7 @@ export const TestResponse$zodSchema: z.ZodType<TestResponse> = z.object({
   ),
   name: z.string().describe("Name of the test"),
   type: TestResponseType$zodSchema.describe(
-    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n",
+    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n",
   ),
   updated_at: z.string().describe(
     "When the test was last updated (ISO 8601 UTC)",

package/src/models/testupdate.ts

@@ -13,6 +13,7 @@ export const TestUpdateType = {
   Response: "response",
   ToolCall: "tool_call",
   Conversation: "conversation",
+  General: "general",
 } as const;
 export type TestUpdateType = ClosedEnum<typeof TestUpdateType>;
 
@@ -20,6 +21,7 @@ export const TestUpdateType$zodSchema = z.enum([
   "response",
   "tool_call",
   "conversation",
+  "general",
 ]);
 
 export type TestUpdate = {
@@ -31,7 +33,7 @@ export type TestUpdate = {
 
 export const TestUpdate$zodSchema: z.ZodType<TestUpdate> = z.object({
   config: z.record(z.string(), z.any()).nullable().optional().describe(
-    "The calibrate test config. Three top-level keys.\n\n- `history`: the required conversation up to the agent's turn. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `evaluation`: the required `{type, ...}`, where `type` matches the test's `type` below.\n- `settings`: an optional object, e.g. `{\"language\": \"en\"}`.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nReplaces the stored config. Omit to leave unchanged",
+    "The calibrate test config.\n\n- `history`: the conversation up to the agent's turn, required for `response` and `conversation` tests and for a `tool_call` test aimed at a conversational agent. Each item is `{role, content}` with `role` one of `user`, `assistant`, `tool`. A `tool` message also carries `tool_call_id` and `name`.\n- `input`: a standalone prompt with no conversation around it, required for `general` tests and for a `tool_call` test aimed at a `general` agent. A string, not a conversation.\n- `evaluation`: the required `{type, ...}`, where `type` matches the test's `type` below.\n- `settings`: an optional object, e.g. `{\"language\": \"en\"}`.\n\nA `tool_call` test carries exactly one of `history` or `input`, and which one it carries decides the agent it can be linked to.\n\n`evaluation` by test type:\n- `response`: judge the agent's reply, graded by the linked evaluators. `{\"type\": \"response\"}`\n- `conversation`: append the reply and judge the whole conversation. `{\"type\": \"conversation\"}`\n- `tool_call`: diff the agent's tool calls against expected ones. Add `tool_calls`, a list of `{tool, arguments, accept_any_arguments?}`.\n- `general`: judge a standalone, non-conversational input/output pair, graded by the linked evaluators. `{\"type\": \"general\"}`\n\nFor `tool_call`, each expected argument value is one of:\n- `{\"match_type\": \"exact\", \"value\": <any>}`: must equal `value`\n- `{\"match_type\": \"llm_judge\", \"criteria\": \"...\"}`: judged against the criteria\n- `{\"match_type\": \"any\"}`: any value, only checks the argument was passed\n\n`response` / `conversation` example:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"What is your return policy?\"}],\n  \"evaluation\": {\"type\": \"response\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\n`tool_call` example, for a conversational agent. Swap `history` for `input` to aim it at a `general` agent:\n```json\n{\n  \"history\": [{\"role\": \"user\", \"content\": \"Book room 101 for tomorrow\"}],\n  \"evaluation\": {\n    \"type\": \"tool_call\",\n    \"tool_calls\": [\n      {\n        \"tool\": \"book_room\",\n        \"arguments\": {\n          \"room\": {\"match_type\": \"exact\", \"value\": \"101\"},\n          \"date\": {\"match_type\": \"llm_judge\", \"criteria\": \"tomorrow's date\"}\n        },\n        \"accept_any_arguments\": false\n      }\n    ]\n  }\n}\n```\n\n`general` example:\n```json\n{\n  \"input\": \"Summarize this article: ...\",\n  \"evaluation\": {\"type\": \"general\"},\n  \"settings\": {\"language\": \"en\"}\n}\n```\n\nEvaluators are linked via the separate `evaluators` field, not inside `config`.\n\nReplaces the stored config. Omit to leave unchanged",
   ),
   evaluators: z.array(RoutersTestsEvaluatorRef$zodSchema).nullable().optional()
     .describe(
@@ -41,6 +43,6 @@ export const TestUpdate$zodSchema: z.ZodType<TestUpdate> = z.object({
     "New test name. Omit to leave unchanged",
   ),
   type: TestUpdateType$zodSchema.nullable().optional().describe(
-    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n\n\nImmutable. Omit it, or send the current value",
+    "What the test judges:\n\n- `response`: judges the generated reply\n- `tool_call`: diffs the generated tool calls\n- `conversation`: judges the full conversation\n- `general`: judges a single plain-text input/output pair with no conversation involved (e.g. summarization, extraction, classification)\n\n\nImmutable. Omit it, or send the current value",
   ),
 });

package/src/models/traceingest.ts

@@ -10,11 +10,23 @@ import {
 import { TraceOutput, TraceOutput$zodSchema } from "./traceoutput.js";
 import { TraceTurn, TraceTurn$zodSchema } from "./traceturn.js";
 
+/**
+ * What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format
+ */
+export type Input = string | Array<TraceTurn>;
+
+export const Input$zodSchema: z.ZodType<Input> = z.union([
+  z.string(),
+  z.array(TraceTurn$zodSchema),
+]).describe(
+  "What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format",
+);
+
 export type TraceIngest = {
   agent_id: string;
   message_id?: string | null | undefined;
   conversation_id?: string | null | undefined;
-  input: Array<TraceTurn>;
+  input: string | Array<TraceTurn>;
   output: TraceOutput;
   metadata?: Array<TraceMetadataEntry> | null | undefined;
 };
@@ -26,8 +38,11 @@ export const TraceIngest$zodSchema: z.ZodType<TraceIngest> = z.object({
   conversation_id: z.string().nullable().optional().describe(
     "Your own ID for the conversation this turn belongs to, stored for reference only. Omit if you have none",
   ),
-  input: z.array(TraceTurn$zodSchema).describe(
-    "Conversation history up to the reported output, oldest turn first, in OpenAI chat format",
+  input: z.union([
+    z.string(),
+    z.array(TraceTurn$zodSchema),
+  ]).describe(
+    "What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format",
   ),
   message_id: z.string().nullable().optional().describe(
     "Your own ID for the last user message in `input`, stored for reference only. Omit if you have none",

package/src/models/tracetoolcall.ts

@@ -7,11 +7,15 @@ import * as z from "zod";
 export type TraceToolCall = {
   tool: string;
   arguments?: { [k: string]: any } | null | undefined;
+  output?: any | undefined;
 };
 
 export const TraceToolCall$zodSchema: z.ZodType<TraceToolCall> = z.object({
   arguments: z.record(z.string(), z.any()).nullable().optional().describe(
     "Argument values the agent passed to the tool. Omit when the call had none",
   ),
+  output: z.any().optional().describe(
+    "What the tool returned for this call. Any JSON value. Omit when you do not record it",
+  ),
   tool: z.string().describe("Name of the tool the agent called"),
 });