@dalmia/calibrate-mcp 0.0.68 → 0.0.69

package/esm/landing-page.js

@@ -915,7 +915,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
         <h1>Instructions</h1>
         <p>One-click installation for Claude Desktop users</p>
         <div class="instruction-item">
-          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.68/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
+          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.69/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
             📥 Download MCP Bundle
           </a>
         </div>

package/esm/lib/config.d.ts

@@ -35,8 +35,8 @@ export declare function serverURLFromOptions(options: SDKOptions): URL | null;
 export declare const SDK_METADATA: {
     readonly language: "typescript";
     readonly openapiDocVersion: "0.1.0";
-    readonly sdkVersion: "0.0.68";
+    readonly sdkVersion: "0.0.69";
     readonly genVersion: "2.915.1";
-    readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.68 2.915.1 0.1.0 @dalmia/calibrate-mcp";
+    readonly userAgent: "speakeasy-sdk/mcp-typescript 0.0.69 2.915.1 0.1.0 @dalmia/calibrate-mcp";
 };
 //# sourceMappingURL=config.d.ts.map

package/esm/lib/config.js

@@ -27,8 +27,8 @@ export function serverURLFromOptions(options) {
 export const SDK_METADATA = {
     language: "typescript",
     openapiDocVersion: "0.1.0",
-    sdkVersion: "0.0.68",
+    sdkVersion: "0.0.69",
     genVersion: "2.915.1",
-    userAgent: "speakeasy-sdk/mcp-typescript 0.0.68 2.915.1 0.1.0 @dalmia/calibrate-mcp",
+    userAgent: "speakeasy-sdk/mcp-typescript 0.0.69 2.915.1 0.1.0 @dalmia/calibrate-mcp",
 };
 //# sourceMappingURL=config.js.map

package/esm/mcp-server/mcp-server.js

@@ -18,7 +18,7 @@ const routes = buildRouteMap({
 export const app = buildApplication(routes, {
     name: "mcp",
     versionInfo: {
-        currentVersion: "0.0.68",
+        currentVersion: "0.0.69",
     },
 });
 run(app, process.argv.slice(2), buildContext(process));

package/esm/mcp-server/server.js

@@ -51,7 +51,7 @@ import { tool$tracesCreate } from "./tools/tracesCreate.js";
 export function createMCPServer(deps) {
     const server = new McpServer({
         name: "CalibrateMcp",
-        version: "0.0.68",
+        version: "0.0.69",
     });
     const getClient = deps.getSDK || (() => new CalibrateMcpCore({
         security: deps.security,

package/esm/models/annotationitempayload.d.ts

@@ -1,6 +1,9 @@
 import * as z from "zod";
+import { ItemMetadataEntry } from "./itemmetadataentry.js";
+import { ItemPayload } from "./itempayload.js";
 export type AnnotationItemPayload = {
-    payload?: any | undefined;
+    payload: ItemPayload;
+    metadata?: Array<ItemMetadataEntry> | null | undefined;
     annotations?: {
         [k: string]: any;
     } | null | undefined;

package/esm/models/annotationitempayload.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"annotationitempayload.d.ts","sourceRoot":"","sources":["../../src/models/annotationitempayload.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,MAAM,MAAM,qBAAqB,GAAG;IAClC,OAAO,CAAC,EAAE,GAAG,GAAG,SAAS,CAAC;IAC1B,WAAW,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,iBAAiB,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;CAC7D,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAYzE,CAAC"}
+{"version":3,"file":"annotationitempayload.d.ts","sourceRoot":"","sources":["../../src/models/annotationitempayload.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EACL,iBAAiB,EAElB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AAEtE,MAAM,MAAM,qBAAqB,GAAG;IAClC,OAAO,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,KAAK,CAAC,iBAAiB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,WAAW,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACtD,iBAAiB,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;CAC7D,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAgBzE,CAAC"}

package/esm/models/annotationitempayload.js

@@ -2,10 +2,14 @@
  * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
  */
 import * as z from "zod";
+import { ItemMetadataEntry$zodSchema, } from "./itemmetadataentry.js";
+import { ItemPayload$zodSchema } from "./itempayload.js";
 export const AnnotationItemPayload$zodSchema = z.object({
     annotations: z.record(z.string(), z.any()).nullable().optional().describe("Human annotations to seed, keyed by evaluator ID. Each evaluator ID must be linked to the task. Put the judgement in `value`, a bool for binary or a number for rating, with optional `reasoning`. Requires `annotator_id`"),
     evaluator_results: z.record(z.string(), z.any()).nullable().optional()
         .describe("Evaluator scores to record, keyed by evaluator ID. Each evaluator ID must be linked to the task. Put the score in `value`, a bool for binary or a number within the scale for rating, with optional `reasoning` and `version_number`. Omit `version_number` to record against the evaluator's live version. An item recording a tool call takes the tool-call evaluator and no other, and every other item takes any evaluator except that one"),
-    payload: z.any().optional().describe("The item's payload. `name` is required and unique within the task for every type. The other fields depend on the task `type`:\n\n- `stt`: `name`, `reference_transcript`, `predicted_transcript`\n- `llm`: `name`, `chat_history` (list of `{role, content}` turns ending at the user turn), `agent_response` (the reply to judge), optional `evaluator_variables`\n- `llm-general`: `name`, `input`, `output`, optional `evaluator_variables`\n- `conversation`: `name`, `transcript` (list of `{role, content}` turns), optional `evaluator_variables`\n\n`evaluator_variables` maps an evaluator ID to that evaluator's `{{\"{{\"}}variable}}` values for this item"),
+    metadata: z.array(ItemMetadataEntry$zodSchema).nullable().optional()
+        .describe("Key-value pairs stored with the item and shown beside it on the labelling form. Omit if you have none"),
+    payload: ItemPayload$zodSchema.describe("What an annotation item holds. Which fields apply depends on the task type"),
 });
 //# sourceMappingURL=annotationitempayload.js.map

package/esm/models/annotationitempayload.js.map

@@ -1 +1 @@
-{"version":3,"file":"annotationitempayload.js","sourceRoot":"","sources":["../../src/models/annotationitempayload.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAQzB,MAAM,CAAC,MAAM,+BAA+B,GAC1C,CAAC,CAAC,MAAM,CAAC;IACP,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,4NAA4N,CAC7N;IACD,iBAAiB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACnE,QAAQ,CACP,gbAAgb,CACjb;IACH,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClC,woBAAwoB,CACzoB;CACF,CAAC,CAAC"}
+{"version":3,"file":"annotationitempayload.js","sourceRoot":"","sources":["../../src/models/annotationitempayload.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAEL,2BAA2B,GAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAStE,MAAM,CAAC,MAAM,+BAA+B,GAC1C,CAAC,CAAC,MAAM,CAAC;IACP,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvE,4NAA4N,CAC7N;IACD,iBAAiB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACnE,QAAQ,CACP,gbAAgb,CACjb;IACH,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACjE,QAAQ,CACP,uGAAuG,CACxG;IACH,OAAO,EAAE,qBAAqB,CAAC,QAAQ,CACrC,4EAA4E,CAC7E;CACF,CAAC,CAAC"}

package/esm/models/chatturn.d.ts

@@ -0,0 +1,14 @@
+import * as z from "zod";
+/**
+ * One message in a conversation, in OpenAI chat format
+ */
+export type ChatTurn = {
+    role: string;
+    content?: string | null | undefined;
+    AdditionalProperties?: {
+        [k: string]: any;
+    };
+};
+export declare const ChatTurn$zodSchema: z.ZodType<ChatTurn>;
+export declare const ChatTurn$zodSchemaOutbound: z.ZodType<any>;
+//# sourceMappingURL=chatturn.d.ts.map

package/esm/models/chatturn.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chatturn.d.ts","sourceRoot":"","sources":["../../src/models/chatturn.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACpC,oBAAoB,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;CAC7C,CAAC;AAEF,eAAO,MAAM,kBAAkB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAalD,CAAC;AAEF,eAAO,MAAM,0BAA0B,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAYrD,CAAC"}

package/esm/models/chatturn.js

@@ -0,0 +1,21 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+import * as z from "zod";
+import { collectExtraKeys as collectExtraKeys$ } from "../lib/schemas.js";
+export const ChatTurn$zodSchema = collectExtraKeys$(z.object({
+    content: z.string().nullable().optional().describe("Message text. Omit for turns that only carry tool calls"),
+    role: z.string().describe("Message author role in the conversation history"),
+}).describe("One message in a conversation, in OpenAI chat format").catchall(z.any()), "AdditionalProperties", true);
+export const ChatTurn$zodSchemaOutbound = z.object({
+    AdditionalProperties: z.record(z.string(), z.any()).optional(),
+    content: z.nullable(z.string()).optional(),
+    role: z.string(),
+}).describe("One message in a conversation, in OpenAI chat format").transform((v) => {
+    const { AdditionalProperties, ...rest } = v;
+    return {
+        ...rest,
+        ...AdditionalProperties,
+    };
+});
+//# sourceMappingURL=chatturn.js.map

package/esm/models/chatturn.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chatturn.js","sourceRoot":"","sources":["../../src/models/chatturn.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,gBAAgB,IAAI,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAW1E,MAAM,CAAC,MAAM,kBAAkB,GAAwB,iBAAiB,CACtE,CAAC,CAAC,MAAM,CAAC;IACP,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAChD,yDAAyD,CAC1D;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,iDAAiD,CAClD;CACF,CAAC,CAAC,QAAQ,CAAC,sDAAsD,CAAC,CAAC,QAAQ,CAC1E,CAAC,CAAC,GAAG,EAAE,CACR,EACD,sBAAsB,EACtB,IAAI,CACL,CAAC;AAEF,MAAM,CAAC,MAAM,0BAA0B,GAAmB,CAAC,CAAC,MAAM,CAAC;IACjE,oBAAoB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC9D,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;CACjB,CAAC,CAAC,QAAQ,CAAC,sDAAsD,CAAC,CAAC,SAAS,CAC3E,CAAC,CAAC,EAAE,EAAE;IACJ,MAAM,EAAE,oBAAoB,EAAE,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC;IAC5C,OAAO;QACL,GAAG,IAAI;QACP,GAAG,oBAAoB;KACxB,CAAC;AACJ,CAAC,CACF,CAAC"}

package/esm/models/itemmetadataentry.d.ts

@@ -0,0 +1,10 @@
+import * as z from "zod";
+/**
+ * One key-value pair stored with an item and shown beside it on the labelling form
+ */
+export type ItemMetadataEntry = {
+    key: string;
+    value: string;
+};
+export declare const ItemMetadataEntry$zodSchema: z.ZodType<ItemMetadataEntry>;
+//# sourceMappingURL=itemmetadataentry.d.ts.map

package/esm/models/itemmetadataentry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"itemmetadataentry.d.ts","sourceRoot":"","sources":["../../src/models/itemmetadataentry.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAE/D,eAAO,MAAM,2BAA2B,EAAE,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAMlE,CAAC"}

package/esm/models/itemmetadataentry.js

@@ -0,0 +1,10 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+import * as z from "zod";
+export const ItemMetadataEntry$zodSchema = z
+    .object({
+    key: z.string().describe("Name of the metadata entry"),
+    value: z.string().describe("Value of the metadata entry"),
+}).describe("One key-value pair stored with an item and shown beside it on the labelling form");
+//# sourceMappingURL=itemmetadataentry.js.map

package/esm/models/itemmetadataentry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"itemmetadataentry.js","sourceRoot":"","sources":["../../src/models/itemmetadataentry.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAOzB,MAAM,CAAC,MAAM,2BAA2B,GAAiC,CAAC;KACvE,MAAM,CAAC;IACN,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4BAA4B,CAAC;IACtD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;CAC1D,CAAC,CAAC,QAAQ,CACT,kFAAkF,CACnF,CAAC"}

package/esm/models/itempayload.d.ts

@@ -0,0 +1,29 @@
+import * as z from "zod";
+import { ChatTurn } from "./chatturn.js";
+import { ToolCall } from "./toolcall.js";
+/**
+ * What an annotation item holds. Which fields apply depends on the task type
+ */
+export type ItemPayload = {
+    name: string;
+    description?: string | null | undefined;
+    reference_transcript?: string | null | undefined;
+    predicted_transcript?: string | null | undefined;
+    text?: string | null | undefined;
+    audio_path?: string | null | undefined;
+    chat_history?: Array<ChatTurn> | null | undefined;
+    agent_response?: string | null | undefined;
+    input?: string | null | undefined;
+    output?: string | null | undefined;
+    transcript?: Array<ChatTurn> | null | undefined;
+    tool_calls?: Array<ToolCall> | null | undefined;
+    expected_tool_calls?: Array<ToolCall> | null | undefined;
+    actual_tool_calls?: Array<ToolCall> | null | undefined;
+    evaluator_variables?: {
+        [k: string]: {
+            [k: string]: any;
+        };
+    } | null | undefined;
+};
+export declare const ItemPayload$zodSchema: z.ZodType<ItemPayload>;
+//# sourceMappingURL=itempayload.d.ts.map

package/esm/models/itempayload.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"itempayload.d.ts","sourceRoot":"","sources":["../../src/models/itempayload.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,QAAQ,EAAsB,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAsB,MAAM,eAAe,CAAC;AAE7D;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACxC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,YAAY,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAClD,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC3C,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,UAAU,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAChD,UAAU,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAChD,mBAAmB,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACzD,iBAAiB,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACvD,mBAAmB,CAAC,EAChB;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG;YAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;SAAE,CAAA;KAAE,GACrC,IAAI,GACJ,SAAS,CAAC;CACf,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAkDxD,CAAC"}

package/esm/models/itempayload.js

@@ -0,0 +1,26 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+import * as z from "zod";
+import { ChatTurn$zodSchema } from "./chatturn.js";
+import { ToolCall$zodSchema } from "./toolcall.js";
+export const ItemPayload$zodSchema = z.object({
+    actual_tool_calls: z.array(ToolCall$zodSchema).nullable().optional().describe("Tool calls the agent made. Sending this marks the item as a tool call for a person to judge. For `llm` and `llm-general` tasks"),
+    agent_response: z.string().nullable().optional().describe("The reply to judge. An empty string when the reply was a tool call. For `llm` tasks"),
+    audio_path: z.string().nullable().optional().describe("Storage key or `s3://` URI of the audio clip. For `tts` tasks"),
+    chat_history: z.array(ChatTurn$zodSchema).nullable().optional().describe("The conversation up to the reply being judged, ending at the user turn. For `llm` tasks"),
+    description: z.string().nullable().optional().describe("A description of the item. For every task type"),
+    evaluator_variables: z.record(z.string(), z.record(z.string(), z.any()))
+        .nullable().optional().describe("Values for each evaluator's `{{\"{{\"}}variable}}` placeholders, keyed by evaluator ID. For `llm`, `llm-general` and `conversation` tasks"),
+    expected_tool_calls: z.array(ToolCall$zodSchema).nullable().optional()
+        .describe("Tool calls the agent should have made. Sending this marks the item as a tool call for a person to judge. For `llm` and `llm-general` tasks"),
+    input: z.string().nullable().optional().describe("The standalone input the agent was given. For `llm-general` tasks"),
+    name: z.string().describe("Name of the item, unique within the task. For every task type"),
+    output: z.string().nullable().optional().describe("The output to judge. For `llm-general` tasks"),
+    predicted_transcript: z.string().nullable().optional().describe("The transcript to judge. For `stt` tasks"),
+    reference_transcript: z.string().nullable().optional().describe("The correct transcript. For `stt` tasks"),
+    text: z.string().nullable().optional().describe("The text that was spoken. For `tts` tasks"),
+    tool_calls: z.array(ToolCall$zodSchema).nullable().optional().describe("Tool calls the agent made in place of a text reply. With no text reply, sending this marks the item as a tool call for a person to judge. For `llm` and `llm-general` tasks"),
+    transcript: z.array(ChatTurn$zodSchema).nullable().optional().describe("The full conversation to judge. For `conversation` tasks"),
+}).describe("What an annotation item holds. Which fields apply depends on the task type");
+//# sourceMappingURL=itempayload.js.map

package/esm/models/itempayload.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"itempayload.js","sourceRoot":"","sources":["../../src/models/itempayload.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAY,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAY,kBAAkB,EAAE,MAAM,eAAe,CAAC;AA0B7D,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC3E,gIAAgI,CACjI;IACD,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvD,qFAAqF,CACtF;IACD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnD,+DAA+D,CAChE;IACD,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtE,yFAAyF,CAC1F;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpD,gDAAgD,CACjD;IACD,mBAAmB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;SACrE,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7B,2IAA2I,CAC5I;IACH,mBAAmB,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACnE,QAAQ,CACP,4IAA4I,CAC7I;IACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC9C,mEAAmE,CACpE;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,+DAA+D,CAChE;IACD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC/C,8CAA8C,CAC/C;IACD,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7D,0CAA0C,CAC3C;IACD,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7D,yCAAyC,CAC1C;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC7C,2CAA2C,CAC5C;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpE,6KAA6K,CAC9K;IACD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpE,0DAA0D,CAC3D;CACF,CAAC,CAAC,QAAQ,CACT,4EAA4E,CAC7E,CAAC"}

package/esm/models/itemupdatepayload.d.ts

@@ -1,7 +1,10 @@
 import * as z from "zod";
+import { ItemMetadataEntry } from "./itemmetadataentry.js";
+import { ItemPayload } from "./itempayload.js";
 export type ItemUpdatePayload = {
     uuid: string;
-    payload?: any | undefined;
+    payload: ItemPayload;
+    metadata?: Array<ItemMetadataEntry> | null | undefined;
 };
 export declare const ItemUpdatePayload$zodSchema: z.ZodType<ItemUpdatePayload>;
 //# sourceMappingURL=itemupdatepayload.d.ts.map

package/esm/models/itemupdatepayload.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"itemupdatepayload.d.ts","sourceRoot":"","sources":["../../src/models/itemupdatepayload.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,MAAM,MAAM,iBAAiB,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,GAAG,GAAG,SAAS,CAAA;CAAE,CAAC;AAE5E,eAAO,MAAM,2BAA2B,EAAE,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAMjE,CAAC"}
+{"version":3,"file":"itemupdatepayload.d.ts","sourceRoot":"","sources":["../../src/models/itemupdatepayload.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EACL,iBAAiB,EAElB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,WAAW,EAAyB,MAAM,kBAAkB,CAAC;AAEtE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,KAAK,CAAC,iBAAiB,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;CACxD,CAAC;AAEF,eAAO,MAAM,2BAA2B,EAAE,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAUjE,CAAC"}

package/esm/models/itemupdatepayload.js

@@ -2,9 +2,13 @@
  * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
  */
 import * as z from "zod";
+import { ItemMetadataEntry$zodSchema, } from "./itemmetadataentry.js";
+import { ItemPayload$zodSchema } from "./itempayload.js";
 export const ItemUpdatePayload$zodSchema = z
     .object({
-    payload: z.any().optional().describe("The item's new payload. `payload['name']` is still required and must be unique within the task"),
+    metadata: z.array(ItemMetadataEntry$zodSchema).nullable().optional()
+        .describe("Key-value pairs stored with the item and shown beside it on the labelling form. Omit to leave the stored pairs as they are. Send an empty list to clear them"),
+    payload: ItemPayload$zodSchema.describe("What an annotation item holds. Which fields apply depends on the task type"),
     uuid: z.string().describe("ID of the item to update"),
 });
 //# sourceMappingURL=itemupdatepayload.js.map

package/esm/models/itemupdatepayload.js.map

@@ -1 +1 @@
-{"version":3,"file":"itemupdatepayload.js","sourceRoot":"","sources":["../../src/models/itemupdatepayload.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAIzB,MAAM,CAAC,MAAM,2BAA2B,GAAiC,CAAC;KACvE,MAAM,CAAC;IACN,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAClC,gGAAgG,CACjG;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0BAA0B,CAAC;CACtD,CAAC,CAAC"}
+{"version":3,"file":"itemupdatepayload.js","sourceRoot":"","sources":["../../src/models/itemupdatepayload.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAEL,2BAA2B,GAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAe,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAQtE,MAAM,CAAC,MAAM,2BAA2B,GAAiC,CAAC;KACvE,MAAM,CAAC;IACN,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SACjE,QAAQ,CACP,8JAA8J,CAC/J;IACH,OAAO,EAAE,qBAAqB,CAAC,QAAQ,CACrC,4EAA4E,CAC7E;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0BAA0B,CAAC;CACtD,CAAC,CAAC"}

package/esm/models/toolcall.d.ts

@@ -0,0 +1,14 @@
+import * as z from "zod";
+/**
+ * One call to a tool
+ */
+export type ToolCall = {
+    tool: string;
+    arguments?: {
+        [k: string]: any;
+    } | null | undefined;
+    output?: any | undefined;
+    accept_any_arguments?: boolean | null | undefined;
+};
+export declare const ToolCall$zodSchema: z.ZodType<ToolCall>;
+//# sourceMappingURL=toolcall.d.ts.map

package/esm/models/toolcall.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"toolcall.d.ts","sourceRoot":"","sources":["../../src/models/toolcall.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAC;IACpD,MAAM,CAAC,EAAE,GAAG,GAAG,SAAS,CAAC;IACzB,oBAAoB,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;CACnD,CAAC;AAEF,eAAO,MAAM,kBAAkB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAWlB,CAAC"}

package/esm/models/toolcall.js

@@ -0,0 +1,11 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+import * as z from "zod";
+export const ToolCall$zodSchema = z.object({
+    accept_any_arguments: z.boolean().nullable().optional().describe("When `true`, only the tool name had to match and `arguments` was ignored. Carried over from a tool-call test's expected calls"),
+    arguments: z.record(z.string(), z.any()).nullable().optional().describe("Argument values the agent passed to the tool. Omit when the call had none"),
+    output: z.any().optional().describe("What the tool returned for this call. Any JSON value. Omit when you do not record it"),
+    tool: z.string().describe("Name of the tool the agent called"),
+}).describe("One call to a tool");
+//# sourceMappingURL=toolcall.js.map

package/esm/models/toolcall.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"toolcall.js","sourceRoot":"","sources":["../../src/models/toolcall.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAYzB,MAAM,CAAC,MAAM,kBAAkB,GAAwB,CAAC,CAAC,MAAM,CAAC;IAC9D,oBAAoB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC9D,+HAA+H,CAChI;IACD,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACrE,2EAA2E,CAC5E;IACD,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACjC,sFAAsF,CACvF;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;CAC/D,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dalmia/calibrate-mcp",
-  "version": "0.0.68",
+  "version": "0.0.69",
   "author": "Speakeasy",
   "type": "module",
   "sideEffects": false,

package/src/landing-page.ts

@@ -930,7 +930,7 @@ http_headers = { "api-key-auth" = "YOUR_API_KEY_AUTH" }`;
         <h1>Instructions</h1>
         <p>One-click installation for Claude Desktop users</p>
         <div class="instruction-item">
-          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.68/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
+          <a href="https://github.com/dalmia/calibrate-mcp/releases/download/v0.0.69/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
             📥 Download MCP Bundle
           </a>
         </div>

package/src/lib/config.ts

@@ -65,8 +65,8 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
 export const SDK_METADATA = {
   language: "typescript",
   openapiDocVersion: "0.1.0",
-  sdkVersion: "0.0.68",
+  sdkVersion: "0.0.69",
   genVersion: "2.915.1",
   userAgent:
-    "speakeasy-sdk/mcp-typescript 0.0.68 2.915.1 0.1.0 @dalmia/calibrate-mcp",
+    "speakeasy-sdk/mcp-typescript 0.0.69 2.915.1 0.1.0 @dalmia/calibrate-mcp",
 } as const;

package/src/mcp-server/mcp-server.ts

@@ -21,7 +21,7 @@ const routes = buildRouteMap({
 export const app = buildApplication(routes, {
   name: "mcp",
   versionInfo: {
-    currentVersion: "0.0.68",
+    currentVersion: "0.0.69",
   },
 });
 

package/src/mcp-server/server.ts

@@ -73,7 +73,7 @@ export function createMCPServer(deps: {
 }) {
   const server = new McpServer({
     name: "CalibrateMcp",
-    version: "0.0.68",
+    version: "0.0.69",
   });
 
   const getClient = deps.getSDK || (() =>

package/src/models/annotationitempayload.ts

@@ -3,9 +3,15 @@
  */
 
 import * as z from "zod";
+import {
+  ItemMetadataEntry,
+  ItemMetadataEntry$zodSchema,
+} from "./itemmetadataentry.js";
+import { ItemPayload, ItemPayload$zodSchema } from "./itempayload.js";
 
 export type AnnotationItemPayload = {
-  payload?: any | undefined;
+  payload: ItemPayload;
+  metadata?: Array<ItemMetadataEntry> | null | undefined;
   annotations?: { [k: string]: any } | null | undefined;
   evaluator_results?: { [k: string]: any } | null | undefined;
 };
@@ -19,7 +25,11 @@ export const AnnotationItemPayload$zodSchema: z.ZodType<AnnotationItemPayload> =
       .describe(
         "Evaluator scores to record, keyed by evaluator ID. Each evaluator ID must be linked to the task. Put the score in `value`, a bool for binary or a number within the scale for rating, with optional `reasoning` and `version_number`. Omit `version_number` to record against the evaluator's live version. An item recording a tool call takes the tool-call evaluator and no other, and every other item takes any evaluator except that one",
       ),
-    payload: z.any().optional().describe(
-      "The item's payload. `name` is required and unique within the task for every type. The other fields depend on the task `type`:\n\n- `stt`: `name`, `reference_transcript`, `predicted_transcript`\n- `llm`: `name`, `chat_history` (list of `{role, content}` turns ending at the user turn), `agent_response` (the reply to judge), optional `evaluator_variables`\n- `llm-general`: `name`, `input`, `output`, optional `evaluator_variables`\n- `conversation`: `name`, `transcript` (list of `{role, content}` turns), optional `evaluator_variables`\n\n`evaluator_variables` maps an evaluator ID to that evaluator's `{{\"{{\"}}variable}}` values for this item",
+    metadata: z.array(ItemMetadataEntry$zodSchema).nullable().optional()
+      .describe(
+        "Key-value pairs stored with the item and shown beside it on the labelling form. Omit if you have none",
+      ),
+    payload: ItemPayload$zodSchema.describe(
+      "What an annotation item holds. Which fields apply depends on the task type",
     ),
   });

package/src/models/chatturn.ts

@@ -0,0 +1,44 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { collectExtraKeys as collectExtraKeys$ } from "../lib/schemas.js";
+
+/**
+ * One message in a conversation, in OpenAI chat format
+ */
+export type ChatTurn = {
+  role: string;
+  content?: string | null | undefined;
+  AdditionalProperties?: { [k: string]: any };
+};
+
+export const ChatTurn$zodSchema: z.ZodType<ChatTurn> = collectExtraKeys$(
+  z.object({
+    content: z.string().nullable().optional().describe(
+      "Message text. Omit for turns that only carry tool calls",
+    ),
+    role: z.string().describe(
+      "Message author role in the conversation history",
+    ),
+  }).describe("One message in a conversation, in OpenAI chat format").catchall(
+    z.any(),
+  ),
+  "AdditionalProperties",
+  true,
+);
+
+export const ChatTurn$zodSchemaOutbound: z.ZodType<any> = z.object({
+  AdditionalProperties: z.record(z.string(), z.any()).optional(),
+  content: z.nullable(z.string()).optional(),
+  role: z.string(),
+}).describe("One message in a conversation, in OpenAI chat format").transform(
+  (v) => {
+    const { AdditionalProperties, ...rest } = v;
+    return {
+      ...rest,
+      ...AdditionalProperties,
+    };
+  },
+);

package/src/models/itemmetadataentry.ts

@@ -0,0 +1,18 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+/**
+ * One key-value pair stored with an item and shown beside it on the labelling form
+ */
+export type ItemMetadataEntry = { key: string; value: string };
+
+export const ItemMetadataEntry$zodSchema: z.ZodType<ItemMetadataEntry> = z
+  .object({
+    key: z.string().describe("Name of the metadata entry"),
+    value: z.string().describe("Value of the metadata entry"),
+  }).describe(
+    "One key-value pair stored with an item and shown beside it on the labelling form",
+  );

package/src/models/itempayload.ts

@@ -0,0 +1,83 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ChatTurn, ChatTurn$zodSchema } from "./chatturn.js";
+import { ToolCall, ToolCall$zodSchema } from "./toolcall.js";
+
+/**
+ * What an annotation item holds. Which fields apply depends on the task type
+ */
+export type ItemPayload = {
+  name: string;
+  description?: string | null | undefined;
+  reference_transcript?: string | null | undefined;
+  predicted_transcript?: string | null | undefined;
+  text?: string | null | undefined;
+  audio_path?: string | null | undefined;
+  chat_history?: Array<ChatTurn> | null | undefined;
+  agent_response?: string | null | undefined;
+  input?: string | null | undefined;
+  output?: string | null | undefined;
+  transcript?: Array<ChatTurn> | null | undefined;
+  tool_calls?: Array<ToolCall> | null | undefined;
+  expected_tool_calls?: Array<ToolCall> | null | undefined;
+  actual_tool_calls?: Array<ToolCall> | null | undefined;
+  evaluator_variables?:
+    | { [k: string]: { [k: string]: any } }
+    | null
+    | undefined;
+};
+
+export const ItemPayload$zodSchema: z.ZodType<ItemPayload> = z.object({
+  actual_tool_calls: z.array(ToolCall$zodSchema).nullable().optional().describe(
+    "Tool calls the agent made. Sending this marks the item as a tool call for a person to judge. For `llm` and `llm-general` tasks",
+  ),
+  agent_response: z.string().nullable().optional().describe(
+    "The reply to judge. An empty string when the reply was a tool call. For `llm` tasks",
+  ),
+  audio_path: z.string().nullable().optional().describe(
+    "Storage key or `s3://` URI of the audio clip. For `tts` tasks",
+  ),
+  chat_history: z.array(ChatTurn$zodSchema).nullable().optional().describe(
+    "The conversation up to the reply being judged, ending at the user turn. For `llm` tasks",
+  ),
+  description: z.string().nullable().optional().describe(
+    "A description of the item. For every task type",
+  ),
+  evaluator_variables: z.record(z.string(), z.record(z.string(), z.any()))
+    .nullable().optional().describe(
+      "Values for each evaluator's `{{\"{{\"}}variable}}` placeholders, keyed by evaluator ID. For `llm`, `llm-general` and `conversation` tasks",
+    ),
+  expected_tool_calls: z.array(ToolCall$zodSchema).nullable().optional()
+    .describe(
+      "Tool calls the agent should have made. Sending this marks the item as a tool call for a person to judge. For `llm` and `llm-general` tasks",
+    ),
+  input: z.string().nullable().optional().describe(
+    "The standalone input the agent was given. For `llm-general` tasks",
+  ),
+  name: z.string().describe(
+    "Name of the item, unique within the task. For every task type",
+  ),
+  output: z.string().nullable().optional().describe(
+    "The output to judge. For `llm-general` tasks",
+  ),
+  predicted_transcript: z.string().nullable().optional().describe(
+    "The transcript to judge. For `stt` tasks",
+  ),
+  reference_transcript: z.string().nullable().optional().describe(
+    "The correct transcript. For `stt` tasks",
+  ),
+  text: z.string().nullable().optional().describe(
+    "The text that was spoken. For `tts` tasks",
+  ),
+  tool_calls: z.array(ToolCall$zodSchema).nullable().optional().describe(
+    "Tool calls the agent made in place of a text reply. With no text reply, sending this marks the item as a tool call for a person to judge. For `llm` and `llm-general` tasks",
+  ),
+  transcript: z.array(ChatTurn$zodSchema).nullable().optional().describe(
+    "The full conversation to judge. For `conversation` tasks",
+  ),
+}).describe(
+  "What an annotation item holds. Which fields apply depends on the task type",
+);

package/src/models/itemupdatepayload.ts

@@ -3,13 +3,26 @@
  */
 
 import * as z from "zod";
+import {
+  ItemMetadataEntry,
+  ItemMetadataEntry$zodSchema,
+} from "./itemmetadataentry.js";
+import { ItemPayload, ItemPayload$zodSchema } from "./itempayload.js";
 
-export type ItemUpdatePayload = { uuid: string; payload?: any | undefined };
+export type ItemUpdatePayload = {
+  uuid: string;
+  payload: ItemPayload;
+  metadata?: Array<ItemMetadataEntry> | null | undefined;
+};
 
 export const ItemUpdatePayload$zodSchema: z.ZodType<ItemUpdatePayload> = z
   .object({
-    payload: z.any().optional().describe(
-      "The item's new payload. `payload['name']` is still required and must be unique within the task",
+    metadata: z.array(ItemMetadataEntry$zodSchema).nullable().optional()
+      .describe(
+        "Key-value pairs stored with the item and shown beside it on the labelling form. Omit to leave the stored pairs as they are. Send an empty list to clear them",
+      ),
+    payload: ItemPayload$zodSchema.describe(
+      "What an annotation item holds. Which fields apply depends on the task type",
     ),
     uuid: z.string().describe("ID of the item to update"),
   });

package/src/models/toolcall.ts

@@ -0,0 +1,28 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+/**
+ * One call to a tool
+ */
+export type ToolCall = {
+  tool: string;
+  arguments?: { [k: string]: any } | null | undefined;
+  output?: any | undefined;
+  accept_any_arguments?: boolean | null | undefined;
+};
+
+export const ToolCall$zodSchema: z.ZodType<ToolCall> = z.object({
+  accept_any_arguments: z.boolean().nullable().optional().describe(
+    "When `true`, only the tool name had to match and `arguments` was ignored. Carried over from a tool-call test's expected calls",
+  ),
+  arguments: z.record(z.string(), z.any()).nullable().optional().describe(
+    "Argument values the agent passed to the tool. Omit when the call had none",
+  ),
+  output: z.any().optional().describe(
+    "What the tool returned for this call. Any JSON value. Omit when you do not record it",
+  ),
+  tool: z.string().describe("Name of the tool the agent called"),
+}).describe("One call to a tool");