@dalmia/calibrate-mcp 0.0.68 → 0.0.70

package/src/models/itempayload.ts

@@ -0,0 +1,128 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ChatTurn, ChatTurn$zodSchema } from "./chatturn.js";
+import { ToolCall, ToolCall$zodSchema } from "./toolcall.js";
+import { TurnText, TurnText$zodSchema } from "./turntext.js";
+
+/**
+ * The reply to judge. An empty string when the reply was a tool call. For `llm` tasks
+ */
+export type AgentResponseUnion = TurnText | string;
+
+export const AgentResponseUnion$zodSchema: z.ZodType<AgentResponseUnion> = z
+  .union([
+    TurnText$zodSchema,
+    z.string(),
+  ]).describe(
+    "The reply to judge. An empty string when the reply was a tool call. For `llm` tasks",
+  );
+
+/**
+ * The standalone input the agent was given. For `llm-general` tasks
+ */
+export type ItemPayloadInput = TurnText | string;
+
+export const ItemPayloadInput$zodSchema: z.ZodType<ItemPayloadInput> = z.union([
+  TurnText$zodSchema,
+  z.string(),
+]).describe(
+  "The standalone input the agent was given. For `llm-general` tasks",
+);
+
+/**
+ * The output to judge. For `llm-general` tasks
+ */
+export type Output = TurnText | string;
+
+export const Output$zodSchema: z.ZodType<Output> = z.union([
+  TurnText$zodSchema,
+  z.string(),
+]).describe("The output to judge. For `llm-general` tasks");
+
+/**
+ * What an annotation item holds. Which fields apply depends on the task type
+ */
+export type ItemPayload = {
+  name: string;
+  description?: string | null | undefined;
+  reference_transcript?: string | null | undefined;
+  predicted_transcript?: string | null | undefined;
+  text?: string | null | undefined;
+  audio_path?: string | null | undefined;
+  chat_history?: Array<ChatTurn> | null | undefined;
+  agent_response?: TurnText | string | null | undefined;
+  input?: TurnText | string | null | undefined;
+  output?: TurnText | string | null | undefined;
+  transcript?: Array<ChatTurn> | null | undefined;
+  tool_calls?: Array<ToolCall> | null | undefined;
+  expected_tool_calls?: Array<ToolCall> | null | undefined;
+  actual_tool_calls?: Array<ToolCall> | null | undefined;
+  evaluator_variables?:
+    | { [k: string]: { [k: string]: any } }
+    | null
+    | undefined;
+};
+
+export const ItemPayload$zodSchema: z.ZodType<ItemPayload> = z.object({
+  actual_tool_calls: z.array(ToolCall$zodSchema).nullable().optional().describe(
+    "Tool calls the agent made. Sending this marks the item as a tool call for a person to judge. For `llm` and `llm-general` tasks",
+  ),
+  agent_response: z.union([
+    TurnText$zodSchema,
+    z.string(),
+  ]).nullable().optional().describe(
+    "The reply to judge. An empty string when the reply was a tool call. For `llm` tasks",
+  ),
+  audio_path: z.string().nullable().optional().describe(
+    "Storage key or `s3://` URI of the audio clip. For `tts` tasks",
+  ),
+  chat_history: z.array(ChatTurn$zodSchema).nullable().optional().describe(
+    "The conversation up to the reply being judged, ending at the user turn. For `llm` tasks",
+  ),
+  description: z.string().nullable().optional().describe(
+    "A description of the item. For every task type",
+  ),
+  evaluator_variables: z.record(z.string(), z.record(z.string(), z.any()))
+    .nullable().optional().describe(
+      "Values for each evaluator's `{{\"{{\"}}variable}}` placeholders, keyed by evaluator ID. For `llm`, `llm-general` and `conversation` tasks",
+    ),
+  expected_tool_calls: z.array(ToolCall$zodSchema).nullable().optional()
+    .describe(
+      "Tool calls the agent should have made. Sending this marks the item as a tool call for a person to judge. For `llm` and `llm-general` tasks",
+    ),
+  input: z.union([
+    TurnText$zodSchema,
+    z.string(),
+  ]).nullable().optional().describe(
+    "The standalone input the agent was given. For `llm-general` tasks",
+  ),
+  name: z.string().describe(
+    "Name of the item, unique within the task. For every task type",
+  ),
+  output: z.union([
+    TurnText$zodSchema,
+    z.string(),
+  ]).nullable().optional().describe(
+    "The output to judge. For `llm-general` tasks",
+  ),
+  predicted_transcript: z.string().nullable().optional().describe(
+    "The transcript to judge. For `stt` tasks",
+  ),
+  reference_transcript: z.string().nullable().optional().describe(
+    "The correct transcript. For `stt` tasks",
+  ),
+  text: z.string().nullable().optional().describe(
+    "The text that was spoken. For `tts` tasks",
+  ),
+  tool_calls: z.array(ToolCall$zodSchema).nullable().optional().describe(
+    "Tool calls the agent made in place of a text reply. With no text reply, sending this marks the item as a tool call for a person to judge. For `llm` and `llm-general` tasks",
+  ),
+  transcript: z.array(ChatTurn$zodSchema).nullable().optional().describe(
+    "The full conversation to judge. For `conversation` tasks",
+  ),
+}).describe(
+  "What an annotation item holds. Which fields apply depends on the task type",
+);

package/src/models/itemupdatepayload.ts

@@ -3,13 +3,26 @@
  */
 
 import * as z from "zod";
+import {
+  ItemMetadataEntry,
+  ItemMetadataEntry$zodSchema,
+} from "./itemmetadataentry.js";
+import { ItemPayload, ItemPayload$zodSchema } from "./itempayload.js";
 
-export type ItemUpdatePayload = { uuid: string; payload?: any | undefined };
+export type ItemUpdatePayload = {
+  uuid: string;
+  payload: ItemPayload;
+  metadata?: Array<ItemMetadataEntry> | null | undefined;
+};
 
 export const ItemUpdatePayload$zodSchema: z.ZodType<ItemUpdatePayload> = z
   .object({
-    payload: z.any().optional().describe(
-      "The item's new payload. `payload['name']` is still required and must be unique within the task",
+    metadata: z.array(ItemMetadataEntry$zodSchema).nullable().optional()
+      .describe(
+        "Key-value pairs stored with the item and shown beside it on the labelling form. Omit to leave the stored pairs as they are. Send an empty list to clear them",
+      ),
+    payload: ItemPayload$zodSchema.describe(
+      "What an annotation item holds. Which fields apply depends on the task type",
     ),
     uuid: z.string().describe("ID of the item to update"),
   });

package/src/models/toolcall.ts

@@ -0,0 +1,28 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+/**
+ * One call to a tool
+ */
+export type ToolCall = {
+  tool: string;
+  arguments?: { [k: string]: any } | null | undefined;
+  output?: any | undefined;
+  accept_any_arguments?: boolean | null | undefined;
+};
+
+export const ToolCall$zodSchema: z.ZodType<ToolCall> = z.object({
+  accept_any_arguments: z.boolean().nullable().optional().describe(
+    "When `true`, only the tool name had to match and `arguments` was ignored. Carried over from a tool-call test's expected calls",
+  ),
+  arguments: z.record(z.string(), z.any()).nullable().optional().describe(
+    "Argument values the agent passed to the tool. Omit when the call had none",
+  ),
+  output: z.any().optional().describe(
+    "What the tool returned for this call. Any JSON value. Omit when you do not record it",
+  ),
+  tool: z.string().describe("Name of the tool the agent called"),
+}).describe("One call to a tool");

package/src/models/traceingest.ts

@@ -13,9 +13,9 @@ import { TraceTurn, TraceTurn$zodSchema } from "./traceturn.js";
 /**
  * What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format
  */
-export type Input = string | Array<TraceTurn>;
+export type TraceIngestInput = string | Array<TraceTurn>;
 
-export const Input$zodSchema: z.ZodType<Input> = z.union([
+export const TraceIngestInput$zodSchema: z.ZodType<TraceIngestInput> = z.union([
   z.string(),
   z.array(TraceTurn$zodSchema),
 ]).describe(

package/src/models/turntext.ts

@@ -0,0 +1,28 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+
+/**
+ * Text with a display name and other forms of it, sent in place of a plain string
+ */
+export type TurnText = {
+  content: string;
+  label?: string | null | undefined;
+  versions?: { [k: string]: string } | null | undefined;
+};
+
+export const TurnText$zodSchema: z.ZodType<TurnText> = z.object({
+  content: z.string().describe(
+    "The text the LLM judge reads and annotators see by default",
+  ),
+  label: z.string().nullable().optional().describe(
+    "Display name for the text in `content`, shown to annotators",
+  ),
+  versions: z.record(z.string(), z.string()).nullable().optional().describe(
+    "Other forms of the same text, such as a translation or a human rewrite, keyed by the name annotators see for each. Shown to annotators only",
+  ),
+}).describe(
+  "Text with a display name and other forms of it, sent in place of a plain string",
+);