@dalmia/calibrate-mcp 0.0.87 → 0.0.88

package/src/models/traceevalrunstatus.ts

@@ -0,0 +1,23 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+export const TraceEvalRunStatus = {
+  Pending: "pending",
+  Processing: "processing",
+  Completed: "completed",
+  Failed: "failed",
+  Skipped: "skipped",
+} as const;
+export type TraceEvalRunStatus = ClosedEnum<typeof TraceEvalRunStatus>;
+
+export const TraceEvalRunStatus$zodSchema = z.enum([
+  "pending",
+  "processing",
+  "completed",
+  "failed",
+  "skipped",
+]);

package/src/models/tracelistresponse.ts

@@ -0,0 +1,39 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  TraceScoreAverage,
+  TraceScoreAverage$zodSchema,
+} from "./tracescoreaverage.js";
+import { TraceSummary, TraceSummary$zodSchema } from "./tracesummary.js";
+
+/**
+ * The page, plus the averages when `include_score_averages` asked for them.
+ */
+export type TraceListResponse = {
+  items: Array<TraceSummary>;
+  total: number;
+  limit?: number | null | undefined;
+  offset?: number | undefined;
+  score_averages?: Array<TraceScoreAverage> | null | undefined;
+};
+
+export const TraceListResponse$zodSchema: z.ZodType<TraceListResponse> = z
+  .object({
+    items: z.array(TraceSummary$zodSchema).describe("The page of results"),
+    limit: z.int().nullable().optional().describe(
+      "The applied `limit` (null when unbounded)",
+    ),
+    offset: z.int().default(0).describe("The applied `offset`"),
+    score_averages: z.array(TraceScoreAverage$zodSchema).nullable().optional()
+      .describe(
+        "One entry per evaluator that scored at least one matching trace, averaged over every matching trace rather than this page. Ask for it with `include_score_averages`",
+      ),
+    total: z.int().describe(
+      "Total number of items matching the query, before pagination",
+    ),
+  }).describe(
+    "The page, plus the averages when `include_score_averages` asked for them.",
+  );

package/src/models/tracescoreaverage.ts

@@ -0,0 +1,72 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+/**
+ * How the evaluator scores:
+ *
+ * @remarks
+ *
+ * - `binary`: pass or fail
+ * - `rating`: a numeric score, using the scale in `output_config`
+ */
+export const TraceScoreAverageOutputType = {
+  Binary: "binary",
+  Rating: "rating",
+} as const;
+/**
+ * How the evaluator scores:
+ *
+ * @remarks
+ *
+ * - `binary`: pass or fail
+ * - `rating`: a numeric score, using the scale in `output_config`
+ */
+export type TraceScoreAverageOutputType = ClosedEnum<
+  typeof TraceScoreAverageOutputType
+>;
+
+export const TraceScoreAverageOutputType$zodSchema = z.enum([
+  "binary",
+  "rating",
+]).describe(
+  "How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n",
+);
+
+export type TraceScoreAverage = {
+  evaluator_uuid: string;
+  name: string;
+  output_type: TraceScoreAverageOutputType;
+  traces_scored: number;
+  average: number;
+  scale_min?: number | null | undefined;
+  scale_max?: number | null | undefined;
+};
+
+export const TraceScoreAverage$zodSchema: z.ZodType<TraceScoreAverage> = z
+  .object({
+    average: z.number().describe(
+      "Mean of those scores. A binary evaluator averages 0 and 1, so its mean is the share that passed",
+    ),
+    evaluator_uuid: z.string().describe(
+      "ID of the evaluator these numbers are for",
+    ),
+    name: z.string().describe(
+      "Name of the evaluator. Uses the last stored name if it has been deleted",
+    ),
+    output_type: TraceScoreAverageOutputType$zodSchema.describe(
+      "How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n",
+    ),
+    scale_max: z.number().nullable().optional().describe(
+      "Highest value on a rating scale",
+    ),
+    scale_min: z.number().nullable().optional().describe(
+      "Lowest value on a rating scale",
+    ),
+    traces_scored: z.int().describe(
+      "How many of the matching traces this evaluator scored",
+    ),
+  });

package/src/models/tracescoreresult.ts

@@ -0,0 +1,107 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+export const TraceScoreResultEvaluatorType = {
+  Tts: "tts",
+  Stt: "stt",
+  Llm: "llm",
+  LlmGeneral: "llm-general",
+  Conversation: "conversation",
+  ToolCall: "tool-call",
+} as const;
+export type TraceScoreResultEvaluatorType = ClosedEnum<
+  typeof TraceScoreResultEvaluatorType
+>;
+
+export const TraceScoreResultEvaluatorType$zodSchema = z.enum([
+  "tts",
+  "stt",
+  "llm",
+  "llm-general",
+  "conversation",
+  "tool-call",
+]);
+
+/**
+ * How the evaluator scores:
+ *
+ * @remarks
+ *
+ * - `binary`: pass or fail
+ * - `rating`: a numeric score, using the scale in `output_config`
+ */
+export const TraceScoreResultOutputType = {
+  Binary: "binary",
+  Rating: "rating",
+} as const;
+/**
+ * How the evaluator scores:
+ *
+ * @remarks
+ *
+ * - `binary`: pass or fail
+ * - `rating`: a numeric score, using the scale in `output_config`
+ */
+export type TraceScoreResultOutputType = ClosedEnum<
+  typeof TraceScoreResultOutputType
+>;
+
+export const TraceScoreResultOutputType$zodSchema = z.enum([
+  "binary",
+  "rating",
+]).describe(
+  "How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n",
+);
+
+export type TraceScoreResult = {
+  evaluator_uuid: string;
+  name: string;
+  evaluator_type?: TraceScoreResultEvaluatorType | null | undefined;
+  output_type: TraceScoreResultOutputType;
+  scale_min?: number | null | undefined;
+  scale_max?: number | null | undefined;
+  value: number;
+  reasoning?: string | null | undefined;
+  evaluator_version_id: string;
+  passed: boolean;
+};
+
+export const TraceScoreResult$zodSchema: z.ZodType<TraceScoreResult> = z.object(
+  {
+    evaluator_type: TraceScoreResultEvaluatorType$zodSchema.nullable()
+      .optional().describe(
+        "What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n",
+      ),
+    evaluator_uuid: z.string().describe(
+      "ID of the evaluator that produced this result",
+    ),
+    evaluator_version_id: z.string().describe(
+      "ID of the evaluator version that produced this result",
+    ),
+    name: z.string().describe(
+      "Name of the evaluator. Uses the last stored name if it has been deleted",
+    ),
+    output_type: TraceScoreResultOutputType$zodSchema.describe(
+      "How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n",
+    ),
+    passed: z.boolean().describe(
+      "Whether this evaluator passed. A binary result passes on 1. A rating passes at the top of its scale",
+    ),
+    reasoning: z.string().nullable().optional().describe(
+      "The judge's rationale for this result",
+    ),
+    scale_max: z.number().nullable().optional().describe(
+      "Highest value on a rating scale",
+    ),
+    scale_min: z.number().nullable().optional().describe(
+      "Lowest value on a rating scale",
+    ),
+    value: z.number().describe(
+      "The judged result. Binary stores 0 or 1. Rating stores the numeric score",
+    ),
+  },
+);

package/src/models/tracesummary.ts

@@ -0,0 +1,112 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod";
+import {
+  TraceEvalRunStatus,
+  TraceEvalRunStatus$zodSchema,
+} from "./traceevalrunstatus.js";
+import {
+  TraceMetadataEntry,
+  TraceMetadataEntry$zodSchema,
+} from "./tracemetadataentry.js";
+import { TraceOutput, TraceOutput$zodSchema } from "./traceoutput.js";
+import {
+  TraceScoreResult,
+  TraceScoreResult$zodSchema,
+} from "./tracescoreresult.js";
+import { TraceToolCall, TraceToolCall$zodSchema } from "./tracetoolcall.js";
+import { TraceTurn, TraceTurn$zodSchema } from "./traceturn.js";
+
+/**
+ * What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format. Sent only with `include_content`
+ */
+export type TraceSummaryInput = string | Array<TraceTurn>;
+
+export const TraceSummaryInput$zodSchema: z.ZodType<TraceSummaryInput> = z
+  .union([
+    z.string(),
+    z.array(TraceTurn$zodSchema),
+  ]).describe(
+    "What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format. Sent only with `include_content`",
+  );
+
+export type TraceSummary = {
+  uuid: string;
+  agent_id: string;
+  message_id?: string | null | undefined;
+  conversation_id?: string | null | undefined;
+  input_preview?: string | null | undefined;
+  response_preview?: string | null | undefined;
+  tool_names: Array<string>;
+  tool_calls: Array<TraceToolCall>;
+  turn_count: number;
+  tool_call_count: number;
+  metadata_count: number;
+  labels: Array<string>;
+  input?: string | Array<TraceTurn> | null | undefined;
+  output?: TraceOutput | null | undefined;
+  metadata?: Array<TraceMetadataEntry> | null | undefined;
+  created_at: string;
+  latest_run_status?: TraceEvalRunStatus | null | undefined;
+  latest_run_error?: string | null | undefined;
+  results?: Array<TraceScoreResult> | undefined;
+};
+
+export const TraceSummary$zodSchema: z.ZodType<TraceSummary> = z.object({
+  agent_id: z.string().describe("ID of the agent that produced the turn"),
+  conversation_id: z.string().nullable().optional().describe(
+    "The conversation ID you sent, if any",
+  ),
+  created_at: z.string().describe("When the trace was created (ISO 8601 UTC)"),
+  input: z.union([
+    z.string(),
+    z.array(TraceTurn$zodSchema),
+  ]).nullable().optional().describe(
+    "What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format. Sent only with `include_content`",
+  ),
+  input_preview: z.string().nullable().optional().describe(
+    "The standalone prompt or the last user message, truncated for display",
+  ),
+  labels: z.array(z.string()).describe(
+    "Your own tags for this turn, such as an environment or a release. Matched exactly when filtering, so keep the spelling stable",
+  ),
+  latest_run_error: z.string().nullable().optional().describe(
+    "Why the latest scoring run was skipped or failed",
+  ),
+  latest_run_status: TraceEvalRunStatus$zodSchema.nullable().optional()
+    .describe("Status of the latest scoring run for this trace"),
+  message_id: z.string().nullable().optional().describe(
+    "The message ID you sent, if any",
+  ),
+  metadata: z.array(TraceMetadataEntry$zodSchema).nullable().optional()
+    .describe(
+      "Key-value pairs stored with the trace. Sent only with `include_content`",
+    ),
+  metadata_count: z.int().describe(
+    "Number of metadata entries stored with the trace",
+  ),
+  output: TraceOutput$zodSchema.nullable().optional().describe(
+    "What the agent produced for this turn. Sent only with `include_content`",
+  ),
+  response_preview: z.string().nullable().optional().describe(
+    "The agent reply, truncated for display",
+  ),
+  results: z.array(TraceScoreResult$zodSchema).optional().describe(
+    "Results for each evaluator on the latest scoring run",
+  ),
+  tool_call_count: z.int().describe(
+    "Number of tool calls the agent issued for this turn",
+  ),
+  tool_calls: z.array(TraceToolCall$zodSchema).describe(
+    "Tools the agent issued on this turn, with the arguments it passed",
+  ),
+  tool_names: z.array(z.string()).describe(
+    "Names of the tools the agent issued on this turn, in order",
+  ),
+  turn_count: z.int().describe(
+    "Number of turns in the stored input. A standalone prompt counts as 1",
+  ),
+  uuid: z.string().describe("Unique ID for the trace"),
+});

package/src/tool-names.ts

@@ -183,5 +183,9 @@ export const toolNames: Array<{ name: string; description: string }>= [
   {
     "name": "create-trace",
     "description": "Store one production agent turn against an agent in the caller's org:\nthe conversation history as input, what the agent produced as output,\nand optional metadata. Use this to capture live traffic for later\nreview and conversion into tests. Fails with 429 once the workspace\nhits its trace limit.\n"
+  },
+  {
+    "name": "list-traces",
+    "description": "List the traces in the caller's org, newest first, one page at a\ntime. Each trace carries a short preview of its last user message\nand reply, its tool calls and labels, and the scores from its\nlatest scoring run with each evaluator's reasoning and evaluator\nversion. Filter by agent, label, creation time, text, or score.\nSet include_content to also get each trace's full input, output\nand metadata.\n"
   }
 ];