@lightsage/mcp 0.3.0 → 0.4.0

package/src/mcp-server/tools/evalsRun.ts

@@ -15,14 +15,16 @@ export const tool$evalsRun: ToolDefinition<typeof args> = {
   name: "evals-run",
   description: `Start an eval run
 
-Queues a durable eval job and returns immediately with a polling ID. Poll \`GET /v2/eval-runs/{run_id}\` until its status reaches a terminal value.
+Queues a durable eval job and returns immediately with a polling ID. Poll \`GET /v2/eval-runs/{run_id}\` until its status reaches a terminal value; once finished it also returns the results and analysis. Follow a run live with the monitor operation.
 
 Send one of these flat request forms:
 
-- **Saved eval:** provide \`eval_id\`, \`configuration_id\`, and \`agent_id\`.
-- **Direct create-and-run:** provide \`prompt\`, \`judges\`, and \`agent_id\`. You may also provide a repository, write-only environment variables, and public skill packages.
+- **Saved eval:** provide \`eval_id\`; optionally replace its prompts, persona, agents, CLIs, MCPs, setup commands, cleanup commands, or runs for this launch. A saved configuration can still provide repository, environment, skills, and legacy tool selections.
+- **Direct create-and-run:** provide \`prompts\`, \`judges\`, and \`agent\`. You may also provide a persona, tags, repository, write-only environment variables, saved or inline CLIs and MCP servers, skills, setup commands, and cleanup commands.
 
-Do not combine saved-run and direct-run fields. No separate \`mode\` field is required.`,
+The run fans out across prompts × agents × runs. Persona is shared metadata and does not multiply attempts. No separate \`mode\` field is required. If the organization lacks the estimated credits, the request fails with \`402\` before dispatching work.
+
+A saved or inline environment containing an empty or placeholder credential is rejected with \`400\` before any evaluation is dispatched.`,
   annotations: {
     "title": "",
     "destructiveHint": false,

package/src/mcp-server/tools/evalsV2EvalsCreate.ts

@@ -15,7 +15,7 @@ export const tool$evalsV2EvalsCreate: ToolDefinition<typeof args> = {
   name: "evals-v2-evals-create",
   description: `Create Eval
 
-Creates an eval with a task prompt, subject, and plain-text success criterion in \`judge\`. The eval can then be run through eval runs.`,
+Creates a reusable simulation definition with one or more task prompts and judges. Optional agents, persona, CLI installations, MCP servers, setup commands, cleanup commands, grounding policy, repeat count, and tags are saved with the eval. \`search_grounding\` supports \`disabled\`, \`enabled\`, and \`default\`; \`default\` preserves normal platform behavior. Creating the eval does not start a run and does not accept secret environment values.`,
   annotations: {
     "title": "",
     "destructiveHint": false,

package/src/mcp-server/tools/evalsV2EvalsList.ts

@@ -10,7 +10,7 @@ export const tool$evalsV2EvalsList: ToolDefinition = {
   name: "evals-v2-evals-list",
   description: `List Evals
 
-Returns all active evals including workflow, MCP, CLI, and skill evals owned by the organization.`,
+Returns all active eval definitions owned by the organization.`,
   annotations: {
     "title": "",
     "destructiveHint": false,

package/src/mcp-server/tools/evalsV2EvalsRetrieve.ts

@@ -15,7 +15,7 @@ export const tool$evalsV2EvalsRetrieve: ToolDefinition<typeof args> = {
   name: "evals-v2-evals-retrieve",
   description: `Retrieve Eval
 
-Returns one eval and its judge configuration.`,
+Returns one reusable eval simulation definition.`,
   annotations: {
     "title": "",
     "destructiveHint": false,

package/src/mcp-server/tools/evalsV2EvalsUpdate.ts

@@ -15,7 +15,7 @@ export const tool$evalsV2EvalsUpdate: ToolDefinition<typeof args> = {
   name: "evals-v2-evals-update",
   description: `Update Eval
 
-Updates an eval's task prompt, subject, plain-text \`judge\`, or name. Omitted fields remain unchanged.`,
+Updates a reusable eval simulation definition. Omitted fields remain unchanged. Empty \`agent\`, \`clis\`, \`mcps\`, \`setup_commands\`, \`cleanup_commands\`, or \`tags\` arrays clear those collections, and \`persona: null\` clears the saved persona.`,
   annotations: {
     "title": "",
     "destructiveHint": false,

package/src/mcp-server/tools/promptsV2PromptRunsList.ts

@@ -15,7 +15,7 @@ export const tool$promptsV2PromptRunsList: ToolDefinition<typeof args> = {
   name: "prompts-v2-prompt-runs-list",
   description: `List Prompt Runs
 
-Returns stored prompt runs without exposing raw runtime or provider configuration. Filterable by prompt, platform, and date range.`,
+Returns stored prompt runs with limit-and-offset pagination, without exposing raw runtime or provider configuration. Filter by prompt, platform, or date range.`,
   annotations: {
     "title": "",
     "destructiveHint": false,

package/src/mcp-server/tools/promptsV2PromptRunsRetrieve.ts

@@ -15,7 +15,7 @@ export const tool$promptsV2PromptRunsRetrieve: ToolDefinition<typeof args> = {
   name: "prompts-v2-prompt-runs-retrieve",
   description: `Retrieve Prompt Run
 
-Returns one stored prompt run, including its mentions and citations.`,
+Returns one stored prompt run, including mentions, citations, and any sanitized coding-agent trace captured for the run.`,
   annotations: {
     "title": "",
     "destructiveHint": false,

package/src/models/createv2evalconfigurationrequest.ts

@@ -4,6 +4,10 @@
  */
 
 import * as z from "zod";
+import {
+  SearchGroundingMode,
+  SearchGroundingMode$zodSchema,
+} from "./searchgroundingmode.js";
 
 export type CreateV2EvalConfigurationRequest = {
   name: string;
@@ -14,6 +18,7 @@ export type CreateV2EvalConfigurationRequest = {
   skill_ids?: Array<string> | undefined;
   skills?: Array<string> | undefined;
   include_docs?: boolean | undefined;
+  search_grounding?: SearchGroundingMode | undefined;
 };
 
 export const CreateV2EvalConfigurationRequest$zodSchema: z.ZodType<
@@ -25,6 +30,9 @@ export const CreateV2EvalConfigurationRequest$zodSchema: z.ZodType<
   mcp_server_ids: z.array(z.string()).optional(),
   name: z.string(),
   repository_id: z.string().optional(),
+  search_grounding: SearchGroundingMode$zodSchema.optional().describe(
+    "Requested web-retrieval behavior for an eval run.",
+  ),
   skill_ids: z.array(z.string()).optional(),
   skills: z.array(z.string()).optional().describe(
     "Public skill packages saved with this configuration, using GitHub `owner/repository` references such as `resend/resend-skills`.",

package/src/models/createv2evalrequest.ts

@@ -4,42 +4,82 @@
  */
 
 import * as z from "zod";
-import { ClosedEnum } from "../types/enums.js";
+import {
+  SearchGroundingMode,
+  SearchGroundingMode$zodSchema,
+} from "./searchgroundingmode.js";
+import {
+  V2InlineMcpDefinition,
+  V2InlineMcpDefinition$zodSchema,
+} from "./v2inlinemcpdefinition.js";
+import {
+  V2SavedMcpReference,
+  V2SavedMcpReference$zodSchema,
+} from "./v2savedmcpreference.js";
 
-export const CreateV2EvalRequestType = {
-  Workflow: "workflow",
-  Mcp: "mcp",
-  Cli: "cli",
-  Skill: "skill",
-} as const;
-export type CreateV2EvalRequestType = ClosedEnum<
-  typeof CreateV2EvalRequestType
->;
+export type CreateV2EvalRequestMcp =
+  | V2InlineMcpDefinition
+  | V2SavedMcpReference;
 
-export const CreateV2EvalRequestType$zodSchema = z.enum([
-  "workflow",
-  "mcp",
-  "cli",
-  "skill",
+export const CreateV2EvalRequestMcp$zodSchema: z.ZodType<
+  CreateV2EvalRequestMcp
+> = z.union([
+  V2InlineMcpDefinition$zodSchema,
+  V2SavedMcpReference$zodSchema,
 ]);
 
 export type CreateV2EvalRequest = {
   name: string;
-  type: CreateV2EvalRequestType;
-  prompt: string;
-  judge: string;
-  mcp_server_id?: string | undefined;
-  cli_id?: string | undefined;
-  skill_id?: string | undefined;
+  prompts: Array<string>;
+  judges: Array<string>;
+  agent?: Array<string> | undefined;
+  persona?: string | undefined;
+  clis?: Array<string> | undefined;
+  mcps?: Array<V2InlineMcpDefinition | V2SavedMcpReference> | undefined;
+  setup_commands?: Array<string> | undefined;
+  cleanup_commands?: Array<string> | undefined;
+  search_grounding?: SearchGroundingMode | undefined;
+  runs?: number | undefined;
+  tags?: Array<string> | undefined;
 };
 
 export const CreateV2EvalRequest$zodSchema: z.ZodType<CreateV2EvalRequest> = z
   .object({
-    cli_id: z.string().optional(),
-    judge: z.string().describe("Plain-text outcome that counts as success."),
-    mcp_server_id: z.string().optional(),
+    agent: z.array(z.string()).optional().describe(
+      "Runnable agents from `GET /v2/agents`. An empty array saves the eval without default agents. CLI clients expose repeatable `--agent`.",
+    ),
+    cleanup_commands: z.array(z.string()).optional().describe(
+      "Best-effort shell commands run sequentially after agent execution and result capture, before sandbox deletion. CLI clients expose repeatable `--cleanup-command`.",
+    ),
+    clis: z.array(z.string()).optional().describe(
+      "Saved CLI IDs or shell installation commands. UUID values resolve saved CLIs from `GET /v2/clis`; every other value is run as an installation command. CLI clients expose repeatable `--cli`.",
+    ),
+    judges: z.array(z.string()).describe(
+      "Plain-text success criteria. CLI clients expose this as repeatable `--judge`.",
+    ),
+    mcps: z.array(z.union([
+      V2InlineMcpDefinition$zodSchema,
+      V2SavedMcpReference$zodSchema,
+    ])).optional().describe(
+      "Saved MCP references or inline portable MCP definitions.",
+    ),
     name: z.string(),
-    prompt: z.string(),
-    skill_id: z.string().optional(),
-    type: CreateV2EvalRequestType$zodSchema,
+    persona: z.string().optional().describe(
+      "Optional perspective applied to the whole eval. It is stored as metadata and does not change prompts or multiply attempts.",
+    ),
+    prompts: z.array(z.string()).describe(
+      "Task prompts. CLI clients expose this as repeatable `--prompt`.",
+    ),
+    runs: z.int().default(1).describe(
+      "Independent attempts for every prompt-agent pair.",
+    ),
+    search_grounding: SearchGroundingMode$zodSchema.optional().describe(
+      "Requested web-retrieval behavior for an eval run.",
+    ),
+    setup_commands: z.array(z.string()).optional().describe(
+      "Shell commands run sequentially in the prepared repository before the agent starts. CLI clients expose repeatable `--setup-command`.",
+    ),
+    tags: z.array(z.string()).optional().describe(
+      "Optional labels for organizing the eval. CLI clients may expose this string array as a repeatable `--tag` flag.",
+    ),
   });

package/src/models/createv2evalrunrequest.ts

@@ -4,48 +4,115 @@
  */
 
 import * as z from "zod";
+import {
+  SearchGroundingMode,
+  SearchGroundingMode$zodSchema,
+} from "./searchgroundingmode.js";
+import {
+  V2InlineMcpDefinition,
+  V2InlineMcpDefinition$zodSchema,
+} from "./v2inlinemcpdefinition.js";
+import {
+  V2SavedMcpReference,
+  V2SavedMcpReference$zodSchema,
+} from "./v2savedmcpreference.js";
+
+export type CreateV2EvalRunRequestMcp =
+  | V2InlineMcpDefinition
+  | V2SavedMcpReference;
+
+export const CreateV2EvalRunRequestMcp$zodSchema: z.ZodType<
+  CreateV2EvalRunRequestMcp
+> = z.union([
+  V2InlineMcpDefinition$zodSchema,
+  V2SavedMcpReference$zodSchema,
+]);
 
 /**
- * Start an eval run using one of two flat request forms. Saved runs require `eval_id`, `configuration_id`, and `agent_id`. Direct create-and-run requests require `prompt`, at least one item in `judges`, and `agent_id`. Do not combine saved-run and direct-run fields; no separate mode field is required.
+ * Start an eval run using one of two flat request forms. Saved runs require `eval_id` and use saved simulation defaults unless the request replaces them. Direct create-and-run requests require `prompts`, `judges`, and `agent`. No separate mode field is required.
  */
 export type CreateV2EvalRunRequest = {
   eval_id?: string | undefined;
   configuration_id?: string | undefined;
-  agent_id: string;
-  prompt?: string | undefined;
-  judges?: Array<string> | undefined;
+  name?: string | undefined;
+  prompts?: Array<string> | null | undefined;
+  judges?: Array<string> | null | undefined;
+  agent?: Array<string> | null | undefined;
+  persona?: string | undefined;
+  clis?: Array<string> | null | undefined;
+  mcps?: Array<V2InlineMcpDefinition | V2SavedMcpReference> | null | undefined;
+  setup_commands?: Array<string> | null | undefined;
+  cleanup_commands?: Array<string> | null | undefined;
+  runs?: number | undefined;
+  tags?: Array<string> | undefined;
   repository?: string | undefined;
   env?: { [k: string]: string } | undefined;
   skills?: Array<string> | undefined;
+  skill_ids?: Array<string> | undefined;
+  search_grounding?: SearchGroundingMode | null | undefined;
 };
 
 export const CreateV2EvalRunRequest$zodSchema: z.ZodType<
   CreateV2EvalRunRequest
 > = z.object({
-  agent_id: z.string().describe(
-    "Required in both request forms. Runnable agent ID from `GET /v2/agents`.",
+  agent: z.array(z.string()).nullable().optional().describe(
+    "Runnable agents from `GET /v2/agents`. Required unless the saved eval already has agents. CLI clients expose repeatable `--agent`.",
+  ),
+  cleanup_commands: z.array(z.string()).nullable().optional().describe(
+    "Best-effort shell commands run sequentially after agent execution and result capture, before sandbox deletion. In saved mode, an explicit empty array disables saved cleanup commands for this launch. CLI clients expose repeatable `--cleanup-command`.",
+  ),
+  clis: z.array(z.string()).nullable().optional().describe(
+    "Saved CLI IDs or shell installation commands. UUID values resolve saved CLIs from `GET /v2/clis`; every other value is run as an installation command. In saved mode, an explicit empty array disables saved CLIs for this launch. CLI clients expose repeatable `--cli`.",
   ),
   configuration_id: z.string().optional().describe(
-    "Saved-run form only. Saved runtime configuration; required when `eval_id` is provided.",
+    "Saved-run form only. Optional saved repository, environment, skills, and legacy tool configuration.",
   ),
   env: z.record(z.string(), z.string()).optional().describe(
-    "Direct create-and-run form only. Write-only environment variables; values are stored only in the queued job's private configuration.",
+    "Write-only environment variables for this launch. Values are stored only in the queued job's private configuration.",
   ),
   eval_id: z.string().optional().describe(
     "Saved-run form only. Existing eval definition to run. Omit this field to use the direct create-and-run form.",
   ),
-  judges: z.array(z.string()).optional().describe(
-    "Direct create-and-run form only. Provide at least one success criterion. CLI clients may expose this string array as a repeatable `--judge` flag.",
+  judges: z.array(z.string()).nullable().optional().describe(
+    "Success criteria. Required in direct mode; in saved mode, replaces saved judges for this launch. CLI clients expose repeatable `--judge`.",
+  ),
+  mcps: z.array(z.union([
+    V2InlineMcpDefinition$zodSchema,
+    V2SavedMcpReference$zodSchema,
+  ])).nullable().optional().describe(
+    "Saved MCP references or inline portable definitions. In saved mode, an explicit empty array disables saved MCPs for this launch.",
+  ),
+  name: z.string().optional().describe(
+    "Direct create-and-run form only. Display name for the durable eval. Defaults to the first prompt.",
   ),
-  prompt: z.string().optional().describe(
-    "Direct create-and-run form only. Task prompt; required when `eval_id` is omitted.",
+  persona: z.string().optional().describe(
+    "Optional perspective for every attempt in this launch. It does not modify prompts or add a fan-out dimension.",
+  ),
+  prompts: z.array(z.string()).nullable().optional().describe(
+    "Task prompts. Required in direct mode; in saved mode, replaces saved prompts for this launch. CLI clients expose repeatable `--prompt`.",
   ),
   repository: z.string().optional().describe(
     "Direct create-and-run form only. A saved repository ID, built-in alias such as `nextjs-starter`, or a public GitHub HTTPS URL.",
   ),
+  runs: z.int().optional().describe(
+    "Independent attempts for every prompt-agent pair. Defaults to the saved eval value or 1 for a direct run.",
+  ),
+  search_grounding: SearchGroundingMode$zodSchema.nullable().optional()
+    .describe(
+      "Web-retrieval policy for this launch. Overrides the selected configuration or saved eval when provided; direct runs default to `default`.",
+    ),
+  setup_commands: z.array(z.string()).nullable().optional().describe(
+    "Shell commands run sequentially in the prepared repository before the agent starts. In saved mode, an explicit empty array disables saved setup commands for this launch. CLI clients expose repeatable `--setup-command`.",
+  ),
+  skill_ids: z.array(z.string()).optional().describe(
+    "Direct create-and-run form only. Reusable skill IDs from `GET /v2/skills` to install before the agent starts.",
+  ),
   skills: z.array(z.string()).optional().describe(
     "Additional public skill packages to install before the agent starts in either request form, using GitHub `owner/repository` references such as `resend/resend-skills`. CLI clients may expose this string array as a repeatable `--skill` flag.",
   ),
+  tags: z.array(z.string()).optional().describe(
+    "Direct create-and-run form only. Optional labels saved on the created eval. CLI clients may expose this string array as a repeatable `--tag` flag.",
+  ),
 }).describe(
-  "Start an eval run using one of two flat request forms. Saved runs require `eval_id`, `configuration_id`, and `agent_id`. Direct create-and-run requests require `prompt`, at least one item in `judges`, and `agent_id`. Do not combine saved-run and direct-run fields; no separate mode field is required.",
+  "Start an eval run using one of two flat request forms. Saved runs require `eval_id` and use saved simulation defaults unless the request replaces them. Direct create-and-run requests require `prompts`, `judges`, and `agent`. No separate mode field is required.",
 );

package/src/models/publiccompetitorrow.ts

@@ -26,11 +26,11 @@ export const PublicCompetitorRow$zodSchema: z.ZodType<PublicCompetitorRow> = z
     ),
     company: PublicCompany$zodSchema,
     share_of_voice: z.number().nullable().optional().describe(
-      "Share of all brand mentions across tracked companies, 0 to 1; rows sum to about 1.",
+      "Fraction of successful prompt runs that mention the company, from 0 to 1; for example, 0.25 means one in four runs. Failed provider runs are excluded. A run can mention several companies, so rows do not sum to 1.",
     ),
     share_of_voice_rank: z.int().nullable().optional(),
     visibility_score: z.number().nullable().optional().describe(
-      "The company's visibility score, from 0 to 1. Rows do not sum to 1.",
+      "Mean recommendation strength across successful prompt runs, from 0 to 1. Each run contributes the company's recommendation strength (0 to 1) when it is mentioned and 0 when it is not. Failed provider runs are excluded. Rows do not sum to 1.",
     ),
     visibility_score_rank: z.int().nullable().optional(),
   });

package/src/models/publiccriterionverdict.ts

@@ -0,0 +1,48 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 8574e6eb13db
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+/**
+ * `PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score.
+ */
+export const Verdict = {
+  Pass: "PASS",
+  Fail: "FAIL",
+  NotVerified: "NOT_VERIFIED",
+} as const;
+/**
+ * `PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score.
+ */
+export type Verdict = ClosedEnum<typeof Verdict>;
+
+export const Verdict$zodSchema = z.enum([
+  "PASS",
+  "FAIL",
+  "NOT_VERIFIED",
+]).describe(
+  "`PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score.",
+);
+
+export type PublicCriterionVerdict = {
+  criterion: string;
+  verdict: Verdict;
+  evidence?: string | null | undefined;
+};
+
+export const PublicCriterionVerdict$zodSchema: z.ZodType<
+  PublicCriterionVerdict
+> = z.object({
+  criterion: z.string().describe(
+    "One mandatory requirement from the judge criteria.",
+  ),
+  evidence: z.string().nullable().optional().describe(
+    "Short evidence the judge gave for the verdict.",
+  ),
+  verdict: Verdict$zodSchema.describe(
+    "`PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score.",
+  ),
+});

package/src/models/publiccurrentvisibilityresponse.ts

@@ -36,12 +36,12 @@ export const PublicCurrentVisibilityResponse$zodSchema: z.ZodType<
   end_date: z.string(),
   has_data: z.boolean(),
   share_of_voice: z.number().nullable().optional().describe(
-    "Share of all brand mentions across tracked companies, 0 to 1.",
+    "Fraction of successful prompt runs that mention the company, from 0 to 1; for example, 0.25 means one in four runs. Failed provider runs are excluded.",
   ),
   share_of_voice_rank: z.int().nullable().optional(),
   start_date: z.string(),
   visibility_score: z.number().nullable().optional().describe(
-    "The visibility score, from 0 to 1.",
+    "Mean recommendation strength across successful prompt runs, from 0 to 1. Each run contributes the company's recommendation strength (0 to 1) when it is mentioned and 0 when it is not. Failed provider runs are excluded.",
   ),
   visibility_score_rank: z.int().nullable().optional(),
 });

package/src/models/publicevalrunenvironment.ts

@@ -0,0 +1,47 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 99dc6a5135ee
+ */
+
+import * as z from "zod";
+
+/**
+ * Non-secret summary of the environment an eval run was queued with.
+ */
+export type PublicEvalRunEnvironment = {
+  id?: string | null | undefined;
+  name?: string | null | undefined;
+  updated_at?: string | null | undefined;
+  queued_at?: string | null | undefined;
+  variable_names?: Array<string> | undefined;
+  inline?: boolean | undefined;
+  changed_since_queued?: boolean | null | undefined;
+};
+
+export const PublicEvalRunEnvironment$zodSchema: z.ZodType<
+  PublicEvalRunEnvironment
+> = z.object({
+  changed_since_queued: z.boolean().nullable().optional().describe(
+    "True when the saved environment has been edited or deleted since the run was queued, so a rerun would use different values. Null when unknown or for inline environments.",
+  ),
+  id: z.string().nullable().optional().describe(
+    "Environment ID, as listed by the environments operation.",
+  ),
+  inline: z.boolean().default(false).describe(
+    "True when variables were supplied inline with the run request instead of from a saved environment.",
+  ),
+  name: z.string().nullable().optional().describe(
+    "Environment name when the run was queued.",
+  ),
+  queued_at: z.string().nullable().optional().describe(
+    "When the run was queued with this environment.",
+  ),
+  updated_at: z.string().nullable().optional().describe(
+    "Environment version the run used: when its name or variables last changed before the run was queued. Null if the environment had not changed since versioning was introduced.",
+  ),
+  variable_names: z.array(z.string()).optional().describe(
+    "Variable names supplied to the run. Values are never returned.",
+  ),
+}).describe(
+  "Non-secret summary of the environment an eval run was queued with.",
+);

package/src/models/publicevalrunprogress.ts

@@ -12,6 +12,7 @@ export type PublicEvalRunProgress = {
   completed?: number | undefined;
   failed?: number | undefined;
   cancelled?: number | undefined;
+  retrying?: number | undefined;
   interrupted?: number | undefined;
   percent?: number | undefined;
 };
@@ -21,11 +22,16 @@ export const PublicEvalRunProgress$zodSchema: z.ZodType<PublicEvalRunProgress> =
     cancelled: z.int().default(0),
     completed: z.int().default(0),
     failed: z.int().default(0),
-    interrupted: z.int().default(0),
+    interrupted: z.int().default(0).describe(
+      "Deprecated compatibility alias for `retrying`; use `retrying`.",
+    ),
     percent: z.number().default(0).describe(
-      "The percentage of child jobs in a terminal state. It can reach `100` while the parent job is still summarizing, so use `status` to detect completion.",
+      "The percentage of child jobs in a final state (`completed`, `failed`, or `cancelled`). It can reach `100` while the parent job is still summarizing, so use `status` to detect completion.",
     ),
     queued: z.int().default(0),
+    retrying: z.int().default(0).describe(
+      "Recoverable child jobs waiting to be retried. These remain active and are excluded from `percent`.",
+    ),
     running: z.int().default(0),
     total: z.int().default(0),
   });

package/src/models/publicgraderresult.ts

@@ -0,0 +1,44 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 73852ac8b2ad
+ */
+
+import * as z from "zod";
+import {
+  PublicCriterionVerdict,
+  PublicCriterionVerdict$zodSchema,
+} from "./publiccriterionverdict.js";
+
+export type PublicGraderResult = {
+  name: string;
+  label?: string | null | undefined;
+  passed?: boolean | null | undefined;
+  score?: number | null | undefined;
+  deductions?: Array<string> | undefined;
+  details?: string | null | undefined;
+  criteria_verdicts?: Array<PublicCriterionVerdict> | undefined;
+};
+
+export const PublicGraderResult$zodSchema: z.ZodType<PublicGraderResult> = z
+  .object({
+    criteria_verdicts: z.array(PublicCriterionVerdict$zodSchema).optional()
+      .describe(
+        "Judges only: a verdict for each mandatory requirement in the judge criteria. Empty for other checks and for runs graded before per-requirement verdicts were recorded.",
+      ),
+    deductions: z.array(z.string()).optional().describe(
+      "Specific shortcomings the check recorded, even when it passed. A judge can pass with a score below 1 and still list deductions here; those checks do not appear in `failure_reasons`.",
+    ),
+    details: z.string().nullable().optional().describe(
+      "Reasoning or check output explaining the verdict.",
+    ),
+    label: z.string().nullable().optional().describe(
+      "Display name for the check, when the grader sets one.",
+    ),
+    name: z.string().describe("Grader name as configured on the eval."),
+    passed: z.boolean().nullable().optional().describe(
+      "Whether this check passed.",
+    ),
+    score: z.number().nullable().optional().describe(
+      "Score from 0 to 1 for checks that score partially, such as judges; null for pass/fail checks.",
+    ),
+  });

package/src/models/publicpromptvisibilitymetrics.ts

@@ -25,10 +25,10 @@ export const PublicPromptVisibilityMetrics$zodSchema: z.ZodType<
     "One per prompt, platform, and run date.",
   ),
   share_of_voice: z.number().nullable().optional().describe(
-    "Share of all brand mentions across tracked companies, 0 to 1.",
+    "Fraction of successful prompt runs that mention the company, from 0 to 1; for example, 0.25 means one in four runs. Failed provider runs are excluded.",
   ),
   visibility_score: z.number().nullable().optional().describe(
-    "The visibility score, from 0 to 1.",
+    "Mean recommendation strength across successful prompt runs, from 0 to 1. Each run contributes the company's recommendation strength (0 to 1) when it is mentioned and 0 when it is not. Failed provider runs are excluded.",
   ),
   visibility_score_rank: z.int().nullable().optional(),
 });

package/src/models/publicvisibilitysnapshot.ts

@@ -23,14 +23,14 @@ export const PublicVisibilitySnapshot$zodSchema: z.ZodType<
   ),
   average_position_rank: z.int().nullable().optional(),
   share_of_voice: z.number().nullable().optional().describe(
-    "Share of all brand mentions across tracked companies, 0 to 1.",
+    "Fraction of successful prompt runs that mention the company, from 0 to 1; for example, 0.25 means one in four runs. Failed provider runs are excluded.",
   ),
   share_of_voice_rank: z.int().nullable().optional(),
   total_citations: z.int().default(0).describe(
     "Owned-domain citations across all platforms; 0, not `null`, with no prompt runs.",
   ),
   visibility_score: z.number().nullable().optional().describe(
-    "The visibility score, from 0 to 1.",
+    "Mean recommendation strength across successful prompt runs, from 0 to 1. Each run contributes the company's recommendation strength (0 to 1) when it is mentioned and 0 when it is not. Failed provider runs are excluded.",
   ),
   visibility_score_rank: z.int().nullable().optional(),
 });

package/src/models/searchgroundingmode.ts

@@ -0,0 +1,26 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: d6aefbc6ebf6
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+/**
+ * Requested web-retrieval behavior for an eval run.
+ */
+export const SearchGroundingMode = {
+  Disabled: "disabled",
+  Enabled: "enabled",
+  Default: "default",
+} as const;
+/**
+ * Requested web-retrieval behavior for an eval run.
+ */
+export type SearchGroundingMode = ClosedEnum<typeof SearchGroundingMode>;
+
+export const SearchGroundingMode$zodSchema = z.enum([
+  "disabled",
+  "enabled",
+  "default",
+]).describe("Requested web-retrieval behavior for an eval run.");

package/src/models/updatev2evalconfigurationrequest.ts

@@ -4,6 +4,10 @@
  */
 
 import * as z from "zod";
+import {
+  SearchGroundingMode,
+  SearchGroundingMode$zodSchema,
+} from "./searchgroundingmode.js";
 
 export type UpdateV2EvalConfigurationRequest = {
   name?: string | undefined;
@@ -14,6 +18,7 @@ export type UpdateV2EvalConfigurationRequest = {
   skill_ids?: Array<string> | null | undefined;
   skills?: Array<string> | null | undefined;
   include_docs?: boolean | undefined;
+  search_grounding?: SearchGroundingMode | null | undefined;
 };
 
 export const UpdateV2EvalConfigurationRequest$zodSchema: z.ZodType<
@@ -25,6 +30,10 @@ export const UpdateV2EvalConfigurationRequest$zodSchema: z.ZodType<
   mcp_server_ids: z.array(z.string()).nullable().optional(),
   name: z.string().optional(),
   repository_id: z.string().optional(),
+  search_grounding: SearchGroundingMode$zodSchema.nullable().optional()
+    .describe(
+      "Replacement web-retrieval policy. Omit it to keep the saved value.",
+    ),
   skill_ids: z.array(z.string()).nullable().optional(),
   skills: z.array(z.string()).nullable().optional().describe(
     "Public skill packages saved with this configuration, using GitHub `owner/repository` references.",