@lightsage/mcp 0.4.5 → 0.4.6

package/src/models/v2agentpreferencematchup.ts

@@ -0,0 +1,42 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 6f8baf2ca4f8
+ */
+
+import * as z from "zod";
+
+export type V2AgentPreferenceMatchup = {
+  wins: number;
+  losses: number;
+  ties: number;
+  comparisons: number;
+  win_rate?: number | null | undefined;
+  competitor_id?: string | null | undefined;
+  competitor_name: string;
+  competitor_domain?: string | null | undefined;
+  eval_ids?: Array<string> | undefined;
+};
+
+export const V2AgentPreferenceMatchup$zodSchema: z.ZodType<
+  V2AgentPreferenceMatchup
+> = z.object({
+  comparisons: z.int().describe("`wins + losses + ties`."),
+  competitor_domain: z.string().nullable().optional(),
+  competitor_id: z.string().nullable().optional().describe(
+    "Null when the competitor is no longer tracked.",
+  ),
+  competitor_name: z.string(),
+  eval_ids: z.array(z.string()).optional().describe(
+    "Agent Preference evals that contributed to this record.",
+  ),
+  losses: z.int().describe(
+    "Attempts where the agent recommended the competitor.",
+  ),
+  ties: z.int().describe(
+    "Attempts where the agent recommended both equally or neither.",
+  ),
+  win_rate: z.number().nullable().optional().describe(
+    "`wins / comparisons` from 0 to 1. Null when there are no comparisons.",
+  ),
+  wins: z.int().describe("Attempts where the agent recommended your company."),
+});

package/src/models/v2agentpreferencerecord.ts

@@ -0,0 +1,30 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: e7cdf1901a18
+ */
+
+import * as z from "zod";
+
+export type V2AgentPreferenceRecord = {
+  wins: number;
+  losses: number;
+  ties: number;
+  comparisons: number;
+  win_rate?: number | null | undefined;
+};
+
+export const V2AgentPreferenceRecord$zodSchema: z.ZodType<
+  V2AgentPreferenceRecord
+> = z.object({
+  comparisons: z.int().describe("`wins + losses + ties`."),
+  losses: z.int().describe(
+    "Attempts where the agent recommended the competitor.",
+  ),
+  ties: z.int().describe(
+    "Attempts where the agent recommended both equally or neither.",
+  ),
+  win_rate: z.number().nullable().optional().describe(
+    "`wins / comparisons` from 0 to 1. Null when there are no comparisons.",
+  ),
+  wins: z.int().describe("Attempts where the agent recommended your company."),
+});

package/src/models/v2agentpreferenceresult.ts

@@ -0,0 +1,35 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 028853e29c4e
+ */
+
+import * as z from "zod";
+import {
+  V2PreferenceResult,
+  V2PreferenceResult$zodSchema,
+} from "./v2preferenceresult.js";
+
+export type V2AgentPreferenceResult = {
+  id: string;
+  eval_id: string;
+  agent?: string | null | undefined;
+  preference_result?: V2PreferenceResult | null | undefined;
+  latency_ms?: number | null | undefined;
+  created_at?: string | null | undefined;
+};
+
+export const V2AgentPreferenceResult$zodSchema: z.ZodType<
+  V2AgentPreferenceResult
+> = z.object({
+  agent: z.string().nullable().optional().describe(
+    "Runnable agent combining the harness and model.",
+  ),
+  created_at: z.string().nullable().optional(),
+  eval_id: z.string(),
+  id: z.string().describe("Result ID for one attempt."),
+  latency_ms: z.int().nullable().optional(),
+  preference_result: V2PreferenceResult$zodSchema.nullable().optional()
+    .describe(
+      "Null when the attempt did not produce a usable recommendation, such as an attempt that ended in an error.",
+    ),
+});

package/src/models/v2agentpreferenceresultslistop.ts

@@ -0,0 +1,39 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: eb30f69d3567
+ */
+
+import * as z from "zod";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import {
+  V2AgentPreferenceResultsResponse,
+  V2AgentPreferenceResultsResponse$zodSchema,
+} from "./v2agentpreferenceresultsresponse.js";
+
+export type V2AgentPreferenceResultsListRequest = {
+  eval_id: string;
+  limit?: number | undefined;
+  offset?: number | undefined;
+};
+
+export const V2AgentPreferenceResultsListRequest$zodSchema: z.ZodType<
+  V2AgentPreferenceResultsListRequest
+> = z.object({
+  eval_id: z.string(),
+  limit: z.int().default(50),
+  offset: z.int().default(0),
+});
+
+export type V2AgentPreferenceResultsListResponse =
+  | V2AgentPreferenceResultsResponse
+  | HTTPValidationError;
+
+export const V2AgentPreferenceResultsListResponse$zodSchema: z.ZodType<
+  V2AgentPreferenceResultsListResponse
+> = z.union([
+  V2AgentPreferenceResultsResponse$zodSchema,
+  HTTPValidationError$zodSchema,
+]);

package/src/models/v2agentpreferenceresultsresponse.ts

@@ -0,0 +1,28 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: f337a60c79fe
+ */
+
+import * as z from "zod";
+import {
+  V2AgentPreferenceResult,
+  V2AgentPreferenceResult$zodSchema,
+} from "./v2agentpreferenceresult.js";
+
+export type V2AgentPreferenceResultsResponse = {
+  data?: Array<V2AgentPreferenceResult> | undefined;
+  count: number;
+  total: number;
+  limit: number;
+  offset: number;
+};
+
+export const V2AgentPreferenceResultsResponse$zodSchema: z.ZodType<
+  V2AgentPreferenceResultsResponse
+> = z.object({
+  count: z.int().describe("The number of items in `data`."),
+  data: z.array(V2AgentPreferenceResult$zodSchema).optional(),
+  limit: z.int(),
+  offset: z.int(),
+  total: z.int().describe("All recorded attempts for this eval."),
+});

package/src/models/v2evalresultsummary.ts

@@ -16,6 +16,10 @@ import {
   SearchGroundingMode,
   SearchGroundingMode$zodSchema,
 } from "./searchgroundingmode.js";
+import {
+  V2PreferenceResult,
+  V2PreferenceResult$zodSchema,
+} from "./v2preferenceresult.js";
 
 export type V2EvalResultSummary = {
   id: string;
@@ -28,6 +32,7 @@ export type V2EvalResultSummary = {
   score?: number | null | undefined;
   failure_reasons?: Array<string> | null | undefined;
   grader_results?: Array<PublicGraderResult> | undefined;
+  preference_result?: V2PreferenceResult | null | undefined;
   latency_ms?: number | null | undefined;
   search_grounding?: SearchGroundingMode | null | undefined;
   web_retrieval_observed?: boolean | null | undefined;
@@ -62,6 +67,10 @@ export const V2EvalResultSummary$zodSchema: z.ZodType<V2EvalResultSummary> = z
     id: z.string().describe("Result ID, not an eval job ID."),
     latency_ms: z.int().nullable().optional(),
     passed: z.boolean().nullable().optional(),
+    preference_result: V2PreferenceResult$zodSchema.nullable().optional()
+      .describe(
+        "Which company the agent recommended. Present only for Agent Preference evals that produced a usable recommendation.",
+      ),
     prompt: z.string().nullable().optional(),
     prompt_index: z.int().default(0),
     repeat_index: z.int().default(0),

package/src/models/v2preferenceresult.ts

@@ -0,0 +1,50 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 3496de5752d0
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+/**
+ * `won` when the agent recommended your company, `lost` when it recommended the competitor, and `tie` when it recommended both equally or neither.
+ */
+export const Outcome = {
+  Won: "won",
+  Lost: "lost",
+  Tie: "tie",
+} as const;
+/**
+ * `won` when the agent recommended your company, `lost` when it recommended the competitor, and `tie` when it recommended both equally or neither.
+ */
+export type Outcome = ClosedEnum<typeof Outcome>;
+
+export const Outcome$zodSchema = z.enum([
+  "won",
+  "lost",
+  "tie",
+]).describe(
+  "`won` when the agent recommended your company, `lost` when it recommended the competitor, and `tie` when it recommended both equally or neither.",
+);
+
+/**
+ * Which company an agent recommended, from your company's side.
+ */
+export type V2PreferenceResult = {
+  outcome: Outcome;
+  winner_name?: string | null | undefined;
+  competitor_name?: string | null | undefined;
+  competitor_domain?: string | null | undefined;
+};
+
+export const V2PreferenceResult$zodSchema: z.ZodType<V2PreferenceResult> = z
+  .object({
+    competitor_domain: z.string().nullable().optional(),
+    competitor_name: z.string().nullable().optional(),
+    outcome: Outcome$zodSchema.describe(
+      "`won` when the agent recommended your company, `lost` when it recommended the competitor, and `tie` when it recommended both equally or neither.",
+    ),
+    winner_name: z.string().nullable().optional().describe(
+      "The recommended company. Null on a tie.",
+    ),
+  }).describe("Which company an agent recommended, from your company's side.");

package/src/tool-names.ts

@@ -188,6 +188,38 @@ export const toolNames: Array<{ name: string; description: string }>= [
     "name": "eval-runs-v2-eval-runs-list",
     "description": "List Runs\n\nReturns jobs created through `POST /v2/eval-runs`, ordered newest first, with lifecycle state and progress."
   },
+  {
+    "name": "agent-preference-v2-agent-preference-competitors-9f0",
+    "description": "List Competitors\n\nReturns the tracked competitors your company can be compared against. Use a returned `id` as `competitor_id` when creating an Agent Preference eval."
+  },
+  {
+    "name": "agent-preference-v2-agent-preference-head-to-273",
+    "description": "Retrieve Head To Head\n\nReturns how often agents recommended your company over each tracked competitor: wins, losses, ties, and win rate per competitor plus a combined record. Calculated from the most recent attempts across all Agent Preference evals, up to `limit`; `truncated` reports whether older attempts were left out."
+  },
+  {
+    "name": "agent-preference-v2-agent-preference-evals-list",
+    "description": "List Evals\n\nReturns all active Agent Preference evals owned by the organization."
+  },
+  {
+    "name": "agent-preference-v2-agent-preference-evals-create",
+    "description": "Create Eval\n\nCreates a head-to-head comparison between your company and one tracked competitor. Each prompt asks an agent to choose, and a fixed judge records which company it recommended. Creating the eval does not start a run; pass the returned `id` as `eval_id` to `POST /v2/eval-runs`."
+  },
+  {
+    "name": "agent-preference-v2-agent-preference-evals-retrieve",
+    "description": "Retrieve Eval\n\nReturns one Agent Preference eval."
+  },
+  {
+    "name": "agent-preference-v2-agent-preference-evals-update",
+    "description": "Update Eval\n\nUpdates an Agent Preference eval. Omitted fields remain unchanged, an empty `agent` or `tags` array clears that collection, and `persona: null` clears the saved persona. The competitor and the judge that decides the comparison cannot be changed."
+  },
+  {
+    "name": "agent-preference-v2-agent-preference-evals-delete",
+    "description": "Delete Eval\n\nPermanently deletes an Agent Preference eval and returns its last saved state."
+  },
+  {
+    "name": "agent-preference-v2-agent-preference-results-list",
+    "description": "List Results\n\nReturns recorded attempts for an Agent Preference eval, newest first, each with whether your company won, lost, or tied against the competitor."
+  },
   {
     "name": "agents-v2-agents-list",
     "description": "List Agents\n\nReturns all runnable harness-and-model combinations as single agent IDs for use in eval runs."