@lightsage/mcp 0.4.23 → 0.4.24

package/src/models/graderdefinition.ts

@@ -27,6 +27,7 @@ export type GraderDefinition = {
   pattern?: string | null | undefined;
   case_sensitive?: boolean | undefined;
   criteria?: string | null | undefined;
+  plugin?: string | null | undefined;
   metric?: Metric | null | undefined;
   min_score?: number | null | undefined;
   max_value?: number | null | undefined;
@@ -48,6 +49,7 @@ export const GraderDefinition$zodSchema: z.ZodType<GraderDefinition> = z.object(
     needle: z.string().nullable().optional(),
     needles: z.array(z.string()).nullable().optional(),
     pattern: z.string().nullable().optional(),
+    plugin: z.string().nullable().optional(),
     type: z.string(),
     weight: z.number().default(1),
   },

package/src/models/publiccriterionverdict.ts

@@ -7,15 +7,16 @@ import * as z from "zod";
 import { ClosedEnum } from "../types/enums.js";
 
 /**
- * `PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score.
+ * `PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score. `NOT_APPLICABLE` when the requirement states its own condition ("If X is used, ...") and that condition did not occur; it neither passes nor fails the run.
  */
 export const Verdict = {
   Pass: "PASS",
   Fail: "FAIL",
   NotVerified: "NOT_VERIFIED",
+  NotApplicable: "NOT_APPLICABLE",
 } as const;
 /**
- * `PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score.
+ * `PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score. `NOT_APPLICABLE` when the requirement states its own condition ("If X is used, ...") and that condition did not occur; it neither passes nor fails the run.
  */
 export type Verdict = ClosedEnum<typeof Verdict>;
 
@@ -23,8 +24,9 @@ export const Verdict$zodSchema = z.enum([
   "PASS",
   "FAIL",
   "NOT_VERIFIED",
+  "NOT_APPLICABLE",
 ]).describe(
-  "`PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score.",
+  "`PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score. `NOT_APPLICABLE` when the requirement states its own condition (\"If X is used, ...\") and that condition did not occur; it neither passes nor fails the run.",
 );
 
 export type PublicCriterionVerdict = {
@@ -43,6 +45,6 @@ export const PublicCriterionVerdict$zodSchema: z.ZodType<
     "Short evidence the judge gave for the verdict.",
   ),
   verdict: Verdict$zodSchema.describe(
-    "`PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score.",
+    "`PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score. `NOT_APPLICABLE` when the requirement states its own condition (\"If X is used, ...\") and that condition did not occur; it neither passes nor fails the run.",
   ),
 });

package/src/models/publicgraderresult.ts

@@ -17,6 +17,7 @@ export type PublicGraderResult = {
   deductions?: Array<string> | undefined;
   details?: string | null | undefined;
   criteria_verdicts?: Array<PublicCriterionVerdict> | undefined;
+  not_applicable?: boolean | undefined;
 };
 
 export const PublicGraderResult$zodSchema: z.ZodType<PublicGraderResult> = z
@@ -35,6 +36,9 @@ export const PublicGraderResult$zodSchema: z.ZodType<PublicGraderResult> = z
       "Display name for the check, when the grader sets one.",
     ),
     name: z.string().describe("Grader name as configured on the eval."),
+    not_applicable: z.boolean().default(false).describe(
+      "Judges only: true when every requirement was `NOT_APPLICABLE`. The check is left out of the run's score and does not fail the run.",
+    ),
     passed: z.boolean().nullable().optional().describe(
       "Whether this check passed.",
     ),

package/src/models/publicpluginevalresultsdownload.ts

@@ -0,0 +1,34 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 56e8f4aa6dc9
+ */
+
+import * as z from "zod";
+
+export type PublicPluginEvalResultsDownload = {
+  plugin: string;
+  runs?: number | null | undefined;
+  prompts?: number | null | undefined;
+  filename?: string | null | undefined;
+  download_url: string;
+  created_at?: string | null | undefined;
+};
+
+export const PublicPluginEvalResultsDownload$zodSchema: z.ZodType<
+  PublicPluginEvalResultsDownload
+> = z.object({
+  created_at: z.iso.datetime({ offset: true }).nullable().optional().describe(
+    "When the results were prepared.",
+  ),
+  download_url: z.string().describe(
+    "Downloads the zip of the day's runs and their insights report. Call it with the same API key.",
+  ),
+  filename: z.string().nullable().optional().describe("The zip file's name."),
+  plugin: z.string().describe("The plugin's slug, such as `firecrawl`."),
+  prompts: z.int().nullable().optional().describe(
+    "Distinct prompts included in the download.",
+  ),
+  runs: z.int().nullable().optional().describe(
+    "Runs included in the download.",
+  ),
+});

package/src/models/publicpluginevalresultsresponse.ts

@@ -0,0 +1,24 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: f47fd4bfe2d1
+ */
+
+import * as z from "zod";
+import {
+  PublicPluginEvalResultsDownload,
+  PublicPluginEvalResultsDownload$zodSchema,
+} from "./publicpluginevalresultsdownload.js";
+
+export type PublicPluginEvalResultsResponse = {
+  date: string;
+  results: Array<PublicPluginEvalResultsDownload>;
+};
+
+export const PublicPluginEvalResultsResponse$zodSchema: z.ZodType<
+  PublicPluginEvalResultsResponse
+> = z.object({
+  date: z.string().describe("The day the results cover, as YYYY-MM-DD."),
+  results: z.array(PublicPluginEvalResultsDownload$zodSchema).describe(
+    "One download per plugin with results on that day.",
+  ),
+});

package/src/models/updatev2evalrequest.ts

@@ -34,6 +34,7 @@ export type UpdateV2EvalRequest = {
   judges?: Array<string> | null | undefined;
   agent?: Array<string> | null | undefined;
   persona?: string | undefined;
+  include_env_vars?: boolean | undefined;
   clis?: Array<string> | null | undefined;
   mcps?: Array<V2InlineMcpDefinition | V2SavedMcpReference> | null | undefined;
   setup_commands?: Array<string> | null | undefined;
@@ -52,6 +53,9 @@ export const UpdateV2EvalRequest$zodSchema: z.ZodType<UpdateV2EvalRequest> = z
     clis: z.array(z.string()).nullable().optional().describe(
       "Replacement saved CLI IDs or shell installation commands. UUID values resolve saved CLIs; every other value is an installation command. Pass an empty array to clear all CLIs.",
     ),
+    include_env_vars: z.boolean().optional().describe(
+      "Replacement environment setting. false runs the eval with no environment; omit it to keep the saved value.",
+    ),
     judges: z.array(z.string()).nullable().optional().describe(
       "Replacement success criteria for the eval.",
     ),

package/src/models/v2eval.ts

@@ -45,6 +45,7 @@ export type V2Eval = {
   prompts?: Array<string> | undefined;
   judges?: Array<string> | undefined;
   agent?: Array<string> | undefined;
+  include_env_vars?: boolean | undefined;
   persona?: string | null | undefined;
   clis?: Array<V2SavedCliReference | V2InlineCliReference> | undefined;
   mcps?: Array<V2InlineMcpDefinition | V2SavedMcpReference> | undefined;
@@ -66,6 +67,9 @@ export const V2Eval$zodSchema: z.ZodType<V2Eval> = z.object({
   ])).optional(),
   created_at: z.string().nullable().optional(),
   id: z.string(),
+  include_env_vars: z.boolean().default(true).describe(
+    "Whether runs get environment variables. false means runs use no environment, as Environment: Off in the app.",
+  ),
   judges: z.array(z.string()).optional(),
   mcps: z.array(z.union([
     V2InlineMcpDefinition$zodSchema,

package/src/models/v2evalresultsummary.ts

@@ -24,6 +24,10 @@ import {
   V2PreferenceResult,
   V2PreferenceResult$zodSchema,
 } from "./v2preferenceresult.js";
+import {
+  V2SetupCommandOutput,
+  V2SetupCommandOutput$zodSchema,
+} from "./v2setupcommandoutput.js";
 
 export type V2EvalResultSummary = {
   id: string;
@@ -46,6 +50,7 @@ export type V2EvalResultSummary = {
   grounding_qualification_reason?: string | null | undefined;
   created_at?: string | null | undefined;
   environment?: PublicEvalRunEnvironment | null | undefined;
+  setup_output?: Array<V2SetupCommandOutput> | undefined;
 };
 
 export const V2EvalResultSummary$zodSchema: z.ZodType<V2EvalResultSummary> = z
@@ -82,6 +87,9 @@ export const V2EvalResultSummary$zodSchema: z.ZodType<V2EvalResultSummary> = z
     score: z.number().nullable().optional().describe("Score from 0 to 1."),
     search_grounding: SearchGroundingMode$zodSchema.nullable().optional(),
     search_observed: z.boolean().nullable().optional(),
+    setup_output: z.array(V2SetupCommandOutput$zodSchema).optional().describe(
+      "Setup commands run before the agent started, in order, with the end of what each printed. Secret values are redacted. Empty when the eval has no setup commands or the attempt ran before setup output was recorded.",
+    ),
     tool_calls: z.array(V2EvalResultToolCall$zodSchema).optional().describe(
       "Every tool call the agent made, in order, with how long each took. Empty when the agent made no tool calls or the model does not use tools.",
     ),

package/src/models/v2pluginevalsresultslistop.ts

@@ -0,0 +1,35 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 4edf2fbdf8be
+ */
+
+import * as z from "zod";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import {
+  PublicPluginEvalResultsResponse,
+  PublicPluginEvalResultsResponse$zodSchema,
+} from "./publicpluginevalresultsresponse.js";
+
+export type V2PluginEvalsResultsListRequest = { date: string };
+
+export const V2PluginEvalsResultsListRequest$zodSchema: z.ZodType<
+  V2PluginEvalsResultsListRequest
+> = z.object({
+  date: z.string().date().describe(
+    "The day to get results for, as YYYY-MM-DD (UTC).",
+  ),
+});
+
+export type V2PluginEvalsResultsListResponse =
+  | PublicPluginEvalResultsResponse
+  | HTTPValidationError;
+
+export const V2PluginEvalsResultsListResponse$zodSchema: z.ZodType<
+  V2PluginEvalsResultsListResponse
+> = z.union([
+  PublicPluginEvalResultsResponse$zodSchema,
+  HTTPValidationError$zodSchema,
+]);

package/src/models/v2promptrunjobresponse.ts

@@ -73,7 +73,7 @@ export const V2PromptRunJobResponse$zodSchema: z.ZodType<
   dispatch_id: z.string().nullable().optional(),
   error: z.string().nullable().optional(),
   estimated_cost: z.number().nullable().optional().describe(
-    "The estimated credit cost for prompt jobs or estimated USD cost for API Performance jobs. Null when no estimate is available.",
+    "Estimated credits for the run, worked out when it was queued. Credits actually used can differ by agent and model. Null when no estimate is available.",
   ),
   eval_count: z.int().nullable().optional(),
   id: z.string().describe(

package/src/models/v2setupcommandoutput.ts

@@ -0,0 +1,31 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: cee0aa951ed5
+ */
+
+import * as z from "zod";
+
+export type V2SetupCommandOutput = {
+  command: string;
+  exit_code?: number | null | undefined;
+  status?: string | null | undefined;
+  output?: string | null | undefined;
+  output_truncated?: boolean | undefined;
+};
+
+export const V2SetupCommandOutput$zodSchema: z.ZodType<V2SetupCommandOutput> = z
+  .object({
+    command: z.string().describe("The setup command as saved on the eval."),
+    exit_code: z.int().nullable().optional().describe(
+      "The command's exit code.",
+    ),
+    output: z.string().nullable().optional().describe(
+      "The end of what the command printed (up to 4,000 characters), with secret values redacted.",
+    ),
+    output_truncated: z.boolean().default(false).describe(
+      "True when earlier output was cut off.",
+    ),
+    status: z.string().nullable().optional().describe(
+      "`succeeded` for a completed command.",
+    ),
+  });

package/src/tool-names.ts

@@ -264,6 +264,10 @@ export const toolNames: Array<{ name: string; description: string }>= [
     "name": "agent-preference-evals-results-list",
     "description": "List Results\n\nReturns recorded attempts for an Agent Preference eval, newest first, each with whether your company won, lost, or tied against the competitor."
   },
+  {
+    "name": "plugin-evals-results-list",
+    "description": "List Plugin Eval Results\n\nReturns a day's plugin eval results, one per plugin with results that day. Each has a `download_url` for a zip of every run with and without the plugin installed, plus the insights report written from them. Call `download_url` with the same API key. Returns 404 when no results exist for the date."
+  },
   {
     "name": "agents-list",
     "description": "List Agents\n\nReturns the coding-agent harness-and-model combinations your plan can run, as single agent IDs. Pass them as `agent` to `POST /v2/prompts/runs` to see how coding agents answer your prompts, with mentions and citations, or to `PUT /v2/prompts/targets` to run them on a schedule; to `POST /v2/agent-preference/evals` to see whether agents recommend your company over one competitor; or to `POST /v2/evals` and `POST /v2/eval-runs` to measure how well agents complete tasks with your product."

package/test/simulations.test.js

@@ -174,7 +174,8 @@ test("MCP tools forward canonical eval resources unchanged", async () => {
         method: "POST",
         url: "/v2/evals",
         apiKey: "ls_simulation_test",
-        body: createRequest,
+        // The schema fills in include_env_vars' default (true) on create.
+        body: { ...createRequest, include_env_vars: true },
       },
       {
         method: "POST",

package/test/tool-overrides.test.js

@@ -0,0 +1,70 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+const { Client } = await import("@modelcontextprotocol/sdk/client/index.js");
+const { InMemoryTransport } = await import(
+  "@modelcontextprotocol/sdk/inMemory.js"
+);
+const { createConsoleLogger } = await import(
+  "../esm/mcp-server/console-logger.js"
+);
+const { createMCPServer } = await import("../esm/mcp-server/server.js");
+const { ANNOTATION_OVERRIDES, TOOL_RENAMES, applyToolOverrides } = await import(
+  "../lib/tool-overrides.js"
+);
+
+async function listTools(allowedTools) {
+  const { server } = createMCPServer({
+    logger: createConsoleLogger("error"),
+    allowedTools,
+    dynamic: false,
+    serverURL: "https://api.lightsage.com",
+    getSDK: () => {
+      throw new Error("not called when listing tools");
+    },
+  });
+  applyToolOverrides(server);
+
+  const [clientTransport, serverTransport] =
+    InMemoryTransport.createLinkedPair();
+  const client = new Client({ name: "test", version: "1.0.0" });
+  await Promise.all([
+    server.connect(serverTransport),
+    client.connect(clientTransport),
+  ]);
+  const { tools } = await client.listTools();
+  await client.close();
+  return new Map(tools.map((tool) => [tool.name, tool]));
+}
+
+test("corrected tools carry the overridden hints", async () => {
+  const tools = await listTools(undefined);
+  for (const [name, overrides] of Object.entries(ANNOTATION_OVERRIDES)) {
+    const tool = tools.get(name);
+    assert.ok(tool, `${name} is listed`);
+    for (const [hint, value] of Object.entries(overrides)) {
+      assert.equal(tool.annotations?.[hint], value, `${name}.${hint}`);
+    }
+    for (const hint of ["readOnlyHint", "destructiveHint", "openWorldHint"]) {
+      assert.equal(typeof tool.annotations?.[hint], "boolean", `${name}.${hint}`);
+    }
+  }
+});
+
+test("unclear generated names are listed under their new names", async () => {
+  const tools = await listTools(undefined);
+  for (const [from, to] of Object.entries(TOOL_RENAMES)) {
+    assert.equal(tools.has(from), false, `${from} is gone`);
+    assert.ok(tools.has(to), `${to} is listed`);
+  }
+});
+
+test("renames apply to tools kept by the allowlist", async () => {
+  const tools = await listTools(["CL-is-list", "prompts-list"]);
+  assert.deepEqual([...tools.keys()].sort(), ["clis-list", "prompts-list"]);
+});
+
+test("tools without overrides keep their generated hints", async () => {
+  const tools = await listTools(["prompts-list"]);
+  assert.equal(tools.get("prompts-list").annotations.readOnlyHint, true);
+});