@lightsage/mcp 0.4.22 → 0.4.24

package/src/mcp-server/server.ts

@@ -59,6 +59,7 @@ import { tool$evalTemplatesUpdate } from "./tools/evalTemplatesUpdate.js";
 import { tool$mcpServersList } from "./tools/mcpServersList.js";
 import { tool$opportunitiesList } from "./tools/opportunitiesList.js";
 import { tool$opportunitiesRetrieve } from "./tools/opportunitiesRetrieve.js";
+import { tool$pluginEvalsResultsList } from "./tools/pluginEvalsResultsList.js";
 import { tool$promptsCreate } from "./tools/promptsCreate.js";
 import { tool$promptsDelete } from "./tools/promptsDelete.js";
 import { tool$promptsList } from "./tools/promptsList.js";
@@ -114,7 +115,7 @@ export function createMCPServer(deps: {
       sizes: ["any"],
     }],
     name: "LightsageMcp",
-    version: "0.4.22",
+    version: "0.4.24",
   });
 
   const getClient = deps.getSDK || (() =>
@@ -225,6 +226,7 @@ export function createMCPServer(deps: {
   tool(tool$agentPreferenceEvalsUpdate);
   tool(tool$agentPreferenceEvalsDelete);
   tool(tool$agentPreferenceEvalsResultsList);
+  tool(tool$pluginEvalsResultsList);
   tool(tool$agentsList);
   tool(tool$repositoriesList);
   tool(tool$repositoriesCreate);

package/src/mcp-server/tools/pluginEvalsResultsList.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: e2687c95e74c
+ */
+
+import { pluginEvalsResultsList } from "../../funcs/pluginEvalsResultsList.js";
+import { V2PluginEvalsResultsListRequest$zodSchema } from "../../models/v2pluginevalsresultslistop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: V2PluginEvalsResultsListRequest$zodSchema,
+};
+
+export const tool$pluginEvalsResultsList: ToolDefinition<typeof args> = {
+  name: "plugin-evals-results-list",
+  description: `List Plugin Eval Results
+
+Returns a day's plugin eval results, one per plugin with results that day. Each has a \`download_url\` for a zip of every run with and without the plugin installed, plus the insights report written from them. Call \`download_url\` with the same API key. Returns 404 when no results exist for the date.`,
+  annotations: {
+    "title": "",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": false,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await pluginEvalsResultsList(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/models/createv2evalrequest.ts

@@ -33,6 +33,7 @@ export type CreateV2EvalRequest = {
   prompts: Array<string>;
   judges: Array<string>;
   agent?: Array<string> | undefined;
+  include_env_vars?: boolean | undefined;
   persona?: string | undefined;
   clis?: Array<string> | undefined;
   mcps?: Array<V2InlineMcpDefinition | V2SavedMcpReference> | undefined;
@@ -54,6 +55,9 @@ export const CreateV2EvalRequest$zodSchema: z.ZodType<CreateV2EvalRequest> = z
     clis: z.array(z.string()).optional().describe(
       "Saved CLI IDs or shell installation commands. UUID values resolve saved CLIs from `GET /v2/clis`; every other value is run as an installation command. CLI clients expose repeatable `--cli`.",
     ),
+    include_env_vars: z.boolean().default(true).describe(
+      "Whether runs get environment variables. Set false to run with no environment: neither a configuration's environment nor the organization's default environment is attached. This is the same setting as Environment: Off in the app.",
+    ),
     judges: z.array(z.string()).describe(
       "Plain-text success criteria. CLI clients expose this as repeatable `--judge`.",
     ),

package/src/models/createv2evalrunrequest.ts

@@ -47,6 +47,7 @@ export type CreateV2EvalRunRequest = {
   tags?: Array<string> | undefined;
   repository?: string | undefined;
   env?: { [k: string]: string } | undefined;
+  include_env_vars?: boolean | undefined;
   skills?: Array<string> | undefined;
   skill_ids?: Array<string> | undefined;
   search_grounding?: SearchGroundingMode | null | undefined;
@@ -73,6 +74,9 @@ export const CreateV2EvalRunRequest$zodSchema: z.ZodType<
   eval_id: z.string().optional().describe(
     "Saved-run form only. Existing eval definition to run. Omit this field to use the direct create-and-run form.",
   ),
+  include_env_vars: z.boolean().optional().describe(
+    "Either request form. false runs this launch with no environment: neither the configuration's environment nor the organization's default environment is attached. Omit it to use the eval's saved setting. Cannot be false when `env` is sent.",
+  ),
   judges: z.array(z.string()).nullable().optional().describe(
     "Success criteria. Required in direct mode; in saved mode, replaces saved judges for this launch. CLI clients expose repeatable `--judge`.",
   ),

package/src/models/graderdefinition.ts

@@ -27,6 +27,7 @@ export type GraderDefinition = {
   pattern?: string | null | undefined;
   case_sensitive?: boolean | undefined;
   criteria?: string | null | undefined;
+  plugin?: string | null | undefined;
   metric?: Metric | null | undefined;
   min_score?: number | null | undefined;
   max_value?: number | null | undefined;
@@ -48,6 +49,7 @@ export const GraderDefinition$zodSchema: z.ZodType<GraderDefinition> = z.object(
     needle: z.string().nullable().optional(),
     needles: z.array(z.string()).nullable().optional(),
     pattern: z.string().nullable().optional(),
+    plugin: z.string().nullable().optional(),
     type: z.string(),
     weight: z.number().default(1),
   },

package/src/models/publiccriterionverdict.ts

@@ -7,15 +7,16 @@ import * as z from "zod";
 import { ClosedEnum } from "../types/enums.js";
 
 /**
- * `PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score.
+ * `PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score. `NOT_APPLICABLE` when the requirement states its own condition ("If X is used, ...") and that condition did not occur; it neither passes nor fails the run.
  */
 export const Verdict = {
   Pass: "PASS",
   Fail: "FAIL",
   NotVerified: "NOT_VERIFIED",
+  NotApplicable: "NOT_APPLICABLE",
 } as const;
 /**
- * `PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score.
+ * `PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score. `NOT_APPLICABLE` when the requirement states its own condition ("If X is used, ...") and that condition did not occur; it neither passes nor fails the run.
  */
 export type Verdict = ClosedEnum<typeof Verdict>;
 
@@ -23,8 +24,9 @@ export const Verdict$zodSchema = z.enum([
   "PASS",
   "FAIL",
   "NOT_VERIFIED",
+  "NOT_APPLICABLE",
 ]).describe(
-  "`PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score.",
+  "`PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score. `NOT_APPLICABLE` when the requirement states its own condition (\"If X is used, ...\") and that condition did not occur; it neither passes nor fails the run.",
 );
 
 export type PublicCriterionVerdict = {
@@ -43,6 +45,6 @@ export const PublicCriterionVerdict$zodSchema: z.ZodType<
     "Short evidence the judge gave for the verdict.",
   ),
   verdict: Verdict$zodSchema.describe(
-    "`PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score.",
+    "`PASS` when the evidence shows the requirement was met, `FAIL` when it shows it was not, and `NOT_VERIFIED` when the evidence cannot confirm it. Any `FAIL` or `NOT_VERIFIED` fails the run, whatever the score. `NOT_APPLICABLE` when the requirement states its own condition (\"If X is used, ...\") and that condition did not occur; it neither passes nor fails the run.",
   ),
 });

package/src/models/publicgraderresult.ts

@@ -17,6 +17,7 @@ export type PublicGraderResult = {
   deductions?: Array<string> | undefined;
   details?: string | null | undefined;
   criteria_verdicts?: Array<PublicCriterionVerdict> | undefined;
+  not_applicable?: boolean | undefined;
 };
 
 export const PublicGraderResult$zodSchema: z.ZodType<PublicGraderResult> = z
@@ -35,6 +36,9 @@ export const PublicGraderResult$zodSchema: z.ZodType<PublicGraderResult> = z
       "Display name for the check, when the grader sets one.",
     ),
     name: z.string().describe("Grader name as configured on the eval."),
+    not_applicable: z.boolean().default(false).describe(
+      "Judges only: true when every requirement was `NOT_APPLICABLE`. The check is left out of the run's score and does not fail the run.",
+    ),
     passed: z.boolean().nullable().optional().describe(
       "Whether this check passed.",
     ),

package/src/models/publicpluginevalresultsdownload.ts

@@ -0,0 +1,34 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 56e8f4aa6dc9
+ */
+
+import * as z from "zod";
+
+export type PublicPluginEvalResultsDownload = {
+  plugin: string;
+  runs?: number | null | undefined;
+  prompts?: number | null | undefined;
+  filename?: string | null | undefined;
+  download_url: string;
+  created_at?: string | null | undefined;
+};
+
+export const PublicPluginEvalResultsDownload$zodSchema: z.ZodType<
+  PublicPluginEvalResultsDownload
+> = z.object({
+  created_at: z.iso.datetime({ offset: true }).nullable().optional().describe(
+    "When the results were prepared.",
+  ),
+  download_url: z.string().describe(
+    "Downloads the zip of the day's runs and their insights report. Call it with the same API key.",
+  ),
+  filename: z.string().nullable().optional().describe("The zip file's name."),
+  plugin: z.string().describe("The plugin's slug, such as `firecrawl`."),
+  prompts: z.int().nullable().optional().describe(
+    "Distinct prompts included in the download.",
+  ),
+  runs: z.int().nullable().optional().describe(
+    "Runs included in the download.",
+  ),
+});

package/src/models/publicpluginevalresultsresponse.ts

@@ -0,0 +1,24 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: f47fd4bfe2d1
+ */
+
+import * as z from "zod";
+import {
+  PublicPluginEvalResultsDownload,
+  PublicPluginEvalResultsDownload$zodSchema,
+} from "./publicpluginevalresultsdownload.js";
+
+export type PublicPluginEvalResultsResponse = {
+  date: string;
+  results: Array<PublicPluginEvalResultsDownload>;
+};
+
+export const PublicPluginEvalResultsResponse$zodSchema: z.ZodType<
+  PublicPluginEvalResultsResponse
+> = z.object({
+  date: z.string().describe("The day the results cover, as YYYY-MM-DD."),
+  results: z.array(PublicPluginEvalResultsDownload$zodSchema).describe(
+    "One download per plugin with results on that day.",
+  ),
+});

package/src/models/updatev2evalrequest.ts

@@ -34,6 +34,7 @@ export type UpdateV2EvalRequest = {
   judges?: Array<string> | null | undefined;
   agent?: Array<string> | null | undefined;
   persona?: string | undefined;
+  include_env_vars?: boolean | undefined;
   clis?: Array<string> | null | undefined;
   mcps?: Array<V2InlineMcpDefinition | V2SavedMcpReference> | null | undefined;
   setup_commands?: Array<string> | null | undefined;
@@ -52,6 +53,9 @@ export const UpdateV2EvalRequest$zodSchema: z.ZodType<UpdateV2EvalRequest> = z
     clis: z.array(z.string()).nullable().optional().describe(
       "Replacement saved CLI IDs or shell installation commands. UUID values resolve saved CLIs; every other value is an installation command. Pass an empty array to clear all CLIs.",
     ),
+    include_env_vars: z.boolean().optional().describe(
+      "Replacement environment setting. false runs the eval with no environment; omit it to keep the saved value.",
+    ),
     judges: z.array(z.string()).nullable().optional().describe(
       "Replacement success criteria for the eval.",
     ),

package/src/models/v2eval.ts

@@ -45,6 +45,7 @@ export type V2Eval = {
   prompts?: Array<string> | undefined;
   judges?: Array<string> | undefined;
   agent?: Array<string> | undefined;
+  include_env_vars?: boolean | undefined;
   persona?: string | null | undefined;
   clis?: Array<V2SavedCliReference | V2InlineCliReference> | undefined;
   mcps?: Array<V2InlineMcpDefinition | V2SavedMcpReference> | undefined;
@@ -66,6 +67,9 @@ export const V2Eval$zodSchema: z.ZodType<V2Eval> = z.object({
   ])).optional(),
   created_at: z.string().nullable().optional(),
   id: z.string(),
+  include_env_vars: z.boolean().default(true).describe(
+    "Whether runs get environment variables. false means runs use no environment, as Environment: Off in the app.",
+  ),
   judges: z.array(z.string()).optional(),
   mcps: z.array(z.union([
     V2InlineMcpDefinition$zodSchema,

package/src/models/v2evalresultsummary.ts

@@ -24,6 +24,10 @@ import {
   V2PreferenceResult,
   V2PreferenceResult$zodSchema,
 } from "./v2preferenceresult.js";
+import {
+  V2SetupCommandOutput,
+  V2SetupCommandOutput$zodSchema,
+} from "./v2setupcommandoutput.js";
 
 export type V2EvalResultSummary = {
   id: string;
@@ -46,6 +50,7 @@ export type V2EvalResultSummary = {
   grounding_qualification_reason?: string | null | undefined;
   created_at?: string | null | undefined;
   environment?: PublicEvalRunEnvironment | null | undefined;
+  setup_output?: Array<V2SetupCommandOutput> | undefined;
 };
 
 export const V2EvalResultSummary$zodSchema: z.ZodType<V2EvalResultSummary> = z
@@ -82,6 +87,9 @@ export const V2EvalResultSummary$zodSchema: z.ZodType<V2EvalResultSummary> = z
     score: z.number().nullable().optional().describe("Score from 0 to 1."),
     search_grounding: SearchGroundingMode$zodSchema.nullable().optional(),
     search_observed: z.boolean().nullable().optional(),
+    setup_output: z.array(V2SetupCommandOutput$zodSchema).optional().describe(
+      "Setup commands run before the agent started, in order, with the end of what each printed. Secret values are redacted. Empty when the eval has no setup commands or the attempt ran before setup output was recorded.",
+    ),
     tool_calls: z.array(V2EvalResultToolCall$zodSchema).optional().describe(
       "Every tool call the agent made, in order, with how long each took. Empty when the agent made no tool calls or the model does not use tools.",
     ),

package/src/models/v2evalresulttoolcall.ts

@@ -7,24 +7,24 @@ import * as z from "zod";
 import { ClosedEnum } from "../types/enums.js";
 
 /**
- * How the time was measured. `agent_reported`: the agent timed the call itself; `completed_at` is when its result reached us and `started_at` is that duration earlier. `event_receipt`: when the call and its result reached us as the agent streamed them, so times include streaming delay. `unavailable`: the agent's log has no timing for this call, for example because the agent reported the call only after it finished, so the times are null.
+ * How the time was measured. `timed_by_agent`: the agent timed the call itself; `completed_at` is when its result reached us and `started_at` is that duration earlier. `timed_from_messages`: the time between the agent's call message and its result message reaching us, so it includes message delay and runs slightly long. `not_timed`: there is no reliable time for this call, for example because the agent reported the call only after it finished, so the times are null.
  */
 export const TimingBasis = {
-  AgentReported: "agent_reported",
-  EventReceipt: "event_receipt",
-  Unavailable: "unavailable",
+  TimedByAgent: "timed_by_agent",
+  TimedFromMessages: "timed_from_messages",
+  NotTimed: "not_timed",
 } as const;
 /**
- * How the time was measured. `agent_reported`: the agent timed the call itself; `completed_at` is when its result reached us and `started_at` is that duration earlier. `event_receipt`: when the call and its result reached us as the agent streamed them, so times include streaming delay. `unavailable`: the agent's log has no timing for this call, for example because the agent reported the call only after it finished, so the times are null.
+ * How the time was measured. `timed_by_agent`: the agent timed the call itself; `completed_at` is when its result reached us and `started_at` is that duration earlier. `timed_from_messages`: the time between the agent's call message and its result message reaching us, so it includes message delay and runs slightly long. `not_timed`: there is no reliable time for this call, for example because the agent reported the call only after it finished, so the times are null.
  */
 export type TimingBasis = ClosedEnum<typeof TimingBasis>;
 
 export const TimingBasis$zodSchema = z.enum([
-  "agent_reported",
-  "event_receipt",
-  "unavailable",
+  "timed_by_agent",
+  "timed_from_messages",
+  "not_timed",
 ]).describe(
-  "How the time was measured. `agent_reported`: the agent timed the call itself; `completed_at` is when its result reached us and `started_at` is that duration earlier. `event_receipt`: when the call and its result reached us as the agent streamed them, so times include streaming delay. `unavailable`: the agent's log has no timing for this call, for example because the agent reported the call only after it finished, so the times are null.",
+  "How the time was measured. `timed_by_agent`: the agent timed the call itself; `completed_at` is when its result reached us and `started_at` is that duration earlier. `timed_from_messages`: the time between the agent's call message and its result message reaching us, so it includes message delay and runs slightly long. `not_timed`: there is no reliable time for this call, for example because the agent reported the call only after it finished, so the times are null.",
 );
 
 /**
@@ -51,7 +51,7 @@ export const V2EvalResultToolCall$zodSchema: z.ZodType<V2EvalResultToolCall> = z
       "When the call was received. Null when not measured.",
     ),
     timing_basis: TimingBasis$zodSchema.describe(
-      "How the time was measured. `agent_reported`: the agent timed the call itself; `completed_at` is when its result reached us and `started_at` is that duration earlier. `event_receipt`: when the call and its result reached us as the agent streamed them, so times include streaming delay. `unavailable`: the agent's log has no timing for this call, for example because the agent reported the call only after it finished, so the times are null.",
+      "How the time was measured. `timed_by_agent`: the agent timed the call itself; `completed_at` is when its result reached us and `started_at` is that duration earlier. `timed_from_messages`: the time between the agent's call message and its result message reaching us, so it includes message delay and runs slightly long. `not_timed`: there is no reliable time for this call, for example because the agent reported the call only after it finished, so the times are null.",
     ),
     tool_call_id: z.string().nullable().optional().describe(
       "ID of the tool invocation, the same as `tool_call_id` on this attempt's trace events. Null when the agent's log did not include call IDs.",

package/src/models/v2pluginevalsresultslistop.ts

@@ -0,0 +1,35 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 4edf2fbdf8be
+ */
+
+import * as z from "zod";
+import {
+  HTTPValidationError,
+  HTTPValidationError$zodSchema,
+} from "./httpvalidationerror.js";
+import {
+  PublicPluginEvalResultsResponse,
+  PublicPluginEvalResultsResponse$zodSchema,
+} from "./publicpluginevalresultsresponse.js";
+
+export type V2PluginEvalsResultsListRequest = { date: string };
+
+export const V2PluginEvalsResultsListRequest$zodSchema: z.ZodType<
+  V2PluginEvalsResultsListRequest
+> = z.object({
+  date: z.string().date().describe(
+    "The day to get results for, as YYYY-MM-DD (UTC).",
+  ),
+});
+
+export type V2PluginEvalsResultsListResponse =
+  | PublicPluginEvalResultsResponse
+  | HTTPValidationError;
+
+export const V2PluginEvalsResultsListResponse$zodSchema: z.ZodType<
+  V2PluginEvalsResultsListResponse
+> = z.union([
+  PublicPluginEvalResultsResponse$zodSchema,
+  HTTPValidationError$zodSchema,
+]);

package/src/models/v2promptrunjobresponse.ts

@@ -73,7 +73,7 @@ export const V2PromptRunJobResponse$zodSchema: z.ZodType<
   dispatch_id: z.string().nullable().optional(),
   error: z.string().nullable().optional(),
   estimated_cost: z.number().nullable().optional().describe(
-    "The estimated credit cost for prompt jobs or estimated USD cost for API Performance jobs. Null when no estimate is available.",
+    "Estimated credits for the run, worked out when it was queued. Credits actually used can differ by agent and model. Null when no estimate is available.",
   ),
   eval_count: z.int().nullable().optional(),
   id: z.string().describe(

package/src/models/v2setupcommandoutput.ts

@@ -0,0 +1,31 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: cee0aa951ed5
+ */
+
+import * as z from "zod";
+
+export type V2SetupCommandOutput = {
+  command: string;
+  exit_code?: number | null | undefined;
+  status?: string | null | undefined;
+  output?: string | null | undefined;
+  output_truncated?: boolean | undefined;
+};
+
+export const V2SetupCommandOutput$zodSchema: z.ZodType<V2SetupCommandOutput> = z
+  .object({
+    command: z.string().describe("The setup command as saved on the eval."),
+    exit_code: z.int().nullable().optional().describe(
+      "The command's exit code.",
+    ),
+    output: z.string().nullable().optional().describe(
+      "The end of what the command printed (up to 4,000 characters), with secret values redacted.",
+    ),
+    output_truncated: z.boolean().default(false).describe(
+      "True when earlier output was cut off.",
+    ),
+    status: z.string().nullable().optional().describe(
+      "`succeeded` for a completed command.",
+    ),
+  });

package/src/tool-names.ts

@@ -264,6 +264,10 @@ export const toolNames: Array<{ name: string; description: string }>= [
     "name": "agent-preference-evals-results-list",
     "description": "List Results\n\nReturns recorded attempts for an Agent Preference eval, newest first, each with whether your company won, lost, or tied against the competitor."
   },
+  {
+    "name": "plugin-evals-results-list",
+    "description": "List Plugin Eval Results\n\nReturns a day's plugin eval results, one per plugin with results that day. Each has a `download_url` for a zip of every run with and without the plugin installed, plus the insights report written from them. Call `download_url` with the same API key. Returns 404 when no results exist for the date."
+  },
   {
     "name": "agents-list",
     "description": "List Agents\n\nReturns the coding-agent harness-and-model combinations your plan can run, as single agent IDs. Pass them as `agent` to `POST /v2/prompts/runs` to see how coding agents answer your prompts, with mentions and citations, or to `PUT /v2/prompts/targets` to run them on a schedule; to `POST /v2/agent-preference/evals` to see whether agents recommend your company over one competitor; or to `POST /v2/evals` and `POST /v2/eval-runs` to measure how well agents complete tasks with your product."

package/test/simulations.test.js

@@ -174,7 +174,8 @@ test("MCP tools forward canonical eval resources unchanged", async () => {
         method: "POST",
         url: "/v2/evals",
         apiKey: "ls_simulation_test",
-        body: createRequest,
+        // The schema fills in include_env_vars' default (true) on create.
+        body: { ...createRequest, include_env_vars: true },
       },
       {
         method: "POST",

package/test/tool-overrides.test.js

@@ -0,0 +1,70 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+const { Client } = await import("@modelcontextprotocol/sdk/client/index.js");
+const { InMemoryTransport } = await import(
+  "@modelcontextprotocol/sdk/inMemory.js"
+);
+const { createConsoleLogger } = await import(
+  "../esm/mcp-server/console-logger.js"
+);
+const { createMCPServer } = await import("../esm/mcp-server/server.js");
+const { ANNOTATION_OVERRIDES, TOOL_RENAMES, applyToolOverrides } = await import(
+  "../lib/tool-overrides.js"
+);
+
+async function listTools(allowedTools) {
+  const { server } = createMCPServer({
+    logger: createConsoleLogger("error"),
+    allowedTools,
+    dynamic: false,
+    serverURL: "https://api.lightsage.com",
+    getSDK: () => {
+      throw new Error("not called when listing tools");
+    },
+  });
+  applyToolOverrides(server);
+
+  const [clientTransport, serverTransport] =
+    InMemoryTransport.createLinkedPair();
+  const client = new Client({ name: "test", version: "1.0.0" });
+  await Promise.all([
+    server.connect(serverTransport),
+    client.connect(clientTransport),
+  ]);
+  const { tools } = await client.listTools();
+  await client.close();
+  return new Map(tools.map((tool) => [tool.name, tool]));
+}
+
+test("corrected tools carry the overridden hints", async () => {
+  const tools = await listTools(undefined);
+  for (const [name, overrides] of Object.entries(ANNOTATION_OVERRIDES)) {
+    const tool = tools.get(name);
+    assert.ok(tool, `${name} is listed`);
+    for (const [hint, value] of Object.entries(overrides)) {
+      assert.equal(tool.annotations?.[hint], value, `${name}.${hint}`);
+    }
+    for (const hint of ["readOnlyHint", "destructiveHint", "openWorldHint"]) {
+      assert.equal(typeof tool.annotations?.[hint], "boolean", `${name}.${hint}`);
+    }
+  }
+});
+
+test("unclear generated names are listed under their new names", async () => {
+  const tools = await listTools(undefined);
+  for (const [from, to] of Object.entries(TOOL_RENAMES)) {
+    assert.equal(tools.has(from), false, `${from} is gone`);
+    assert.ok(tools.has(to), `${to} is listed`);
+  }
+});
+
+test("renames apply to tools kept by the allowlist", async () => {
+  const tools = await listTools(["CL-is-list", "prompts-list"]);
+  assert.deepEqual([...tools.keys()].sort(), ["clis-list", "prompts-list"]);
+});
+
+test("tools without overrides keep their generated hints", async () => {
+  const tools = await listTools(["prompts-list"]);
+  assert.equal(tools.get("prompts-list").annotations.readOnlyHint, true);
+});