@lightsage/mcp 0.4.0 → 0.4.2

package/src/mcp-server/tools/evalTemplatesV2EvalTemplatesRetrieve.ts

@@ -0,0 +1,45 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: c0e5159c6665
+ */
+
+import { evalTemplatesV2EvalTemplatesRetrieve } from "../../funcs/evalTemplatesV2EvalTemplatesRetrieve.js";
+import { V2EvalTemplatesRetrieveRequest$zodSchema } from "../../models/v2evaltemplatesretrieveop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: V2EvalTemplatesRetrieveRequest$zodSchema,
+};
+
+export const tool$evalTemplatesV2EvalTemplatesRetrieve: ToolDefinition<
+  typeof args
+> = {
+  name: "eval-templates-v2-eval-templates-retrieve",
+  description: `Retrieve an Eval template
+
+Returns one active reusable Eval template.`,
+  annotations: {
+    "title": "",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": false,
+    "readOnlyHint": true,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await evalTemplatesV2EvalTemplatesRetrieve(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/evalTemplatesV2EvalTemplatesUpdate.ts

@@ -0,0 +1,45 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 1f25df1eae88
+ */
+
+import { evalTemplatesV2EvalTemplatesUpdate } from "../../funcs/evalTemplatesV2EvalTemplatesUpdate.js";
+import { V2EvalTemplatesUpdateRequest$zodSchema } from "../../models/v2evaltemplatesupdateop.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: V2EvalTemplatesUpdateRequest$zodSchema,
+};
+
+export const tool$evalTemplatesV2EvalTemplatesUpdate: ToolDefinition<
+  typeof args
+> = {
+  name: "eval-templates-v2-eval-templates-update",
+  description: `Update an Eval template
+
+Updates an Eval template. Supply \`expected_template_version\` to reject an edit based on stale data. Existing evals are never changed.`,
+  annotations: {
+    "title": "",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": false,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await evalTemplatesV2EvalTemplatesUpdate(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/evalsV2EvalsBatchCreate.ts

@@ -0,0 +1,43 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 18f4c7ff36d5
+ */
+
+import { evalsV2EvalsBatchCreate } from "../../funcs/evalsV2EvalsBatchCreate.js";
+import { CreateEvalTemplateBatchRequest$zodSchema } from "../../models/createevaltemplatebatchrequest.js";
+import { formatResult, ToolDefinition } from "../tools.js";
+
+const args = {
+  request: CreateEvalTemplateBatchRequest$zodSchema,
+};
+
+export const tool$evalsV2EvalsBatchCreate: ToolDefinition<typeof args> = {
+  name: "evals-v2-evals-batch-create",
+  description: `Create evals from a template
+
+Step 2 of the reusable template workflow. Provide the \`id\` and \`version\` returned by \`POST /v2/eval-templates\` as \`template_id\` and \`expected_template_version\`, then provide only prompt-specific fields in \`evals\`. The template supplies only the settings named in its \`included_components\`; omitted settings keep the values supplied by the creation workflow or its defaults. Included values are copied at creation time, so later template changes do not alter the created evals. Competitor selections create one additional eval per selected competitor for every input item, and the response groups each base eval with those variants. Brand-scoped runtime resources are validated against the organization's primary brand before any eval is created.`,
+  annotations: {
+    "title": "",
+    "destructiveHint": false,
+    "idempotentHint": false,
+    "openWorldHint": false,
+    "readOnlyHint": false,
+  },
+  args,
+  tool: async (client, args, ctx) => {
+    const [result] = await evalsV2EvalsBatchCreate(
+      client,
+      args.request,
+      { fetchOptions: { signal: ctx.signal } },
+    ).$inspect();
+
+    if (!result.ok) {
+      return {
+        content: [{ type: "text", text: result.error.message }],
+        isError: true,
+      };
+    }
+
+    return formatResult(result.value);
+  },
+};

package/src/mcp-server/tools/promptsV2PromptsRetrieve.ts

@@ -15,7 +15,7 @@ export const tool$promptsV2PromptsRetrieve: ToolDefinition<typeof args> = {
   name: "prompts-v2-prompts-retrieve",
   description: `Retrieve Prompt
 
-Returns a prompt with its topic, platforms, tags, and lifecycle status.`,
+Returns a prompt with its topic, platforms, tags, lifecycle status, and coding-agent run settings.`,
   annotations: {
     "title": "",
     "destructiveHint": false,

package/src/mcp-server/tools/promptsV2PromptsUpdate.ts

@@ -15,7 +15,7 @@ export const tool$promptsV2PromptsUpdate: ToolDefinition<typeof args> = {
   name: "prompts-v2-prompts-update",
   description: `Update Prompt
 
-Updates a prompt's text, topic, platforms, or lifecycle status. Omitted fields remain unchanged.`,
+Updates a prompt's text, topic, platforms, lifecycle status, or coding-agent run settings. Omitted fields remain unchanged.`,
   annotations: {
     "title": "",
     "destructiveHint": false,

package/src/models/createevaltemplatebatchrequest.ts

@@ -0,0 +1,35 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 6c3f16ae5afb
+ */
+
+import * as z from "zod";
+import {
+  EvalTemplateBatchItem,
+  EvalTemplateBatchItem$zodSchema,
+} from "./evaltemplatebatchitem.js";
+
+/**
+ * Reuse one template across up to 100 prompt-specific eval definitions.
+ */
+export type CreateEvalTemplateBatchRequest = {
+  template_id: string;
+  expected_template_version?: number | undefined;
+  evals: Array<EvalTemplateBatchItem>;
+};
+
+export const CreateEvalTemplateBatchRequest$zodSchema: z.ZodType<
+  CreateEvalTemplateBatchRequest
+> = z.object({
+  evals: z.array(EvalTemplateBatchItem$zodSchema).describe(
+    "Prompt-specific evals to create from the referenced template. Template-owned fields are intentionally rejected here to avoid ambiguous overrides.",
+  ),
+  expected_template_version: z.int().optional().describe(
+    "Template `version` returned when the template was read or created. When supplied, creation fails if the template changed before reuse.",
+  ),
+  template_id: z.string().describe(
+    "Template `id` returned by `POST /v2/eval-templates`.",
+  ),
+}).describe(
+  "Reuse one template across up to 100 prompt-specific eval definitions.",
+);

package/src/models/createevaltemplaterequest.ts

@@ -0,0 +1,151 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 42ac0a46d319
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+import {
+  EvalTemplateRuntime,
+  EvalTemplateRuntime$zodSchema,
+} from "./evaltemplateruntime.js";
+import {
+  EvalTemplateSchedule,
+  EvalTemplateSchedule$zodSchema,
+} from "./evaltemplateschedule.js";
+import {
+  GraderDefinition,
+  GraderDefinition$zodSchema,
+} from "./graderdefinition.js";
+
+/**
+ * Kind of eval the template creates. A template only applies to evals of the same type.
+ */
+export const CreateEvalTemplateRequestEvalType = {
+  CustomEval: "custom_eval",
+  AgentPreference: "agent_preference",
+} as const;
+/**
+ * Kind of eval the template creates. A template only applies to evals of the same type.
+ */
+export type CreateEvalTemplateRequestEvalType = ClosedEnum<
+  typeof CreateEvalTemplateRequestEvalType
+>;
+
+export const CreateEvalTemplateRequestEvalType$zodSchema = z.enum([
+  "custom_eval",
+  "agent_preference",
+]).describe(
+  "Kind of eval the template creates. A template only applies to evals of the same type.",
+);
+
+export const CreateEvalTemplateRequestIncludedComponent = {
+  Judges: "judges",
+  Targets: "targets",
+  Persona: "persona",
+  Competitors: "competitors",
+  Tag: "tag",
+  Category: "category",
+  Runs: "runs",
+  Schedule: "schedule",
+  Repository: "repository",
+  Environment: "environment",
+  SearchGrounding: "search_grounding",
+  Skills: "skills",
+  McpServers: "mcp_servers",
+  Clis: "clis",
+  BrandDocs: "brand_docs",
+  GithubPr: "github_pr",
+  RuntimeCommands: "runtime_commands",
+} as const;
+export type CreateEvalTemplateRequestIncludedComponent = ClosedEnum<
+  typeof CreateEvalTemplateRequestIncludedComponent
+>;
+
+export const CreateEvalTemplateRequestIncludedComponent$zodSchema = z.enum([
+  "judges",
+  "targets",
+  "persona",
+  "competitors",
+  "tag",
+  "category",
+  "runs",
+  "schedule",
+  "repository",
+  "environment",
+  "search_grounding",
+  "skills",
+  "mcp_servers",
+  "clis",
+  "brand_docs",
+  "github_pr",
+  "runtime_commands",
+]);
+
+/**
+ * Create reusable configuration before supplying prompt-specific evals.
+ */
+export type CreateEvalTemplateRequest = {
+  name: string;
+  description?: string | undefined;
+  eval_type?: CreateEvalTemplateRequestEvalType | undefined;
+  included_components: Array<CreateEvalTemplateRequestIncludedComponent>;
+  persona?: string | undefined;
+  category_name?: string | undefined;
+  tag?: string | undefined;
+  competitor_ids?: Array<string> | undefined;
+  judges?: Array<GraderDefinition> | undefined;
+  target_keys?: Array<string> | undefined;
+  schedule?: EvalTemplateSchedule | undefined;
+  runs_per_execution?: number | undefined;
+  runtime?: EvalTemplateRuntime | undefined;
+};
+
+export const CreateEvalTemplateRequest$zodSchema: z.ZodType<
+  CreateEvalTemplateRequest
+> = z.object({
+  category_name: z.string().optional().describe(
+    "Optional category copied when `category` is included.",
+  ),
+  competitor_ids: z.array(z.string()).optional().describe(
+    "Competitor IDs that create additional variants when `competitors` is included.",
+  ),
+  description: z.string().optional().describe(
+    "Optional guidance describing when to use the template.",
+  ),
+  eval_type: CreateEvalTemplateRequestEvalType$zodSchema.default("custom_eval")
+    .describe(
+      "Kind of eval the template creates. A template only applies to evals of the same type.",
+    ),
+  included_components: z.array(
+    CreateEvalTemplateRequestIncludedComponent$zodSchema,
+  ).describe(
+    "Settings copied when the template is applied. Omitted components do not override the new eval; included components copy their saved value, including an empty value.",
+  ),
+  judges: z.array(GraderDefinition$zodSchema).optional().describe(
+    "Judge definitions copied when `judges` is included. May be empty to explicitly create the eval without template judges.",
+  ),
+  name: z.string().describe(
+    "Organization-visible name for this reusable template.",
+  ),
+  persona: z.string().optional().describe(
+    "Optional persona copied when `persona` is included.",
+  ),
+  runs_per_execution: z.int().default(1).describe(
+    "Attempt count copied when `runs` is included.",
+  ),
+  runtime: EvalTemplateRuntime$zodSchema.optional().describe(
+    "Reusable runtime resources copied into each created eval.",
+  ),
+  schedule: EvalTemplateSchedule$zodSchema.optional().describe(
+    "Schedule copied into every eval created from the template.",
+  ),
+  tag: z.string().optional().describe(
+    "Optional tag copied when `tag` is included.",
+  ),
+  target_keys: z.array(z.string()).optional().describe(
+    "Runnable target keys copied when `targets` is included, such as `agent:codex:gpt-5.6-terra`.",
+  ),
+}).describe(
+  "Create reusable configuration before supplying prompt-specific evals.",
+);

package/src/models/createv2promptrequest.ts

@@ -4,11 +4,16 @@
  */
 
 import * as z from "zod";
+import {
+  V2PromptRunSettingsInput,
+  V2PromptRunSettingsInput$zodSchema,
+} from "./v2promptrunsettingsinput.js";
 
 export type CreateV2PromptRequest = {
   topic_id: string;
   text: string;
   platform_ids?: Array<string> | undefined;
+  run_settings?: V2PromptRunSettingsInput | null | undefined;
 };
 
 export const CreateV2PromptRequest$zodSchema: z.ZodType<CreateV2PromptRequest> =
@@ -16,6 +21,10 @@ export const CreateV2PromptRequest$zodSchema: z.ZodType<CreateV2PromptRequest> =
     platform_ids: z.array(z.string()).optional().describe(
       "Answer-platform IDs from the prompt configuration. Omit to attach none.",
     ),
+    run_settings: V2PromptRunSettingsInput$zodSchema.nullable().optional()
+      .describe(
+        "Run settings that apply to coding-agent runs of this prompt. Omitted fields keep their current or default values.",
+      ),
     text: z.string(),
     topic_id: z.string().describe(
       "Prompt topic ID from `GET /v2/prompts/topics`.",

package/src/models/evaltemplate.ts

@@ -0,0 +1,154 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: ea68162f8775
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+import {
+  EvalTemplateRuntime,
+  EvalTemplateRuntime$zodSchema,
+} from "./evaltemplateruntime.js";
+import {
+  EvalTemplateSchedule,
+  EvalTemplateSchedule$zodSchema,
+} from "./evaltemplateschedule.js";
+import {
+  GraderDefinition,
+  GraderDefinition$zodSchema,
+} from "./graderdefinition.js";
+
+/**
+ * Kind of eval the template creates. A template only applies to evals of the same type.
+ */
+export const EvalTemplateEvalType = {
+  CustomEval: "custom_eval",
+  AgentPreference: "agent_preference",
+} as const;
+/**
+ * Kind of eval the template creates. A template only applies to evals of the same type.
+ */
+export type EvalTemplateEvalType = ClosedEnum<typeof EvalTemplateEvalType>;
+
+export const EvalTemplateEvalType$zodSchema = z.enum([
+  "custom_eval",
+  "agent_preference",
+]).describe(
+  "Kind of eval the template creates. A template only applies to evals of the same type.",
+);
+
+export const EvalTemplateIncludedComponent = {
+  Judges: "judges",
+  Targets: "targets",
+  Persona: "persona",
+  Competitors: "competitors",
+  Tag: "tag",
+  Category: "category",
+  Runs: "runs",
+  Schedule: "schedule",
+  Repository: "repository",
+  Environment: "environment",
+  SearchGrounding: "search_grounding",
+  Skills: "skills",
+  McpServers: "mcp_servers",
+  Clis: "clis",
+  BrandDocs: "brand_docs",
+  GithubPr: "github_pr",
+  RuntimeCommands: "runtime_commands",
+} as const;
+export type EvalTemplateIncludedComponent = ClosedEnum<
+  typeof EvalTemplateIncludedComponent
+>;
+
+export const EvalTemplateIncludedComponent$zodSchema = z.enum([
+  "judges",
+  "targets",
+  "persona",
+  "competitors",
+  "tag",
+  "category",
+  "runs",
+  "schedule",
+  "repository",
+  "environment",
+  "search_grounding",
+  "skills",
+  "mcp_servers",
+  "clis",
+  "brand_docs",
+  "github_pr",
+  "runtime_commands",
+]);
+
+export type EvalTemplate = {
+  name: string;
+  description?: string | null | undefined;
+  eval_type?: EvalTemplateEvalType | undefined;
+  included_components: Array<EvalTemplateIncludedComponent>;
+  persona?: string | null | undefined;
+  category_name?: string | null | undefined;
+  tag?: string | null | undefined;
+  competitor_ids?: Array<string> | undefined;
+  judges?: Array<GraderDefinition> | undefined;
+  target_keys?: Array<string> | undefined;
+  schedule?: EvalTemplateSchedule | undefined;
+  runs_per_execution?: number | undefined;
+  runtime?: EvalTemplateRuntime | undefined;
+  id: string;
+  org_id: string;
+  version: number;
+  created_by?: string | null | undefined;
+  created_at: string;
+  updated_at: string;
+  deleted_at?: string | null | undefined;
+};
+
+export const EvalTemplate$zodSchema: z.ZodType<EvalTemplate> = z.object({
+  category_name: z.string().nullable().optional().describe(
+    "Optional category copied when `category` is included.",
+  ),
+  competitor_ids: z.array(z.string()).optional().describe(
+    "Competitor IDs that create additional variants when `competitors` is included.",
+  ),
+  created_at: z.iso.datetime({ offset: true }),
+  created_by: z.string().nullable().optional(),
+  deleted_at: z.iso.datetime({ offset: true }).nullable().optional(),
+  description: z.string().nullable().optional().describe(
+    "Optional guidance describing when to use the template.",
+  ),
+  eval_type: EvalTemplateEvalType$zodSchema.default("custom_eval").describe(
+    "Kind of eval the template creates. A template only applies to evals of the same type.",
+  ),
+  id: z.string(),
+  included_components: z.array(EvalTemplateIncludedComponent$zodSchema)
+    .describe(
+      "Settings copied when the template is applied. Omitted components do not override the new eval; included components copy their saved value, including an empty value.",
+    ),
+  judges: z.array(GraderDefinition$zodSchema).optional().describe(
+    "Judge definitions copied when `judges` is included. May be empty to explicitly create the eval without template judges.",
+  ),
+  name: z.string().describe(
+    "Organization-visible name for this reusable template.",
+  ),
+  org_id: z.string(),
+  persona: z.string().nullable().optional().describe(
+    "Optional persona copied when `persona` is included.",
+  ),
+  runs_per_execution: z.int().default(1).describe(
+    "Attempt count copied when `runs` is included.",
+  ),
+  runtime: EvalTemplateRuntime$zodSchema.optional().describe(
+    "Reusable runtime resources copied into each created eval.",
+  ),
+  schedule: EvalTemplateSchedule$zodSchema.optional().describe(
+    "Schedule copied into every eval created from the template.",
+  ),
+  tag: z.string().nullable().optional().describe(
+    "Optional tag copied when `tag` is included.",
+  ),
+  target_keys: z.array(z.string()).optional().describe(
+    "Runnable target keys copied when `targets` is included, such as `agent:codex:gpt-5.6-terra`.",
+  ),
+  updated_at: z.iso.datetime({ offset: true }),
+  version: z.int(),
+});

package/src/models/evaltemplatebatchcreatedeval.ts

@@ -0,0 +1,26 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 98054871a9ae
+ */
+
+import * as z from "zod";
+
+export type EvalTemplateBatchCreatedEval = {
+  id: string;
+  eval_id?: string | null | undefined;
+  name: string;
+  prompt: string;
+  competitor_id?: string | null | undefined;
+  competitor_name?: string | null | undefined;
+};
+
+export const EvalTemplateBatchCreatedEval$zodSchema: z.ZodType<
+  EvalTemplateBatchCreatedEval
+> = z.object({
+  competitor_id: z.string().nullable().optional(),
+  competitor_name: z.string().nullable().optional(),
+  eval_id: z.string().nullable().optional(),
+  id: z.string(),
+  name: z.string(),
+  prompt: z.string(),
+});

package/src/models/evaltemplatebatchitem.ts

@@ -0,0 +1,54 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 23c0fa6bda51
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+/**
+ * Eval subject type used for organization and reporting.
+ */
+export const EvalTemplateBatchItemType = {
+  Workflow: "workflow",
+  Mcp: "mcp",
+  Cli: "cli",
+  Skill: "skill",
+} as const;
+/**
+ * Eval subject type used for organization and reporting.
+ */
+export type EvalTemplateBatchItemType = ClosedEnum<
+  typeof EvalTemplateBatchItemType
+>;
+
+export const EvalTemplateBatchItemType$zodSchema = z.enum([
+  "workflow",
+  "mcp",
+  "cli",
+  "skill",
+]).describe("Eval subject type used for organization and reporting.");
+
+/**
+ * Prompt-specific fields combined with the referenced template.
+ */
+export type EvalTemplateBatchItem = {
+  name: string;
+  prompts: Array<string>;
+  type?: EvalTemplateBatchItemType | undefined;
+  subject_id?: string | null | undefined;
+};
+
+export const EvalTemplateBatchItem$zodSchema: z.ZodType<EvalTemplateBatchItem> =
+  z.object({
+    name: z.string().describe("Name of the prompt-specific eval to create."),
+    prompts: z.array(z.string()).describe(
+      "Task prompts owned by this eval. Judges, targets, scheduling, and runtime resources come from the template.",
+    ),
+    subject_id: z.string().nullable().optional().describe(
+      "Optional subject identifier associated with this eval.",
+    ),
+    type: EvalTemplateBatchItemType$zodSchema.default("workflow").describe(
+      "Eval subject type used for organization and reporting.",
+    ),
+  }).describe("Prompt-specific fields combined with the referenced template.");

package/src/models/evaltemplatebatchresponse.ts

@@ -0,0 +1,30 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: a63a246560d8
+ */
+
+import * as z from "zod";
+import {
+  EvalTemplateBatchResult,
+  EvalTemplateBatchResult$zodSchema,
+} from "./evaltemplatebatchresult.js";
+
+export type EvalTemplateBatchResponse = {
+  template_id: string;
+  template_version: number;
+  created_count: number;
+  base_count: number;
+  variant_count: number;
+  results?: Array<EvalTemplateBatchResult> | undefined;
+};
+
+export const EvalTemplateBatchResponse$zodSchema: z.ZodType<
+  EvalTemplateBatchResponse
+> = z.object({
+  base_count: z.int(),
+  created_count: z.int(),
+  results: z.array(EvalTemplateBatchResult$zodSchema).optional(),
+  template_id: z.string(),
+  template_version: z.int(),
+  variant_count: z.int(),
+});

package/src/models/evaltemplatebatchresult.ts

@@ -0,0 +1,26 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: f2e21f14a77e
+ */
+
+import * as z from "zod";
+import {
+  EvalTemplateBatchCreatedEval,
+  EvalTemplateBatchCreatedEval$zodSchema,
+} from "./evaltemplatebatchcreatedeval.js";
+
+export type EvalTemplateBatchResult = {
+  input_index: number;
+  name: string;
+  base: EvalTemplateBatchCreatedEval;
+  variants?: Array<EvalTemplateBatchCreatedEval> | undefined;
+};
+
+export const EvalTemplateBatchResult$zodSchema: z.ZodType<
+  EvalTemplateBatchResult
+> = z.object({
+  base: EvalTemplateBatchCreatedEval$zodSchema,
+  input_index: z.int(),
+  name: z.string(),
+  variants: z.array(EvalTemplateBatchCreatedEval$zodSchema).optional(),
+});