@lightsage/mcp 0.4.0 → 0.4.1

package/src/models/createevaltemplaterequest.ts

@@ -0,0 +1,151 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 42ac0a46d319
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+import {
+  EvalTemplateRuntime,
+  EvalTemplateRuntime$zodSchema,
+} from "./evaltemplateruntime.js";
+import {
+  EvalTemplateSchedule,
+  EvalTemplateSchedule$zodSchema,
+} from "./evaltemplateschedule.js";
+import {
+  GraderDefinition,
+  GraderDefinition$zodSchema,
+} from "./graderdefinition.js";
+
+/**
+ * Kind of eval the template creates. A template only applies to evals of the same type.
+ */
+export const CreateEvalTemplateRequestEvalType = {
+  CustomEval: "custom_eval",
+  AgentPreference: "agent_preference",
+} as const;
+/**
+ * Kind of eval the template creates. A template only applies to evals of the same type.
+ */
+export type CreateEvalTemplateRequestEvalType = ClosedEnum<
+  typeof CreateEvalTemplateRequestEvalType
+>;
+
+export const CreateEvalTemplateRequestEvalType$zodSchema = z.enum([
+  "custom_eval",
+  "agent_preference",
+]).describe(
+  "Kind of eval the template creates. A template only applies to evals of the same type.",
+);
+
+export const CreateEvalTemplateRequestIncludedComponent = {
+  Judges: "judges",
+  Targets: "targets",
+  Persona: "persona",
+  Competitors: "competitors",
+  Tag: "tag",
+  Category: "category",
+  Runs: "runs",
+  Schedule: "schedule",
+  Repository: "repository",
+  Environment: "environment",
+  SearchGrounding: "search_grounding",
+  Skills: "skills",
+  McpServers: "mcp_servers",
+  Clis: "clis",
+  BrandDocs: "brand_docs",
+  GithubPr: "github_pr",
+  RuntimeCommands: "runtime_commands",
+} as const;
+export type CreateEvalTemplateRequestIncludedComponent = ClosedEnum<
+  typeof CreateEvalTemplateRequestIncludedComponent
+>;
+
+export const CreateEvalTemplateRequestIncludedComponent$zodSchema = z.enum([
+  "judges",
+  "targets",
+  "persona",
+  "competitors",
+  "tag",
+  "category",
+  "runs",
+  "schedule",
+  "repository",
+  "environment",
+  "search_grounding",
+  "skills",
+  "mcp_servers",
+  "clis",
+  "brand_docs",
+  "github_pr",
+  "runtime_commands",
+]);
+
+/**
+ * Create reusable configuration before supplying prompt-specific evals.
+ */
+export type CreateEvalTemplateRequest = {
+  name: string;
+  description?: string | undefined;
+  eval_type?: CreateEvalTemplateRequestEvalType | undefined;
+  included_components: Array<CreateEvalTemplateRequestIncludedComponent>;
+  persona?: string | undefined;
+  category_name?: string | undefined;
+  tag?: string | undefined;
+  competitor_ids?: Array<string> | undefined;
+  judges?: Array<GraderDefinition> | undefined;
+  target_keys?: Array<string> | undefined;
+  schedule?: EvalTemplateSchedule | undefined;
+  runs_per_execution?: number | undefined;
+  runtime?: EvalTemplateRuntime | undefined;
+};
+
+export const CreateEvalTemplateRequest$zodSchema: z.ZodType<
+  CreateEvalTemplateRequest
+> = z.object({
+  category_name: z.string().optional().describe(
+    "Optional category copied when `category` is included.",
+  ),
+  competitor_ids: z.array(z.string()).optional().describe(
+    "Competitor IDs that create additional variants when `competitors` is included.",
+  ),
+  description: z.string().optional().describe(
+    "Optional guidance describing when to use the template.",
+  ),
+  eval_type: CreateEvalTemplateRequestEvalType$zodSchema.default("custom_eval")
+    .describe(
+      "Kind of eval the template creates. A template only applies to evals of the same type.",
+    ),
+  included_components: z.array(
+    CreateEvalTemplateRequestIncludedComponent$zodSchema,
+  ).describe(
+    "Settings copied when the template is applied. Omitted components do not override the new eval; included components copy their saved value, including an empty value.",
+  ),
+  judges: z.array(GraderDefinition$zodSchema).optional().describe(
+    "Judge definitions copied when `judges` is included. May be empty to explicitly create the eval without template judges.",
+  ),
+  name: z.string().describe(
+    "Organization-visible name for this reusable template.",
+  ),
+  persona: z.string().optional().describe(
+    "Optional persona copied when `persona` is included.",
+  ),
+  runs_per_execution: z.int().default(1).describe(
+    "Attempt count copied when `runs` is included.",
+  ),
+  runtime: EvalTemplateRuntime$zodSchema.optional().describe(
+    "Reusable runtime resources copied into each created eval.",
+  ),
+  schedule: EvalTemplateSchedule$zodSchema.optional().describe(
+    "Schedule copied into every eval created from the template.",
+  ),
+  tag: z.string().optional().describe(
+    "Optional tag copied when `tag` is included.",
+  ),
+  target_keys: z.array(z.string()).optional().describe(
+    "Runnable target keys copied when `targets` is included, such as `agent:codex:gpt-5.6-terra`.",
+  ),
+}).describe(
+  "Create reusable configuration before supplying prompt-specific evals.",
+);

package/src/models/evaltemplate.ts

@@ -0,0 +1,154 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: ea68162f8775
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+import {
+  EvalTemplateRuntime,
+  EvalTemplateRuntime$zodSchema,
+} from "./evaltemplateruntime.js";
+import {
+  EvalTemplateSchedule,
+  EvalTemplateSchedule$zodSchema,
+} from "./evaltemplateschedule.js";
+import {
+  GraderDefinition,
+  GraderDefinition$zodSchema,
+} from "./graderdefinition.js";
+
+/**
+ * Kind of eval the template creates. A template only applies to evals of the same type.
+ */
+export const EvalTemplateEvalType = {
+  CustomEval: "custom_eval",
+  AgentPreference: "agent_preference",
+} as const;
+/**
+ * Kind of eval the template creates. A template only applies to evals of the same type.
+ */
+export type EvalTemplateEvalType = ClosedEnum<typeof EvalTemplateEvalType>;
+
+export const EvalTemplateEvalType$zodSchema = z.enum([
+  "custom_eval",
+  "agent_preference",
+]).describe(
+  "Kind of eval the template creates. A template only applies to evals of the same type.",
+);
+
+export const EvalTemplateIncludedComponent = {
+  Judges: "judges",
+  Targets: "targets",
+  Persona: "persona",
+  Competitors: "competitors",
+  Tag: "tag",
+  Category: "category",
+  Runs: "runs",
+  Schedule: "schedule",
+  Repository: "repository",
+  Environment: "environment",
+  SearchGrounding: "search_grounding",
+  Skills: "skills",
+  McpServers: "mcp_servers",
+  Clis: "clis",
+  BrandDocs: "brand_docs",
+  GithubPr: "github_pr",
+  RuntimeCommands: "runtime_commands",
+} as const;
+export type EvalTemplateIncludedComponent = ClosedEnum<
+  typeof EvalTemplateIncludedComponent
+>;
+
+export const EvalTemplateIncludedComponent$zodSchema = z.enum([
+  "judges",
+  "targets",
+  "persona",
+  "competitors",
+  "tag",
+  "category",
+  "runs",
+  "schedule",
+  "repository",
+  "environment",
+  "search_grounding",
+  "skills",
+  "mcp_servers",
+  "clis",
+  "brand_docs",
+  "github_pr",
+  "runtime_commands",
+]);
+
+export type EvalTemplate = {
+  name: string;
+  description?: string | null | undefined;
+  eval_type?: EvalTemplateEvalType | undefined;
+  included_components: Array<EvalTemplateIncludedComponent>;
+  persona?: string | null | undefined;
+  category_name?: string | null | undefined;
+  tag?: string | null | undefined;
+  competitor_ids?: Array<string> | undefined;
+  judges?: Array<GraderDefinition> | undefined;
+  target_keys?: Array<string> | undefined;
+  schedule?: EvalTemplateSchedule | undefined;
+  runs_per_execution?: number | undefined;
+  runtime?: EvalTemplateRuntime | undefined;
+  id: string;
+  org_id: string;
+  version: number;
+  created_by?: string | null | undefined;
+  created_at: string;
+  updated_at: string;
+  deleted_at?: string | null | undefined;
+};
+
+export const EvalTemplate$zodSchema: z.ZodType<EvalTemplate> = z.object({
+  category_name: z.string().nullable().optional().describe(
+    "Optional category copied when `category` is included.",
+  ),
+  competitor_ids: z.array(z.string()).optional().describe(
+    "Competitor IDs that create additional variants when `competitors` is included.",
+  ),
+  created_at: z.iso.datetime({ offset: true }),
+  created_by: z.string().nullable().optional(),
+  deleted_at: z.iso.datetime({ offset: true }).nullable().optional(),
+  description: z.string().nullable().optional().describe(
+    "Optional guidance describing when to use the template.",
+  ),
+  eval_type: EvalTemplateEvalType$zodSchema.default("custom_eval").describe(
+    "Kind of eval the template creates. A template only applies to evals of the same type.",
+  ),
+  id: z.string(),
+  included_components: z.array(EvalTemplateIncludedComponent$zodSchema)
+    .describe(
+      "Settings copied when the template is applied. Omitted components do not override the new eval; included components copy their saved value, including an empty value.",
+    ),
+  judges: z.array(GraderDefinition$zodSchema).optional().describe(
+    "Judge definitions copied when `judges` is included. May be empty to explicitly create the eval without template judges.",
+  ),
+  name: z.string().describe(
+    "Organization-visible name for this reusable template.",
+  ),
+  org_id: z.string(),
+  persona: z.string().nullable().optional().describe(
+    "Optional persona copied when `persona` is included.",
+  ),
+  runs_per_execution: z.int().default(1).describe(
+    "Attempt count copied when `runs` is included.",
+  ),
+  runtime: EvalTemplateRuntime$zodSchema.optional().describe(
+    "Reusable runtime resources copied into each created eval.",
+  ),
+  schedule: EvalTemplateSchedule$zodSchema.optional().describe(
+    "Schedule copied into every eval created from the template.",
+  ),
+  tag: z.string().nullable().optional().describe(
+    "Optional tag copied when `tag` is included.",
+  ),
+  target_keys: z.array(z.string()).optional().describe(
+    "Runnable target keys copied when `targets` is included, such as `agent:codex:gpt-5.6-terra`.",
+  ),
+  updated_at: z.iso.datetime({ offset: true }),
+  version: z.int(),
+});

package/src/models/evaltemplatebatchcreatedeval.ts

@@ -0,0 +1,26 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 98054871a9ae
+ */
+
+import * as z from "zod";
+
+export type EvalTemplateBatchCreatedEval = {
+  id: string;
+  eval_id?: string | null | undefined;
+  name: string;
+  prompt: string;
+  competitor_id?: string | null | undefined;
+  competitor_name?: string | null | undefined;
+};
+
+export const EvalTemplateBatchCreatedEval$zodSchema: z.ZodType<
+  EvalTemplateBatchCreatedEval
+> = z.object({
+  competitor_id: z.string().nullable().optional(),
+  competitor_name: z.string().nullable().optional(),
+  eval_id: z.string().nullable().optional(),
+  id: z.string(),
+  name: z.string(),
+  prompt: z.string(),
+});

package/src/models/evaltemplatebatchitem.ts

@@ -0,0 +1,54 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 23c0fa6bda51
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+/**
+ * Eval subject type used for organization and reporting.
+ */
+export const EvalTemplateBatchItemType = {
+  Workflow: "workflow",
+  Mcp: "mcp",
+  Cli: "cli",
+  Skill: "skill",
+} as const;
+/**
+ * Eval subject type used for organization and reporting.
+ */
+export type EvalTemplateBatchItemType = ClosedEnum<
+  typeof EvalTemplateBatchItemType
+>;
+
+export const EvalTemplateBatchItemType$zodSchema = z.enum([
+  "workflow",
+  "mcp",
+  "cli",
+  "skill",
+]).describe("Eval subject type used for organization and reporting.");
+
+/**
+ * Prompt-specific fields combined with the referenced template.
+ */
+export type EvalTemplateBatchItem = {
+  name: string;
+  prompts: Array<string>;
+  type?: EvalTemplateBatchItemType | undefined;
+  subject_id?: string | null | undefined;
+};
+
+export const EvalTemplateBatchItem$zodSchema: z.ZodType<EvalTemplateBatchItem> =
+  z.object({
+    name: z.string().describe("Name of the prompt-specific eval to create."),
+    prompts: z.array(z.string()).describe(
+      "Task prompts owned by this eval. Judges, targets, scheduling, and runtime resources come from the template.",
+    ),
+    subject_id: z.string().nullable().optional().describe(
+      "Optional subject identifier associated with this eval.",
+    ),
+    type: EvalTemplateBatchItemType$zodSchema.default("workflow").describe(
+      "Eval subject type used for organization and reporting.",
+    ),
+  }).describe("Prompt-specific fields combined with the referenced template.");

package/src/models/evaltemplatebatchresponse.ts

@@ -0,0 +1,30 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: a63a246560d8
+ */
+
+import * as z from "zod";
+import {
+  EvalTemplateBatchResult,
+  EvalTemplateBatchResult$zodSchema,
+} from "./evaltemplatebatchresult.js";
+
+export type EvalTemplateBatchResponse = {
+  template_id: string;
+  template_version: number;
+  created_count: number;
+  base_count: number;
+  variant_count: number;
+  results?: Array<EvalTemplateBatchResult> | undefined;
+};
+
+export const EvalTemplateBatchResponse$zodSchema: z.ZodType<
+  EvalTemplateBatchResponse
+> = z.object({
+  base_count: z.int(),
+  created_count: z.int(),
+  results: z.array(EvalTemplateBatchResult$zodSchema).optional(),
+  template_id: z.string(),
+  template_version: z.int(),
+  variant_count: z.int(),
+});

package/src/models/evaltemplatebatchresult.ts

@@ -0,0 +1,26 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: f2e21f14a77e
+ */
+
+import * as z from "zod";
+import {
+  EvalTemplateBatchCreatedEval,
+  EvalTemplateBatchCreatedEval$zodSchema,
+} from "./evaltemplatebatchcreatedeval.js";
+
+export type EvalTemplateBatchResult = {
+  input_index: number;
+  name: string;
+  base: EvalTemplateBatchCreatedEval;
+  variants?: Array<EvalTemplateBatchCreatedEval> | undefined;
+};
+
+export const EvalTemplateBatchResult$zodSchema: z.ZodType<
+  EvalTemplateBatchResult
+> = z.object({
+  base: EvalTemplateBatchCreatedEval$zodSchema,
+  input_index: z.int(),
+  name: z.string(),
+  variants: z.array(EvalTemplateBatchCreatedEval$zodSchema).optional(),
+});

package/src/models/evaltemplateruntime.ts

@@ -0,0 +1,64 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: c5af571c2713
+ */
+
+import * as z from "zod";
+import {
+  SearchGroundingMode,
+  SearchGroundingMode$zodSchema,
+} from "./searchgroundingmode.js";
+
+/**
+ * Reusable runtime resources copied into each created eval.
+ */
+export type EvalTemplateRuntime = {
+  starting_project_id?: string | null | undefined;
+  environment_profile_ids?: Array<string> | undefined;
+  include_env_vars?: boolean | undefined;
+  search_grounding?: SearchGroundingMode | undefined;
+  skill_ids?: Array<string> | undefined;
+  mcp_server_ids?: Array<string> | undefined;
+  cli_ids?: Array<string> | undefined;
+  include_brand_docs?: boolean | undefined;
+  run_on_github_pr?: boolean | undefined;
+  setup_commands?: Array<string> | undefined;
+  cleanup_commands?: Array<string> | undefined;
+};
+
+export const EvalTemplateRuntime$zodSchema: z.ZodType<EvalTemplateRuntime> = z
+  .object({
+    cleanup_commands: z.array(z.string()).optional().describe(
+      "Shell commands run after each eval attempt, in order.",
+    ),
+    cli_ids: z.array(z.string()).optional().describe(
+      "Reusable CLI IDs copied into each created eval.",
+    ),
+    environment_profile_ids: z.array(z.string()).optional().describe(
+      "Environment profile IDs copied into each created eval.",
+    ),
+    include_brand_docs: z.boolean().default(false).describe(
+      "Whether organization documentation is included at runtime.",
+    ),
+    include_env_vars: z.boolean().default(true).describe(
+      "Whether selected environment profiles are enabled.",
+    ),
+    mcp_server_ids: z.array(z.string()).optional().describe(
+      "Reusable MCP server IDs copied into each created eval.",
+    ),
+    run_on_github_pr: z.boolean().default(false).describe(
+      "Whether each created eval is configured for GitHub pull requests.",
+    ),
+    search_grounding: SearchGroundingMode$zodSchema.optional().describe(
+      "Requested web-retrieval behavior for an eval run.",
+    ),
+    setup_commands: z.array(z.string()).optional().describe(
+      "Shell commands run before each eval attempt, in order.",
+    ),
+    skill_ids: z.array(z.string()).optional().describe(
+      "Reusable skill IDs copied into each created eval.",
+    ),
+    starting_project_id: z.string().nullable().optional().describe(
+      "Optional reusable repository workspace ID.",
+    ),
+  }).describe("Reusable runtime resources copied into each created eval.");

package/src/models/evaltemplateschedule.ts

@@ -0,0 +1,44 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: e2a92236d519
+ */
+
+import * as z from "zod";
+import { ClosedEnum } from "../types/enums.js";
+
+/**
+ * Schedule cadence used when `enabled` is true.
+ */
+export const Frequency = {
+  Daily: "daily",
+  EveryOtherDay: "every_other_day",
+  Weekly: "weekly",
+} as const;
+/**
+ * Schedule cadence used when `enabled` is true.
+ */
+export type Frequency = ClosedEnum<typeof Frequency>;
+
+export const Frequency$zodSchema = z.enum([
+  "daily",
+  "every_other_day",
+  "weekly",
+]).describe("Schedule cadence used when `enabled` is true.");
+
+/**
+ * Schedule copied into every eval created from the template.
+ */
+export type EvalTemplateSchedule = {
+  enabled?: boolean | undefined;
+  frequency?: Frequency | undefined;
+};
+
+export const EvalTemplateSchedule$zodSchema: z.ZodType<EvalTemplateSchedule> = z
+  .object({
+    enabled: z.boolean().default(false).describe(
+      "Whether each eval created from the template receives a schedule.",
+    ),
+    frequency: Frequency$zodSchema.default("weekly").describe(
+      "Schedule cadence used when `enabled` is true.",
+    ),
+  }).describe("Schedule copied into every eval created from the template.");

package/src/models/evaltemplatesresponse.ts

@@ -0,0 +1,18 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: 72a47845de15
+ */
+
+import * as z from "zod";
+import { EvalTemplate, EvalTemplate$zodSchema } from "./evaltemplate.js";
+
+export type EvalTemplatesResponse = {
+  data?: Array<EvalTemplate> | undefined;
+  count: number;
+};
+
+export const EvalTemplatesResponse$zodSchema: z.ZodType<EvalTemplatesResponse> =
+  z.object({
+    count: z.int(),
+    data: z.array(EvalTemplate$zodSchema).optional(),
+  });

package/src/models/graderdefinition.ts

@@ -0,0 +1,34 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ * @generated-id: e683eba11663
+ */
+
+import * as z from "zod";
+
+export type GraderDefinition = {
+  name: string;
+  type: string;
+  needle?: string | null | undefined;
+  needles?: Array<string> | null | undefined;
+  pattern?: string | null | undefined;
+  case_sensitive?: boolean | undefined;
+  criteria?: string | null | undefined;
+  weight?: number | undefined;
+  blocking?: boolean | undefined;
+};
+
+export const GraderDefinition$zodSchema: z.ZodType<GraderDefinition> = z.object(
+  {
+    blocking: z.boolean().default(false).describe(
+      "Whether a failed or missing result from this grader fails the entire eval regardless of the aggregate score.",
+    ),
+    case_sensitive: z.boolean().default(false),
+    criteria: z.string().nullable().optional(),
+    name: z.string(),
+    needle: z.string().nullable().optional(),
+    needles: z.array(z.string()).nullable().optional(),
+    pattern: z.string().nullable().optional(),
+    type: z.string(),
+    weight: z.number().default(1),
+  },
+);