@acruxcoreai/mcp-server 0.2.0 → 0.3.0

package/dist/index.mjs

@@ -747,21 +747,29 @@ var evaluationTools = [
     title: "Create and start an experiment run",
     toolset: "evaluations",
     readOnlyHint: false,
-    description: "Create an experiment over a dataset and immediately start a run of it. The grid is every (version x model) pair you list. When `prompt` is given, a baseline cell for its production alias is added automatically unless that version is already in the grid. Returns as soon as the run is queued \u2014 the worker executes it asynchronously, so poll get_experiment_run for results. Note the judge model (gpt-4o-mini by default) must itself be registered in the gateway or every score fails.",
+    description: "Create an experiment over a dataset and immediately start a run of it. The grid is every (version x model) pair you list. When `prompt` is given, a baseline cell for its production alias is added automatically unless that version is already in the grid. Returns as soon as the run is queued \u2014 the worker executes it asynchronously, so poll get_experiment_run for results. `judge_model` scores every answer and must be registered in the gateway (see list_models). `judge_prompt` grades with one of your prompts instead of the built-in judge.",
     inputSchema: {
       dataset_id: z2.string().uuid().describe("Dataset to evaluate against."),
       version_ids: z2.array(z2.string().uuid()).min(1).describe("Prompt-version UUIDs to sweep. At least one."),
       models: z2.array(z2.string().min(1)).min(1).describe("Gateway model public names to sweep."),
       prompt: z2.string().optional().describe("Prompt name or UUID under test \u2014 enables the automatic baseline cell."),
       name: z2.string().optional().describe("Experiment name, for the run history."),
-      alias: z2.string().optional().describe('Which alias the automatic baseline points at. Default "production".')
+      alias: z2.string().optional().describe('Which alias the automatic baseline points at. Default "production".'),
+      judge_model: z2.string().min(1).describe("Gateway model that scores every answer. Required."),
+      judge_prompt: z2.string().min(1).optional().describe(
+        "Name or UUID of a team prompt that grades instead of the built-in judge. The run uses its production version (else its latest). Default: the built-in judge."
+      )
     },
     async handler(args, { client, config }) {
       const body = {
         dataset_id: args.dataset_id,
         version_ids: args.version_ids,
-        models: args.models
+        models: args.models,
+        judge_model: args.judge_model
       };
+      if (args.judge_prompt !== void 0) {
+        body.judge_prompt_id = await resolvePromptRef(client, args.judge_prompt);
+      }
       if (args.name !== void 0) body.name = args.name;
       if (args.alias !== void 0) body.alias = args.alias;
       if (args.prompt !== void 0) body.prompt_id = await resolvePromptRef(client, args.prompt);
@@ -828,7 +836,7 @@ var evaluationTools = [
     title: "Start an optimizer run for a prompt",
     toolset: "evaluations",
     readOnlyHint: false,
-    description: 'Ask the optimizer to improve a prompt against a dataset. An optimizer model drafts up to `draft_count` rewrites of the version behind `alias` (default "production"), then every rewrite and that baseline run over the dataset on each of `models`, and `judge_model` scores every answer. Returns the run id as soon as the run is queued \u2014 drafting and scoring take minutes, so poll get_optimize_report. Nothing is promoted automatically: a rewrite only ships when promote_optimize_candidate is called. The dataset\'s examples need `criteria`, or the judge has nothing to grade against. Every model named here must be registered in the gateway (see list_models).',
+    description: 'Ask the optimizer to improve a prompt against a dataset. An optimizer model drafts `draft_count` rewrites of the version behind `alias` (default "production"), then every rewrite and that baseline run over the dataset on each of `models`, and `judge_model` scores every answer. Returns the run id as soon as the run is queued \u2014 drafting and scoring take minutes, so poll get_optimize_report. Nothing is promoted automatically: a rewrite only ships when promote_optimize_candidate is called. The dataset\'s examples need `criteria`, or the built-in judge has nothing to grade against. `judge_prompt` grades with one of your prompts instead, and the optimizer reads it too, so the rewrites aim at that rubric. Every model named here must be registered in the gateway (see list_models).',
     inputSchema: {
       prompt: z2.string().min(1).describe("Prompt name or UUID to improve."),
       dataset_id: z2.string().uuid().describe("Dataset whose examples the rewrites are scored on."),
@@ -837,7 +845,10 @@ var evaluationTools = [
       optimizer_model: z2.string().min(1).optional().describe("Gateway model that writes the rewrites. Default: the first entry of `models`."),
       draft_count: z2.number().int().min(1).max(6).optional().describe("How many rewrites to draft, 1 to 6. Default 3."),
       alias: z2.string().min(1).optional().describe('Alias whose version is the baseline to beat. Default "production".'),
-      optimizer_prompt: z2.string().optional().describe("Name or UUID of a team prompt to use as the optimizer's instructions.")
+      optimizer_prompt: z2.string().optional().describe("Name or UUID of a team prompt to use as the optimizer's instructions."),
+      judge_prompt: z2.string().min(1).optional().describe(
+        "Name or UUID of a team prompt that grades instead of the built-in judge. The run uses its production version (else its latest). Default: the built-in judge."
+      )
     },
     async handler(args, { client, config }) {
       const promptId = await resolvePromptRef(client, args.prompt);
@@ -852,6 +863,9 @@ var evaluationTools = [
       if (args.optimizer_prompt !== void 0) {
         body.optimizer_prompt_id = await resolvePromptRef(client, args.optimizer_prompt);
       }
+      if (args.judge_prompt !== void 0) {
+        body.judge_prompt_id = await resolvePromptRef(client, args.judge_prompt);
+      }
       const result = await client.post(`/prompts/${promptId}/optimize`, body);
       const warning = result.prompt_mismatch_warning ? " Warning: some dataset examples were built from a different prompt \u2014 see prompt_mismatch_warning." : "";
       return okResult(
@@ -920,12 +934,17 @@ var evaluationTools = [
       });
       const best = bestCandidate(candidates);
       const winners = candidates.filter((candidate) => candidate.beatsBaseline).length;
-      const summary = best ? `Optimize run ${run_id} succeeded. ${winners} of ${candidates.length} rewrite(s) beat the baseline. Recommended: ${best.label} (candidate_id ${best.candidateId}), ` + best.scores.map((score) => `${formatDelta(score.scoreDelta)} on ${score.model}`).join(", ") + ". Promote it with promote_optimize_candidate only if the user agrees." : `Optimize run ${run_id} succeeded, but none of its ${candidates.length} rewrite(s) beat the baseline. Nothing is worth promoting; keep the current version.`;
+      const rejected = report?.optimizerRejections ?? [];
+      const rejectedNote = rejected.length > 0 ? ` ${rejected.length} more rewrite(s) were dropped before scoring: ${rejected.join(" | ")}.` : "";
+      const summary = (best ? `Optimize run ${run_id} succeeded. ${winners} of ${candidates.length} rewrite(s) beat the baseline. Recommended: ${best.label} (candidate_id ${best.candidateId}), ` + best.scores.map((score) => `${formatDelta(score.scoreDelta)} on ${score.model}`).join(", ") + ". Promote it with promote_optimize_candidate only if the user agrees." : `Optimize run ${run_id} succeeded, but none of its ${candidates.length} rewrite(s) beat the baseline. Nothing is worth promoting; keep the current version.`) + rejectedNote;
       const payload = {
         runId: run_id,
         status: run.status,
         judgeModel: report?.judge?.model ?? null,
+        // Null means the built-in judge graded the run.
+        judgePromptVersionId: report?.judge?.promptVersionId ?? null,
         recommended: best?.candidateId ?? null,
+        rejectedRewrites: rejected,
         baseline: (report?.cells ?? []).filter((cell) => cell.isProductionBaseline).map((cell) => ({
           label: cell.variantLabel,
           model: cell.model,
@@ -1550,7 +1569,7 @@ function selectTools(config) {
 }
 
 // src/server.ts
-var SERVER_VERSION = "0.2.0";
+var SERVER_VERSION = "0.3.0";
 function buildInstructions(toolNames, readOnly) {
   const parts = [
     `AcruxCore is an LLM-ops platform: a prompt registry, a versioned tool catalog, an AI gateway, tracing, and evaluation.