@acruxcoreai/mcp-server 0.1.0 → 0.3.0

package/dist/index.mjs

@@ -194,10 +194,15 @@ function toApiError(label, status, body) {
 }
 
 // src/format.ts
+var TRUNCATION_MARKER = "\n\n[truncated: response exceeded the size limit \u2014 narrow the query, use a smaller `limit`, or fetch one item at a time]";
+function isTruncated(result) {
+  const first = result.content[0];
+  return first?.type === "text" && first.text.endsWith(TRUNCATION_MARKER);
+}
 function truncateText(text, maxBytes) {
   const encoder = new TextEncoder();
   if (encoder.encode(text).length <= maxBytes) return text;
-  const marker = "\n\n[truncated: response exceeded the size limit \u2014 narrow the query, use a smaller `limit`, or fetch one item at a time]";
+  const marker = TRUNCATION_MARKER;
   const budget = Math.max(0, maxBytes - encoder.encode(marker).length);
   let end = Math.min(text.length, budget);
   while (end > 0 && encoder.encode(text.slice(0, end)).length > budget) {
@@ -583,10 +588,87 @@ var catalogTools = [
 
 // src/toolsets/evaluations.ts
 import { z as z2 } from "zod";
+
+// src/optimize-report.ts
+function summarizeCandidates(grid, cells) {
+  const cellByKey = new Map(cells.map((cell) => [cell.cellKey, cell]));
+  const baselineByModel = new Map(
+    cells.filter((cell) => cell.isProductionBaseline).map((cell) => [cell.model, cell])
+  );
+  const byCandidate = /* @__PURE__ */ new Map();
+  for (const gridCell of grid) {
+    if (gridCell.variantKind !== "candidate" || !gridCell.promptCandidateId) continue;
+    const reportCell = cellByKey.get(gridCell.cellKey);
+    const baseline = baselineByModel.get(gridCell.model);
+    let summary = byCandidate.get(gridCell.promptCandidateId);
+    if (!summary) {
+      summary = {
+        candidateId: gridCell.promptCandidateId,
+        label: gridCell.variantLabel,
+        scores: [],
+        beatsBaseline: false
+      };
+      byCandidate.set(gridCell.promptCandidateId, summary);
+    }
+    summary.scores.push({
+      model: gridCell.model,
+      avgScore: reportCell?.avgScore ?? null,
+      passRate: reportCell?.passRate ?? null,
+      baselineAvgScore: baseline?.avgScore ?? null,
+      scoreDelta: reportCell?.deltaVsBaseline?.score ?? null,
+      verdict: reportCell?.deltaVsBaseline?.label ?? "unknown"
+    });
+  }
+  const summaries = [...byCandidate.values()];
+  for (const summary of summaries) {
+    summary.beatsBaseline = summary.scores.some((score) => score.verdict === "improved") && summary.scores.every((score) => score.verdict === "improved" || score.verdict === "flat");
+  }
+  return summaries.sort((a, b) => a.label.localeCompare(b.label));
+}
+function bestCandidate(candidates) {
+  let best = null;
+  let bestDelta = -Infinity;
+  for (const candidate of candidates) {
+    if (!candidate.beatsBaseline) continue;
+    const deltas = candidate.scores.map((score) => score.scoreDelta).filter((delta) => delta !== null);
+    const mean = deltas.reduce((sum, delta) => sum + delta, 0) / Math.max(1, deltas.length);
+    if (mean > bestDelta) {
+      best = candidate;
+      bestDelta = mean;
+    }
+  }
+  return best;
+}
+
+// src/toolsets/evaluations.ts
 var historyMessageSchema = z2.object({
   role: z2.enum(["system", "user", "assistant", "tool"]),
   content: z2.string()
 });
+async function loadOptimizeRun(client, runId) {
+  const run = await client.get(`/runs/${runId}`);
+  const notDraftedYet = run.grid.length === 0;
+  if (run.kind !== "optimize" && !notDraftedYet) {
+    throw new Error(
+      `Run ${runId} is an evaluation run, not an optimize run. Read it with get_experiment_run.`
+    );
+  }
+  if (run.status !== "succeeded") return { run, report: null, candidates: [] };
+  const report = await client.get(`/runs/${runId}/report`);
+  return { run, report, candidates: summarizeCandidates(run.grid, report.cells) };
+}
+function baselineOf(run) {
+  const cell = run.grid.find((row) => row.isProductionBaseline && row.promptVersionId);
+  if (!cell?.promptVersionId) {
+    throw new Error(`Run ${run.id} has no baseline version, so a promote cannot be checked.`);
+  }
+  const alias = /^v\d+$/.test(cell.variantLabel) ? null : cell.variantLabel;
+  return { versionId: cell.promptVersionId, alias };
+}
+function formatDelta(delta) {
+  if (delta === null) return "n/a";
+  return `${delta >= 0 ? "+" : ""}${delta.toFixed(1)}`;
+}
 var evaluationTools = [
   defineTool({
     name: "list_datasets",
@@ -665,21 +747,29 @@ var evaluationTools = [
     title: "Create and start an experiment run",
     toolset: "evaluations",
     readOnlyHint: false,
-    description: "Create an experiment over a dataset and immediately start a run of it. The grid is every (version x model) pair you list. When `prompt` is given, a baseline cell for its production alias is added automatically unless that version is already in the grid. Returns as soon as the run is queued \u2014 the worker executes it asynchronously, so poll get_experiment_run for results. Note the judge model (gpt-4o-mini by default) must itself be registered in the gateway or every score fails.",
+    description: "Create an experiment over a dataset and immediately start a run of it. The grid is every (version x model) pair you list. When `prompt` is given, a baseline cell for its production alias is added automatically unless that version is already in the grid. Returns as soon as the run is queued \u2014 the worker executes it asynchronously, so poll get_experiment_run for results. `judge_model` scores every answer and must be registered in the gateway (see list_models). `judge_prompt` grades with one of your prompts instead of the built-in judge.",
     inputSchema: {
       dataset_id: z2.string().uuid().describe("Dataset to evaluate against."),
       version_ids: z2.array(z2.string().uuid()).min(1).describe("Prompt-version UUIDs to sweep. At least one."),
       models: z2.array(z2.string().min(1)).min(1).describe("Gateway model public names to sweep."),
       prompt: z2.string().optional().describe("Prompt name or UUID under test \u2014 enables the automatic baseline cell."),
       name: z2.string().optional().describe("Experiment name, for the run history."),
-      alias: z2.string().optional().describe('Which alias the automatic baseline points at. Default "production".')
+      alias: z2.string().optional().describe('Which alias the automatic baseline points at. Default "production".'),
+      judge_model: z2.string().min(1).describe("Gateway model that scores every answer. Required."),
+      judge_prompt: z2.string().min(1).optional().describe(
+        "Name or UUID of a team prompt that grades instead of the built-in judge. The run uses its production version (else its latest). Default: the built-in judge."
+      )
     },
     async handler(args, { client, config }) {
       const body = {
         dataset_id: args.dataset_id,
         version_ids: args.version_ids,
-        models: args.models
+        models: args.models,
+        judge_model: args.judge_model
       };
+      if (args.judge_prompt !== void 0) {
+        body.judge_prompt_id = await resolvePromptRef(client, args.judge_prompt);
+      }
       if (args.name !== void 0) body.name = args.name;
       if (args.alias !== void 0) body.alias = args.alias;
       if (args.prompt !== void 0) body.prompt_id = await resolvePromptRef(client, args.prompt);
@@ -721,23 +811,214 @@ var evaluationTools = [
     title: "Get a run, optionally with its comparison report",
     toolset: "evaluations",
     readOnlyHint: true,
-    description: "Read one run's status and results. With include_report=true it also fetches the comparison report: the full (variant x model) matrix with per-cell averages, each cell's delta against the same-model production baseline, a leaderboard and an advisory winner. Ask for the report only once the run has succeeded \u2014 an in-flight run has nothing to compare.",
+    description: "Read one run's status and results. With include_report=true it also fetches the comparison report: the full (variant x model) matrix with per-cell averages, each cell's delta against the same-model production baseline, a leaderboard and an advisory winner. Ask for the report only once the run has succeeded \u2014 an in-flight run has nothing to compare. For an optimize run, get_optimize_report gives the same scores per candidate, with the ids promote_optimize_candidate needs.",
     inputSchema: {
       run_id: z2.string().uuid().describe("Run UUID."),
       include_report: z2.boolean().optional().describe("Also fetch the comparison report.")
     },
     async handler({ run_id, include_report }, { client, config }) {
-      const run = await client.get(`/runs/${run_id}`);
+      const run = await client.get(
+        `/runs/${run_id}`
+      );
       let report = void 0;
       if (include_report) {
         report = await client.get(`/runs/${run_id}/report`);
       }
       return okResult(
-        `Run ${run_id} is ${run.status} with ${run.results?.length ?? 0} result(s).`,
+        `Run ${run_id} is ${run.status} with ${run.results?.total ?? 0} result(s).`,
         { run, report },
         config.maxTextBytes
       );
     }
+  }),
+  defineTool({
+    name: "optimize_prompt",
+    title: "Start an optimizer run for a prompt",
+    toolset: "evaluations",
+    readOnlyHint: false,
+    description: 'Ask the optimizer to improve a prompt against a dataset. An optimizer model drafts `draft_count` rewrites of the version behind `alias` (default "production"), then every rewrite and that baseline run over the dataset on each of `models`, and `judge_model` scores every answer. Returns the run id as soon as the run is queued \u2014 drafting and scoring take minutes, so poll get_optimize_report. Nothing is promoted automatically: a rewrite only ships when promote_optimize_candidate is called. The dataset\'s examples need `criteria`, or the built-in judge has nothing to grade against. `judge_prompt` grades with one of your prompts instead, and the optimizer reads it too, so the rewrites aim at that rubric. Every model named here must be registered in the gateway (see list_models).',
+    inputSchema: {
+      prompt: z2.string().min(1).describe("Prompt name or UUID to improve."),
+      dataset_id: z2.string().uuid().describe("Dataset whose examples the rewrites are scored on."),
+      models: z2.array(z2.string().min(1)).min(1).describe("Gateway model public names to test each rewrite and the baseline on."),
+      judge_model: z2.string().min(1).describe("Gateway model that scores every answer. Required."),
+      optimizer_model: z2.string().min(1).optional().describe("Gateway model that writes the rewrites. Default: the first entry of `models`."),
+      draft_count: z2.number().int().min(1).max(6).optional().describe("How many rewrites to draft, 1 to 6. Default 3."),
+      alias: z2.string().min(1).optional().describe('Alias whose version is the baseline to beat. Default "production".'),
+      optimizer_prompt: z2.string().optional().describe("Name or UUID of a team prompt to use as the optimizer's instructions."),
+      judge_prompt: z2.string().min(1).optional().describe(
+        "Name or UUID of a team prompt that grades instead of the built-in judge. The run uses its production version (else its latest). Default: the built-in judge."
+      )
+    },
+    async handler(args, { client, config }) {
+      const promptId = await resolvePromptRef(client, args.prompt);
+      const body = {
+        dataset_id: args.dataset_id,
+        models: args.models,
+        judge_model: args.judge_model
+      };
+      if (args.optimizer_model !== void 0) body.optimizer_model = args.optimizer_model;
+      if (args.draft_count !== void 0) body.draft_count = args.draft_count;
+      if (args.alias !== void 0) body.alias = args.alias;
+      if (args.optimizer_prompt !== void 0) {
+        body.optimizer_prompt_id = await resolvePromptRef(client, args.optimizer_prompt);
+      }
+      if (args.judge_prompt !== void 0) {
+        body.judge_prompt_id = await resolvePromptRef(client, args.judge_prompt);
+      }
+      const result = await client.post(`/prompts/${promptId}/optimize`, body);
+      const warning = result.prompt_mismatch_warning ? " Warning: some dataset examples were built from a different prompt \u2014 see prompt_mismatch_warning." : "";
+      return okResult(
+        `Optimize run ${result.run_id} is ${result.status}; ${result.optimizer_model} drafts the rewrites, then they are scored against the ${args.alias ?? "production"} baseline. Poll get_optimize_report with this run id.${warning}`,
+        result,
+        config.maxTextBytes
+      );
+    }
+  }),
+  defineTool({
+    name: "get_optimize_report",
+    title: "Read an optimizer run's candidates and scores",
+    toolset: "evaluations",
+    readOnlyHint: true,
+    description: "Read an optimize run started by optimize_prompt. While it is queued or running this returns only its status \u2014 call again later. Once it has succeeded it lists every drafted rewrite with its candidate id, the optimizer's rationale, the rewritten messages, and its average score next to the baseline's on each model. `beatsBaseline` is true only for a rewrite that improved on at least one model and was improved or flat on every other; `recommended` is the one among those with the largest gain, or null when no rewrite beat the baseline. When the rewrites are too long to fit in one result, their messages are left out (`messagesOmitted`); pass `candidate_id` to read one rewrite in full.",
+    inputSchema: {
+      run_id: z2.string().uuid().describe("Optimize run UUID, from optimize_prompt."),
+      candidate_id: z2.string().uuid().optional().describe("Return only this candidate, with its full messages.")
+    },
+    async handler({ run_id, candidate_id }, { client, config }) {
+      const { run, report, candidates } = await loadOptimizeRun(client, run_id);
+      if (run.status === "queued" || run.status === "running") {
+        return okResult(
+          `Optimize run ${run_id} is ${run.status}. No scores yet \u2014 call get_optimize_report again shortly.`,
+          { runId: run_id, status: run.status },
+          config.maxTextBytes
+        );
+      }
+      if (run.status === "failed") {
+        return okResult(
+          `Optimize run ${run_id} failed: ${run.error ?? "no reason recorded"}. Nothing to promote.`,
+          { runId: run_id, status: run.status, error: run.error },
+          config.maxTextBytes
+        );
+      }
+      if (candidate_id !== void 0) {
+        const candidate = candidates.find((row) => row.candidateId === candidate_id);
+        if (!candidate) {
+          throw new Error(`Candidate ${candidate_id} is not a rewrite from run ${run_id}.`);
+        }
+        const detail = await client.get(
+          `/runs/${run_id}/candidates/${candidate_id}`
+        );
+        return okResult(
+          `${candidate.label} (candidate_id ${candidate_id}) from optimize run ${run_id}; beatsBaseline: ${candidate.beatsBaseline}.`,
+          {
+            runId: run_id,
+            candidate: { ...candidate, rationale: detail.rationale, messages: detail.messages }
+          },
+          config.maxTextBytes
+        );
+      }
+      const reads = await Promise.allSettled(
+        candidates.map(
+          (candidate) => client.get(`/runs/${run_id}/candidates/${candidate.candidateId}`)
+        )
+      );
+      const withContent = candidates.map((candidate, index) => {
+        const read = reads[index];
+        return read.status === "fulfilled" ? { ...candidate, rationale: read.value.rationale, messages: read.value.messages } : {
+          ...candidate,
+          rationale: null,
+          messages: null,
+          readError: read.reason instanceof Error ? read.reason.message : String(read.reason)
+        };
+      });
+      const best = bestCandidate(candidates);
+      const winners = candidates.filter((candidate) => candidate.beatsBaseline).length;
+      const rejected = report?.optimizerRejections ?? [];
+      const rejectedNote = rejected.length > 0 ? ` ${rejected.length} more rewrite(s) were dropped before scoring: ${rejected.join(" | ")}.` : "";
+      const summary = (best ? `Optimize run ${run_id} succeeded. ${winners} of ${candidates.length} rewrite(s) beat the baseline. Recommended: ${best.label} (candidate_id ${best.candidateId}), ` + best.scores.map((score) => `${formatDelta(score.scoreDelta)} on ${score.model}`).join(", ") + ". Promote it with promote_optimize_candidate only if the user agrees." : `Optimize run ${run_id} succeeded, but none of its ${candidates.length} rewrite(s) beat the baseline. Nothing is worth promoting; keep the current version.`) + rejectedNote;
+      const payload = {
+        runId: run_id,
+        status: run.status,
+        judgeModel: report?.judge?.model ?? null,
+        // Null means the built-in judge graded the run.
+        judgePromptVersionId: report?.judge?.promptVersionId ?? null,
+        recommended: best?.candidateId ?? null,
+        rejectedRewrites: rejected,
+        baseline: (report?.cells ?? []).filter((cell) => cell.isProductionBaseline).map((cell) => ({
+          label: cell.variantLabel,
+          model: cell.model,
+          avgScore: cell.avgScore,
+          passRate: cell.passRate
+        })),
+        candidates: withContent
+      };
+      const full = okResult(summary, payload, config.maxTextBytes);
+      if (!isTruncated(full)) return full;
+      return okResult(
+        `${summary} The rewrites are too long to show together, so their messages are left out; call get_optimize_report again with candidate_id to read one.`,
+        {
+          ...payload,
+          messagesOmitted: true,
+          candidates: withContent.map(({ messages: _messages, ...rest }) => rest)
+        },
+        config.maxTextBytes
+      );
+    }
+  }),
+  defineTool({
+    name: "promote_optimize_candidate",
+    title: "Ship an optimizer rewrite as a new prompt version",
+    toolset: "evaluations",
+    readOnlyHint: false,
+    destructiveHint: true,
+    description: "Commit one rewrite from a succeeded optimize run as a new prompt version and move `alias` to it, so live traffic on that alias starts using it. `alias` defaults to the alias the run was scored against (production when the run used the latest version). Take `candidate_id` from get_optimize_report. Refuses a rewrite that did not beat the baseline, and refuses when the alias now points at a version other than the one the run scored against; set `promote_without_improvement` only when the user has explicitly asked to ship that rewrite anyway. Undo by moving the alias back with promote_prompt_alias.",
+    inputSchema: {
+      run_id: z2.string().uuid().describe("Optimize run UUID."),
+      candidate_id: z2.string().uuid().describe("Candidate id from get_optimize_report."),
+      alias: z2.string().min(1).optional().describe("Alias to move. Default: the alias the run was scored against."),
+      promote_without_improvement: z2.boolean().optional().describe(
+        "Ship a rewrite that did not beat the baseline, or replace a version the run never scored. Only on the user's explicit say-so."
+      )
+    },
+    async handler(args, { client, config }) {
+      const { run, candidates } = await loadOptimizeRun(client, args.run_id);
+      if (run.status !== "succeeded") {
+        throw new Error(
+          `Optimize run ${args.run_id} is ${run.status}, so its rewrites are not fully scored. Promote only from a succeeded run.`
+        );
+      }
+      const candidate = candidates.find((row) => row.candidateId === args.candidate_id);
+      if (!candidate) {
+        throw new Error(
+          `Candidate ${args.candidate_id} is not a rewrite from run ${args.run_id}. Take the id from get_optimize_report.`
+        );
+      }
+      if (!candidate.beatsBaseline && !args.promote_without_improvement) {
+        const scores = candidate.scores.map((score) => `${formatDelta(score.scoreDelta)} (${score.verdict}) on ${score.model}`).join(", ");
+        throw new Error(
+          `Refused: ${candidate.label} did not beat the baseline (${scores}). Promoting it would ship a prompt that scored no better than the current one. Set promote_without_improvement to true only if the user explicitly asked for this rewrite.`
+        );
+      }
+      const baseline = baselineOf(run);
+      const alias = args.alias ?? baseline.alias ?? "production";
+      const detail = await client.get(
+        `/runs/${args.run_id}/candidates/${args.candidate_id}`
+      );
+      const aliases = await client.get(`/prompts/${detail.promptId}/aliases`);
+      const live = aliases.find((row) => row.alias === alias);
+      if (live && live.versionId !== baseline.versionId && !args.promote_without_improvement) {
+        throw new Error(
+          `Refused: alias "${alias}" now points at version ${live.versionNumber}, but run ${args.run_id} scored its rewrites against a different version. Nothing compared ${candidate.label} with version ${live.versionNumber}. Start a new optimize run against this alias, or set promote_without_improvement to true only if the user explicitly asked to replace it anyway.`
+        );
+      }
+      const result = await client.post(`/runs/${args.run_id}/promote`, { prompt_candidate_id: args.candidate_id, alias });
+      return okResult(
+        `Promoted ${candidate.label} as version ${result.version.versionNumber}; alias "${result.alias.alias}" now points at it.`,
+        result,
+        config.maxTextBytes
+      );
+    }
   })
 ];
 
@@ -764,6 +1045,9 @@ function gatewayMeta(headers) {
   }
   return meta;
 }
+function traceSuffix(meta) {
+  return meta["trace-id"] ? ` Open it with get_trace, trace_id ${meta["trace-id"]}.` : "";
+}
 function firstChoiceText(body) {
   const choice = body?.choices?.[0];
   const content = choice?.message?.content;
@@ -808,9 +1092,7 @@ var gatewayTools = [
       });
       const meta = gatewayMeta(response.headers);
       return okResult(
-        `Ran "${name}"@${args.alias ?? "production"} on ${meta.model ?? "the resolved model"}${meta["cost-usd"] ? ` for $${meta["cost-usd"]}` : ""}. Reply: ${firstChoiceText(
-          response.data
-        ).slice(0, 400)}`,
+        `Ran "${name}"@${args.alias ?? "production"} on ${meta.model ?? "the resolved model"}${meta["cost-usd"] ? ` for $${meta["cost-usd"]}` : ""}.${traceSuffix(meta)} Reply: ` + firstChoiceText(response.data).slice(0, 400),
         { completion: response.data, gateway: meta },
         config.maxTextBytes
       );
@@ -851,9 +1133,7 @@ var gatewayTools = [
       });
       const meta = gatewayMeta(response.headers);
       return okResult(
-        `Completion from ${meta.model ?? args.model}${meta["cost-usd"] ? ` ($${meta["cost-usd"]})` : ""}: ${firstChoiceText(
-          response.data
-        ).slice(0, 400)}`,
+        `Completion from ${meta.model ?? args.model}${meta["cost-usd"] ? ` ($${meta["cost-usd"]})` : ""}.${traceSuffix(meta)} Reply: ` + firstChoiceText(response.data).slice(0, 400),
         { completion: response.data, gateway: meta },
         config.maxTextBytes
       );
@@ -893,6 +1173,37 @@ var gatewayTools = [
         config.maxTextBytes
       );
     }
+  }),
+  defineTool({
+    name: "get_spending_limits",
+    title: "Get budgets and virtual-key limits",
+    toolset: "gateway",
+    readOnlyHint: true,
+    description: "Every spend budget on the team \u2014 team-wide or on one virtual key \u2014 with its period, limit, live spend, what is left and when it resets, plus each virtual key's rate limits and model/provider allow-lists. Use this after a gateway call is refused with HTTP 402 (budget spent) or 429 (rate limit) to see which limit caused it. Read-only: budgets and keys are changed in the dashboard.",
+    inputSchema: {},
+    async handler(_args, { client, config }) {
+      const [budgets, keys] = await Promise.all([
+        client.get("/gateway/budgets"),
+        client.get("/gateway/keys")
+      ]);
+      const keyNames = new Map(keys.map((k) => [k.id, k.name]));
+      const limits = budgets.map((b) => ({
+        id: b.id,
+        scope: b.virtualKeyId === null ? "team" : "virtual_key",
+        virtualKeyId: b.virtualKeyId,
+        virtualKeyName: b.virtualKeyId === null ? null : keyNames.get(b.virtualKeyId) ?? null,
+        period: b.period,
+        limitUsd: b.limitUsd,
+        spendUsd: b.spendUsd,
+        remainingUsd: Math.max(0, b.limitUsd - b.spendUsd),
+        exhausted: b.spendUsd >= b.limitUsd,
+        resetsAt: b.resetsAt
+      }));
+      const summary = limits.length === 0 ? `No budgets on this team, so the gateway never refuses a call for spend. ${keys.length} virtual key(s).` : `${limits.length} budget(s), ${keys.length} virtual key(s). ` + limits.map(
+        (l) => `${l.scope === "team" ? "Team" : `Key "${l.virtualKeyName ?? l.virtualKeyId}"`} ${l.period}: $${l.spendUsd} of $${l.limitUsd} spent` + (l.exhausted ? " (EXHAUSTED)" : "") + (l.resetsAt ? `, resets ${l.resetsAt}` : "")
+      ).join("; ") + ".";
+      return okResult(summary, { budgets: limits, virtualKeys: keys }, config.maxTextBytes);
+    }
   })
 ];
 
@@ -1258,7 +1569,7 @@ function selectTools(config) {
 }
 
 // src/server.ts
-var SERVER_VERSION = "0.1.0";
+var SERVER_VERSION = "0.3.0";
 function buildInstructions(toolNames, readOnly) {
   const parts = [
     `AcruxCore is an LLM-ops platform: a prompt registry, a versioned tool catalog, an AI gateway, tracing, and evaluation.