@acruxcoreai/mcp-server 0.1.0 → 0.2.0

package/dist/bin.mjs

@@ -275,10 +275,15 @@ function toApiError(label, status, body) {
 }
 
 // src/format.ts
+var TRUNCATION_MARKER = "\n\n[truncated: response exceeded the size limit \u2014 narrow the query, use a smaller `limit`, or fetch one item at a time]";
+function isTruncated(result) {
+  const first = result.content[0];
+  return first?.type === "text" && first.text.endsWith(TRUNCATION_MARKER);
+}
 function truncateText(text, maxBytes) {
   const encoder = new TextEncoder();
   if (encoder.encode(text).length <= maxBytes) return text;
-  const marker = "\n\n[truncated: response exceeded the size limit \u2014 narrow the query, use a smaller `limit`, or fetch one item at a time]";
+  const marker = TRUNCATION_MARKER;
   const budget = Math.max(0, maxBytes - encoder.encode(marker).length);
   let end = Math.min(text.length, budget);
   while (end > 0 && encoder.encode(text.slice(0, end)).length > budget) {
@@ -664,10 +669,87 @@ var catalogTools = [
 
 // src/toolsets/evaluations.ts
 import { z as z2 } from "zod";
+
+// src/optimize-report.ts
+function summarizeCandidates(grid, cells) {
+  const cellByKey = new Map(cells.map((cell) => [cell.cellKey, cell]));
+  const baselineByModel = new Map(
+    cells.filter((cell) => cell.isProductionBaseline).map((cell) => [cell.model, cell])
+  );
+  const byCandidate = /* @__PURE__ */ new Map();
+  for (const gridCell of grid) {
+    if (gridCell.variantKind !== "candidate" || !gridCell.promptCandidateId) continue;
+    const reportCell = cellByKey.get(gridCell.cellKey);
+    const baseline = baselineByModel.get(gridCell.model);
+    let summary = byCandidate.get(gridCell.promptCandidateId);
+    if (!summary) {
+      summary = {
+        candidateId: gridCell.promptCandidateId,
+        label: gridCell.variantLabel,
+        scores: [],
+        beatsBaseline: false
+      };
+      byCandidate.set(gridCell.promptCandidateId, summary);
+    }
+    summary.scores.push({
+      model: gridCell.model,
+      avgScore: reportCell?.avgScore ?? null,
+      passRate: reportCell?.passRate ?? null,
+      baselineAvgScore: baseline?.avgScore ?? null,
+      scoreDelta: reportCell?.deltaVsBaseline?.score ?? null,
+      verdict: reportCell?.deltaVsBaseline?.label ?? "unknown"
+    });
+  }
+  const summaries = [...byCandidate.values()];
+  for (const summary of summaries) {
+    summary.beatsBaseline = summary.scores.some((score) => score.verdict === "improved") && summary.scores.every((score) => score.verdict === "improved" || score.verdict === "flat");
+  }
+  return summaries.sort((a, b) => a.label.localeCompare(b.label));
+}
+function bestCandidate(candidates) {
+  let best = null;
+  let bestDelta = -Infinity;
+  for (const candidate of candidates) {
+    if (!candidate.beatsBaseline) continue;
+    const deltas = candidate.scores.map((score) => score.scoreDelta).filter((delta) => delta !== null);
+    const mean = deltas.reduce((sum, delta) => sum + delta, 0) / Math.max(1, deltas.length);
+    if (mean > bestDelta) {
+      best = candidate;
+      bestDelta = mean;
+    }
+  }
+  return best;
+}
+
+// src/toolsets/evaluations.ts
 var historyMessageSchema = z2.object({
   role: z2.enum(["system", "user", "assistant", "tool"]),
   content: z2.string()
 });
+async function loadOptimizeRun(client, runId) {
+  const run = await client.get(`/runs/${runId}`);
+  const notDraftedYet = run.grid.length === 0;
+  if (run.kind !== "optimize" && !notDraftedYet) {
+    throw new Error(
+      `Run ${runId} is an evaluation run, not an optimize run. Read it with get_experiment_run.`
+    );
+  }
+  if (run.status !== "succeeded") return { run, report: null, candidates: [] };
+  const report = await client.get(`/runs/${runId}/report`);
+  return { run, report, candidates: summarizeCandidates(run.grid, report.cells) };
+}
+function baselineOf(run) {
+  const cell = run.grid.find((row) => row.isProductionBaseline && row.promptVersionId);
+  if (!cell?.promptVersionId) {
+    throw new Error(`Run ${run.id} has no baseline version, so a promote cannot be checked.`);
+  }
+  const alias = /^v\d+$/.test(cell.variantLabel) ? null : cell.variantLabel;
+  return { versionId: cell.promptVersionId, alias };
+}
+function formatDelta(delta) {
+  if (delta === null) return "n/a";
+  return `${delta >= 0 ? "+" : ""}${delta.toFixed(1)}`;
+}
 var evaluationTools = [
   defineTool({
     name: "list_datasets",
@@ -802,23 +884,203 @@ var evaluationTools = [
     title: "Get a run, optionally with its comparison report",
     toolset: "evaluations",
     readOnlyHint: true,
-    description: "Read one run's status and results. With include_report=true it also fetches the comparison report: the full (variant x model) matrix with per-cell averages, each cell's delta against the same-model production baseline, a leaderboard and an advisory winner. Ask for the report only once the run has succeeded \u2014 an in-flight run has nothing to compare.",
+    description: "Read one run's status and results. With include_report=true it also fetches the comparison report: the full (variant x model) matrix with per-cell averages, each cell's delta against the same-model production baseline, a leaderboard and an advisory winner. Ask for the report only once the run has succeeded \u2014 an in-flight run has nothing to compare. For an optimize run, get_optimize_report gives the same scores per candidate, with the ids promote_optimize_candidate needs.",
     inputSchema: {
       run_id: z2.string().uuid().describe("Run UUID."),
       include_report: z2.boolean().optional().describe("Also fetch the comparison report.")
     },
     async handler({ run_id, include_report }, { client, config }) {
-      const run = await client.get(`/runs/${run_id}`);
+      const run = await client.get(
+        `/runs/${run_id}`
+      );
       let report = void 0;
       if (include_report) {
         report = await client.get(`/runs/${run_id}/report`);
       }
       return okResult(
-        `Run ${run_id} is ${run.status} with ${run.results?.length ?? 0} result(s).`,
+        `Run ${run_id} is ${run.status} with ${run.results?.total ?? 0} result(s).`,
         { run, report },
         config.maxTextBytes
       );
     }
+  }),
+  defineTool({
+    name: "optimize_prompt",
+    title: "Start an optimizer run for a prompt",
+    toolset: "evaluations",
+    readOnlyHint: false,
+    description: 'Ask the optimizer to improve a prompt against a dataset. An optimizer model drafts up to `draft_count` rewrites of the version behind `alias` (default "production"), then every rewrite and that baseline run over the dataset on each of `models`, and `judge_model` scores every answer. Returns the run id as soon as the run is queued \u2014 drafting and scoring take minutes, so poll get_optimize_report. Nothing is promoted automatically: a rewrite only ships when promote_optimize_candidate is called. The dataset\'s examples need `criteria`, or the judge has nothing to grade against. Every model named here must be registered in the gateway (see list_models).',
+    inputSchema: {
+      prompt: z2.string().min(1).describe("Prompt name or UUID to improve."),
+      dataset_id: z2.string().uuid().describe("Dataset whose examples the rewrites are scored on."),
+      models: z2.array(z2.string().min(1)).min(1).describe("Gateway model public names to test each rewrite and the baseline on."),
+      judge_model: z2.string().min(1).describe("Gateway model that scores every answer. Required."),
+      optimizer_model: z2.string().min(1).optional().describe("Gateway model that writes the rewrites. Default: the first entry of `models`."),
+      draft_count: z2.number().int().min(1).max(6).optional().describe("How many rewrites to draft, 1 to 6. Default 3."),
+      alias: z2.string().min(1).optional().describe('Alias whose version is the baseline to beat. Default "production".'),
+      optimizer_prompt: z2.string().optional().describe("Name or UUID of a team prompt to use as the optimizer's instructions.")
+    },
+    async handler(args, { client, config }) {
+      const promptId = await resolvePromptRef(client, args.prompt);
+      const body = {
+        dataset_id: args.dataset_id,
+        models: args.models,
+        judge_model: args.judge_model
+      };
+      if (args.optimizer_model !== void 0) body.optimizer_model = args.optimizer_model;
+      if (args.draft_count !== void 0) body.draft_count = args.draft_count;
+      if (args.alias !== void 0) body.alias = args.alias;
+      if (args.optimizer_prompt !== void 0) {
+        body.optimizer_prompt_id = await resolvePromptRef(client, args.optimizer_prompt);
+      }
+      const result = await client.post(`/prompts/${promptId}/optimize`, body);
+      const warning = result.prompt_mismatch_warning ? " Warning: some dataset examples were built from a different prompt \u2014 see prompt_mismatch_warning." : "";
+      return okResult(
+        `Optimize run ${result.run_id} is ${result.status}; ${result.optimizer_model} drafts the rewrites, then they are scored against the ${args.alias ?? "production"} baseline. Poll get_optimize_report with this run id.${warning}`,
+        result,
+        config.maxTextBytes
+      );
+    }
+  }),
+  defineTool({
+    name: "get_optimize_report",
+    title: "Read an optimizer run's candidates and scores",
+    toolset: "evaluations",
+    readOnlyHint: true,
+    description: "Read an optimize run started by optimize_prompt. While it is queued or running this returns only its status \u2014 call again later. Once it has succeeded it lists every drafted rewrite with its candidate id, the optimizer's rationale, the rewritten messages, and its average score next to the baseline's on each model. `beatsBaseline` is true only for a rewrite that improved on at least one model and was improved or flat on every other; `recommended` is the one among those with the largest gain, or null when no rewrite beat the baseline. When the rewrites are too long to fit in one result, their messages are left out (`messagesOmitted`); pass `candidate_id` to read one rewrite in full.",
+    inputSchema: {
+      run_id: z2.string().uuid().describe("Optimize run UUID, from optimize_prompt."),
+      candidate_id: z2.string().uuid().optional().describe("Return only this candidate, with its full messages.")
+    },
+    async handler({ run_id, candidate_id }, { client, config }) {
+      const { run, report, candidates } = await loadOptimizeRun(client, run_id);
+      if (run.status === "queued" || run.status === "running") {
+        return okResult(
+          `Optimize run ${run_id} is ${run.status}. No scores yet \u2014 call get_optimize_report again shortly.`,
+          { runId: run_id, status: run.status },
+          config.maxTextBytes
+        );
+      }
+      if (run.status === "failed") {
+        return okResult(
+          `Optimize run ${run_id} failed: ${run.error ?? "no reason recorded"}. Nothing to promote.`,
+          { runId: run_id, status: run.status, error: run.error },
+          config.maxTextBytes
+        );
+      }
+      if (candidate_id !== void 0) {
+        const candidate = candidates.find((row) => row.candidateId === candidate_id);
+        if (!candidate) {
+          throw new Error(`Candidate ${candidate_id} is not a rewrite from run ${run_id}.`);
+        }
+        const detail = await client.get(
+          `/runs/${run_id}/candidates/${candidate_id}`
+        );
+        return okResult(
+          `${candidate.label} (candidate_id ${candidate_id}) from optimize run ${run_id}; beatsBaseline: ${candidate.beatsBaseline}.`,
+          {
+            runId: run_id,
+            candidate: { ...candidate, rationale: detail.rationale, messages: detail.messages }
+          },
+          config.maxTextBytes
+        );
+      }
+      const reads = await Promise.allSettled(
+        candidates.map(
+          (candidate) => client.get(`/runs/${run_id}/candidates/${candidate.candidateId}`)
+        )
+      );
+      const withContent = candidates.map((candidate, index) => {
+        const read = reads[index];
+        return read.status === "fulfilled" ? { ...candidate, rationale: read.value.rationale, messages: read.value.messages } : {
+          ...candidate,
+          rationale: null,
+          messages: null,
+          readError: read.reason instanceof Error ? read.reason.message : String(read.reason)
+        };
+      });
+      const best = bestCandidate(candidates);
+      const winners = candidates.filter((candidate) => candidate.beatsBaseline).length;
+      const summary = best ? `Optimize run ${run_id} succeeded. ${winners} of ${candidates.length} rewrite(s) beat the baseline. Recommended: ${best.label} (candidate_id ${best.candidateId}), ` + best.scores.map((score) => `${formatDelta(score.scoreDelta)} on ${score.model}`).join(", ") + ". Promote it with promote_optimize_candidate only if the user agrees." : `Optimize run ${run_id} succeeded, but none of its ${candidates.length} rewrite(s) beat the baseline. Nothing is worth promoting; keep the current version.`;
+      const payload = {
+        runId: run_id,
+        status: run.status,
+        judgeModel: report?.judge?.model ?? null,
+        recommended: best?.candidateId ?? null,
+        baseline: (report?.cells ?? []).filter((cell) => cell.isProductionBaseline).map((cell) => ({
+          label: cell.variantLabel,
+          model: cell.model,
+          avgScore: cell.avgScore,
+          passRate: cell.passRate
+        })),
+        candidates: withContent
+      };
+      const full = okResult(summary, payload, config.maxTextBytes);
+      if (!isTruncated(full)) return full;
+      return okResult(
+        `${summary} The rewrites are too long to show together, so their messages are left out; call get_optimize_report again with candidate_id to read one.`,
+        {
+          ...payload,
+          messagesOmitted: true,
+          candidates: withContent.map(({ messages: _messages, ...rest }) => rest)
+        },
+        config.maxTextBytes
+      );
+    }
+  }),
+  defineTool({
+    name: "promote_optimize_candidate",
+    title: "Ship an optimizer rewrite as a new prompt version",
+    toolset: "evaluations",
+    readOnlyHint: false,
+    destructiveHint: true,
+    description: "Commit one rewrite from a succeeded optimize run as a new prompt version and move `alias` to it, so live traffic on that alias starts using it. `alias` defaults to the alias the run was scored against (production when the run used the latest version). Take `candidate_id` from get_optimize_report. Refuses a rewrite that did not beat the baseline, and refuses when the alias now points at a version other than the one the run scored against; set `promote_without_improvement` only when the user has explicitly asked to ship that rewrite anyway. Undo by moving the alias back with promote_prompt_alias.",
+    inputSchema: {
+      run_id: z2.string().uuid().describe("Optimize run UUID."),
+      candidate_id: z2.string().uuid().describe("Candidate id from get_optimize_report."),
+      alias: z2.string().min(1).optional().describe("Alias to move. Default: the alias the run was scored against."),
+      promote_without_improvement: z2.boolean().optional().describe(
+        "Ship a rewrite that did not beat the baseline, or replace a version the run never scored. Only on the user's explicit say-so."
+      )
+    },
+    async handler(args, { client, config }) {
+      const { run, candidates } = await loadOptimizeRun(client, args.run_id);
+      if (run.status !== "succeeded") {
+        throw new Error(
+          `Optimize run ${args.run_id} is ${run.status}, so its rewrites are not fully scored. Promote only from a succeeded run.`
+        );
+      }
+      const candidate = candidates.find((row) => row.candidateId === args.candidate_id);
+      if (!candidate) {
+        throw new Error(
+          `Candidate ${args.candidate_id} is not a rewrite from run ${args.run_id}. Take the id from get_optimize_report.`
+        );
+      }
+      if (!candidate.beatsBaseline && !args.promote_without_improvement) {
+        const scores = candidate.scores.map((score) => `${formatDelta(score.scoreDelta)} (${score.verdict}) on ${score.model}`).join(", ");
+        throw new Error(
+          `Refused: ${candidate.label} did not beat the baseline (${scores}). Promoting it would ship a prompt that scored no better than the current one. Set promote_without_improvement to true only if the user explicitly asked for this rewrite.`
+        );
+      }
+      const baseline = baselineOf(run);
+      const alias = args.alias ?? baseline.alias ?? "production";
+      const detail = await client.get(
+        `/runs/${args.run_id}/candidates/${args.candidate_id}`
+      );
+      const aliases = await client.get(`/prompts/${detail.promptId}/aliases`);
+      const live = aliases.find((row) => row.alias === alias);
+      if (live && live.versionId !== baseline.versionId && !args.promote_without_improvement) {
+        throw new Error(
+          `Refused: alias "${alias}" now points at version ${live.versionNumber}, but run ${args.run_id} scored its rewrites against a different version. Nothing compared ${candidate.label} with version ${live.versionNumber}. Start a new optimize run against this alias, or set promote_without_improvement to true only if the user explicitly asked to replace it anyway.`
+        );
+      }
+      const result = await client.post(`/runs/${args.run_id}/promote`, { prompt_candidate_id: args.candidate_id, alias });
+      return okResult(
+        `Promoted ${candidate.label} as version ${result.version.versionNumber}; alias "${result.alias.alias}" now points at it.`,
+        result,
+        config.maxTextBytes
+      );
+    }
   })
 ];
 
@@ -845,6 +1107,9 @@ function gatewayMeta(headers) {
   }
   return meta;
 }
+function traceSuffix(meta) {
+  return meta["trace-id"] ? ` Open it with get_trace, trace_id ${meta["trace-id"]}.` : "";
+}
 function firstChoiceText(body) {
   const choice = body?.choices?.[0];
   const content = choice?.message?.content;
@@ -889,9 +1154,7 @@ var gatewayTools = [
       });
       const meta = gatewayMeta(response.headers);
       return okResult(
-        `Ran "${name}"@${args.alias ?? "production"} on ${meta.model ?? "the resolved model"}${meta["cost-usd"] ? ` for $${meta["cost-usd"]}` : ""}. Reply: ${firstChoiceText(
-          response.data
-        ).slice(0, 400)}`,
+        `Ran "${name}"@${args.alias ?? "production"} on ${meta.model ?? "the resolved model"}${meta["cost-usd"] ? ` for $${meta["cost-usd"]}` : ""}.${traceSuffix(meta)} Reply: ` + firstChoiceText(response.data).slice(0, 400),
         { completion: response.data, gateway: meta },
         config.maxTextBytes
       );
@@ -932,9 +1195,7 @@ var gatewayTools = [
       });
       const meta = gatewayMeta(response.headers);
       return okResult(
-        `Completion from ${meta.model ?? args.model}${meta["cost-usd"] ? ` ($${meta["cost-usd"]})` : ""}: ${firstChoiceText(
-          response.data
-        ).slice(0, 400)}`,
+        `Completion from ${meta.model ?? args.model}${meta["cost-usd"] ? ` ($${meta["cost-usd"]})` : ""}.${traceSuffix(meta)} Reply: ` + firstChoiceText(response.data).slice(0, 400),
         { completion: response.data, gateway: meta },
         config.maxTextBytes
       );
@@ -974,6 +1235,37 @@ var gatewayTools = [
         config.maxTextBytes
       );
     }
+  }),
+  defineTool({
+    name: "get_spending_limits",
+    title: "Get budgets and virtual-key limits",
+    toolset: "gateway",
+    readOnlyHint: true,
+    description: "Every spend budget on the team \u2014 team-wide or on one virtual key \u2014 with its period, limit, live spend, what is left and when it resets, plus each virtual key's rate limits and model/provider allow-lists. Use this after a gateway call is refused with HTTP 402 (budget spent) or 429 (rate limit) to see which limit caused it. Read-only: budgets and keys are changed in the dashboard.",
+    inputSchema: {},
+    async handler(_args, { client, config }) {
+      const [budgets, keys] = await Promise.all([
+        client.get("/gateway/budgets"),
+        client.get("/gateway/keys")
+      ]);
+      const keyNames = new Map(keys.map((k) => [k.id, k.name]));
+      const limits = budgets.map((b) => ({
+        id: b.id,
+        scope: b.virtualKeyId === null ? "team" : "virtual_key",
+        virtualKeyId: b.virtualKeyId,
+        virtualKeyName: b.virtualKeyId === null ? null : keyNames.get(b.virtualKeyId) ?? null,
+        period: b.period,
+        limitUsd: b.limitUsd,
+        spendUsd: b.spendUsd,
+        remainingUsd: Math.max(0, b.limitUsd - b.spendUsd),
+        exhausted: b.spendUsd >= b.limitUsd,
+        resetsAt: b.resetsAt
+      }));
+      const summary = limits.length === 0 ? `No budgets on this team, so the gateway never refuses a call for spend. ${keys.length} virtual key(s).` : `${limits.length} budget(s), ${keys.length} virtual key(s). ` + limits.map(
+        (l) => `${l.scope === "team" ? "Team" : `Key "${l.virtualKeyName ?? l.virtualKeyId}"`} ${l.period}: $${l.spendUsd} of $${l.limitUsd} spent` + (l.exhausted ? " (EXHAUSTED)" : "") + (l.resetsAt ? `, resets ${l.resetsAt}` : "")
+      ).join("; ") + ".";
+      return okResult(summary, { budgets: limits, virtualKeys: keys }, config.maxTextBytes);
+    }
   })
 ];
 
@@ -1339,7 +1631,7 @@ function selectTools(config) {
 }
 
 // src/server.ts
-var SERVER_VERSION = "0.1.0";
+var SERVER_VERSION = "0.2.0";
 function buildInstructions(toolNames, readOnly) {
   const parts = [
     `AcruxCore is an LLM-ops platform: a prompt registry, a versioned tool catalog, an AI gateway, tracing, and evaluation.