@acruxcoreai/mcp-server 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/README.md +3 -2
- package/dist/bin.js +325 -14
- package/dist/bin.js.map +1 -1
- package/dist/bin.mjs +325 -14
- package/dist/bin.mjs.map +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +325 -14
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +325 -14
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.d.mts
CHANGED
|
@@ -162,7 +162,7 @@ declare function resolveConfig(sources: ConfigSources): McpServerConfig;
|
|
|
162
162
|
declare function parseToolsets(raw: string): Toolset[];
|
|
163
163
|
|
|
164
164
|
/** Version reported to clients in `initialize`. Kept in step with package.json. */
|
|
165
|
-
declare const SERVER_VERSION = "0.
|
|
165
|
+
declare const SERVER_VERSION = "0.3.0";
|
|
166
166
|
/** What the caller gets back, so a host can inspect what was registered. */
|
|
167
167
|
interface BuiltServer {
|
|
168
168
|
/** The configured MCP server, not yet connected to a transport. */
|
package/dist/index.d.ts
CHANGED
|
@@ -162,7 +162,7 @@ declare function resolveConfig(sources: ConfigSources): McpServerConfig;
|
|
|
162
162
|
declare function parseToolsets(raw: string): Toolset[];
|
|
163
163
|
|
|
164
164
|
/** Version reported to clients in `initialize`. Kept in step with package.json. */
|
|
165
|
-
declare const SERVER_VERSION = "0.
|
|
165
|
+
declare const SERVER_VERSION = "0.3.0";
|
|
166
166
|
/** What the caller gets back, so a host can inspect what was registered. */
|
|
167
167
|
interface BuiltServer {
|
|
168
168
|
/** The configured MCP server, not yet connected to a transport. */
|
package/dist/index.js
CHANGED
|
@@ -242,10 +242,15 @@ function toApiError(label, status, body) {
|
|
|
242
242
|
}
|
|
243
243
|
|
|
244
244
|
// src/format.ts
|
|
245
|
+
var TRUNCATION_MARKER = "\n\n[truncated: response exceeded the size limit \u2014 narrow the query, use a smaller `limit`, or fetch one item at a time]";
|
|
246
|
+
function isTruncated(result) {
|
|
247
|
+
const first = result.content[0];
|
|
248
|
+
return first?.type === "text" && first.text.endsWith(TRUNCATION_MARKER);
|
|
249
|
+
}
|
|
245
250
|
function truncateText(text, maxBytes) {
|
|
246
251
|
const encoder = new TextEncoder();
|
|
247
252
|
if (encoder.encode(text).length <= maxBytes) return text;
|
|
248
|
-
const marker =
|
|
253
|
+
const marker = TRUNCATION_MARKER;
|
|
249
254
|
const budget = Math.max(0, maxBytes - encoder.encode(marker).length);
|
|
250
255
|
let end = Math.min(text.length, budget);
|
|
251
256
|
while (end > 0 && encoder.encode(text.slice(0, end)).length > budget) {
|
|
@@ -628,10 +633,87 @@ var catalogTools = [
|
|
|
628
633
|
|
|
629
634
|
// src/toolsets/evaluations.ts
|
|
630
635
|
var import_zod2 = require("zod");
|
|
636
|
+
|
|
637
|
+
// src/optimize-report.ts
|
|
638
|
+
function summarizeCandidates(grid, cells) {
|
|
639
|
+
const cellByKey = new Map(cells.map((cell) => [cell.cellKey, cell]));
|
|
640
|
+
const baselineByModel = new Map(
|
|
641
|
+
cells.filter((cell) => cell.isProductionBaseline).map((cell) => [cell.model, cell])
|
|
642
|
+
);
|
|
643
|
+
const byCandidate = /* @__PURE__ */ new Map();
|
|
644
|
+
for (const gridCell of grid) {
|
|
645
|
+
if (gridCell.variantKind !== "candidate" || !gridCell.promptCandidateId) continue;
|
|
646
|
+
const reportCell = cellByKey.get(gridCell.cellKey);
|
|
647
|
+
const baseline = baselineByModel.get(gridCell.model);
|
|
648
|
+
let summary = byCandidate.get(gridCell.promptCandidateId);
|
|
649
|
+
if (!summary) {
|
|
650
|
+
summary = {
|
|
651
|
+
candidateId: gridCell.promptCandidateId,
|
|
652
|
+
label: gridCell.variantLabel,
|
|
653
|
+
scores: [],
|
|
654
|
+
beatsBaseline: false
|
|
655
|
+
};
|
|
656
|
+
byCandidate.set(gridCell.promptCandidateId, summary);
|
|
657
|
+
}
|
|
658
|
+
summary.scores.push({
|
|
659
|
+
model: gridCell.model,
|
|
660
|
+
avgScore: reportCell?.avgScore ?? null,
|
|
661
|
+
passRate: reportCell?.passRate ?? null,
|
|
662
|
+
baselineAvgScore: baseline?.avgScore ?? null,
|
|
663
|
+
scoreDelta: reportCell?.deltaVsBaseline?.score ?? null,
|
|
664
|
+
verdict: reportCell?.deltaVsBaseline?.label ?? "unknown"
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
const summaries = [...byCandidate.values()];
|
|
668
|
+
for (const summary of summaries) {
|
|
669
|
+
summary.beatsBaseline = summary.scores.some((score) => score.verdict === "improved") && summary.scores.every((score) => score.verdict === "improved" || score.verdict === "flat");
|
|
670
|
+
}
|
|
671
|
+
return summaries.sort((a, b) => a.label.localeCompare(b.label));
|
|
672
|
+
}
|
|
673
|
+
function bestCandidate(candidates) {
|
|
674
|
+
let best = null;
|
|
675
|
+
let bestDelta = -Infinity;
|
|
676
|
+
for (const candidate of candidates) {
|
|
677
|
+
if (!candidate.beatsBaseline) continue;
|
|
678
|
+
const deltas = candidate.scores.map((score) => score.scoreDelta).filter((delta) => delta !== null);
|
|
679
|
+
const mean = deltas.reduce((sum, delta) => sum + delta, 0) / Math.max(1, deltas.length);
|
|
680
|
+
if (mean > bestDelta) {
|
|
681
|
+
best = candidate;
|
|
682
|
+
bestDelta = mean;
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
return best;
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
// src/toolsets/evaluations.ts
|
|
631
689
|
var historyMessageSchema = import_zod2.z.object({
|
|
632
690
|
role: import_zod2.z.enum(["system", "user", "assistant", "tool"]),
|
|
633
691
|
content: import_zod2.z.string()
|
|
634
692
|
});
|
|
693
|
+
async function loadOptimizeRun(client, runId) {
|
|
694
|
+
const run = await client.get(`/runs/${runId}`);
|
|
695
|
+
const notDraftedYet = run.grid.length === 0;
|
|
696
|
+
if (run.kind !== "optimize" && !notDraftedYet) {
|
|
697
|
+
throw new Error(
|
|
698
|
+
`Run ${runId} is an evaluation run, not an optimize run. Read it with get_experiment_run.`
|
|
699
|
+
);
|
|
700
|
+
}
|
|
701
|
+
if (run.status !== "succeeded") return { run, report: null, candidates: [] };
|
|
702
|
+
const report = await client.get(`/runs/${runId}/report`);
|
|
703
|
+
return { run, report, candidates: summarizeCandidates(run.grid, report.cells) };
|
|
704
|
+
}
|
|
705
|
+
function baselineOf(run) {
|
|
706
|
+
const cell = run.grid.find((row) => row.isProductionBaseline && row.promptVersionId);
|
|
707
|
+
if (!cell?.promptVersionId) {
|
|
708
|
+
throw new Error(`Run ${run.id} has no baseline version, so a promote cannot be checked.`);
|
|
709
|
+
}
|
|
710
|
+
const alias = /^v\d+$/.test(cell.variantLabel) ? null : cell.variantLabel;
|
|
711
|
+
return { versionId: cell.promptVersionId, alias };
|
|
712
|
+
}
|
|
713
|
+
function formatDelta(delta) {
|
|
714
|
+
if (delta === null) return "n/a";
|
|
715
|
+
return `${delta >= 0 ? "+" : ""}${delta.toFixed(1)}`;
|
|
716
|
+
}
|
|
635
717
|
var evaluationTools = [
|
|
636
718
|
defineTool({
|
|
637
719
|
name: "list_datasets",
|
|
@@ -710,21 +792,29 @@ var evaluationTools = [
|
|
|
710
792
|
title: "Create and start an experiment run",
|
|
711
793
|
toolset: "evaluations",
|
|
712
794
|
readOnlyHint: false,
|
|
713
|
-
description: "Create an experiment over a dataset and immediately start a run of it. The grid is every (version x model) pair you list. When `prompt` is given, a baseline cell for its production alias is added automatically unless that version is already in the grid. Returns as soon as the run is queued \u2014 the worker executes it asynchronously, so poll get_experiment_run for results.
|
|
795
|
+
description: "Create an experiment over a dataset and immediately start a run of it. The grid is every (version x model) pair you list. When `prompt` is given, a baseline cell for its production alias is added automatically unless that version is already in the grid. Returns as soon as the run is queued \u2014 the worker executes it asynchronously, so poll get_experiment_run for results. `judge_model` scores every answer and must be registered in the gateway (see list_models). `judge_prompt` grades with one of your prompts instead of the built-in judge.",
|
|
714
796
|
inputSchema: {
|
|
715
797
|
dataset_id: import_zod2.z.string().uuid().describe("Dataset to evaluate against."),
|
|
716
798
|
version_ids: import_zod2.z.array(import_zod2.z.string().uuid()).min(1).describe("Prompt-version UUIDs to sweep. At least one."),
|
|
717
799
|
models: import_zod2.z.array(import_zod2.z.string().min(1)).min(1).describe("Gateway model public names to sweep."),
|
|
718
800
|
prompt: import_zod2.z.string().optional().describe("Prompt name or UUID under test \u2014 enables the automatic baseline cell."),
|
|
719
801
|
name: import_zod2.z.string().optional().describe("Experiment name, for the run history."),
|
|
720
|
-
alias: import_zod2.z.string().optional().describe('Which alias the automatic baseline points at. Default "production".')
|
|
802
|
+
alias: import_zod2.z.string().optional().describe('Which alias the automatic baseline points at. Default "production".'),
|
|
803
|
+
judge_model: import_zod2.z.string().min(1).describe("Gateway model that scores every answer. Required."),
|
|
804
|
+
judge_prompt: import_zod2.z.string().min(1).optional().describe(
|
|
805
|
+
"Name or UUID of a team prompt that grades instead of the built-in judge. The run uses its production version (else its latest). Default: the built-in judge."
|
|
806
|
+
)
|
|
721
807
|
},
|
|
722
808
|
async handler(args, { client, config }) {
|
|
723
809
|
const body = {
|
|
724
810
|
dataset_id: args.dataset_id,
|
|
725
811
|
version_ids: args.version_ids,
|
|
726
|
-
models: args.models
|
|
812
|
+
models: args.models,
|
|
813
|
+
judge_model: args.judge_model
|
|
727
814
|
};
|
|
815
|
+
if (args.judge_prompt !== void 0) {
|
|
816
|
+
body.judge_prompt_id = await resolvePromptRef(client, args.judge_prompt);
|
|
817
|
+
}
|
|
728
818
|
if (args.name !== void 0) body.name = args.name;
|
|
729
819
|
if (args.alias !== void 0) body.alias = args.alias;
|
|
730
820
|
if (args.prompt !== void 0) body.prompt_id = await resolvePromptRef(client, args.prompt);
|
|
@@ -766,23 +856,214 @@ var evaluationTools = [
|
|
|
766
856
|
title: "Get a run, optionally with its comparison report",
|
|
767
857
|
toolset: "evaluations",
|
|
768
858
|
readOnlyHint: true,
|
|
769
|
-
description: "Read one run's status and results. With include_report=true it also fetches the comparison report: the full (variant x model) matrix with per-cell averages, each cell's delta against the same-model production baseline, a leaderboard and an advisory winner. Ask for the report only once the run has succeeded \u2014 an in-flight run has nothing to compare.",
|
|
859
|
+
description: "Read one run's status and results. With include_report=true it also fetches the comparison report: the full (variant x model) matrix with per-cell averages, each cell's delta against the same-model production baseline, a leaderboard and an advisory winner. Ask for the report only once the run has succeeded \u2014 an in-flight run has nothing to compare. For an optimize run, get_optimize_report gives the same scores per candidate, with the ids promote_optimize_candidate needs.",
|
|
770
860
|
inputSchema: {
|
|
771
861
|
run_id: import_zod2.z.string().uuid().describe("Run UUID."),
|
|
772
862
|
include_report: import_zod2.z.boolean().optional().describe("Also fetch the comparison report.")
|
|
773
863
|
},
|
|
774
864
|
async handler({ run_id, include_report }, { client, config }) {
|
|
775
|
-
const run = await client.get(
|
|
865
|
+
const run = await client.get(
|
|
866
|
+
`/runs/${run_id}`
|
|
867
|
+
);
|
|
776
868
|
let report = void 0;
|
|
777
869
|
if (include_report) {
|
|
778
870
|
report = await client.get(`/runs/${run_id}/report`);
|
|
779
871
|
}
|
|
780
872
|
return okResult(
|
|
781
|
-
`Run ${run_id} is ${run.status} with ${run.results?.
|
|
873
|
+
`Run ${run_id} is ${run.status} with ${run.results?.total ?? 0} result(s).`,
|
|
782
874
|
{ run, report },
|
|
783
875
|
config.maxTextBytes
|
|
784
876
|
);
|
|
785
877
|
}
|
|
878
|
+
}),
|
|
879
|
+
defineTool({
|
|
880
|
+
name: "optimize_prompt",
|
|
881
|
+
title: "Start an optimizer run for a prompt",
|
|
882
|
+
toolset: "evaluations",
|
|
883
|
+
readOnlyHint: false,
|
|
884
|
+
description: 'Ask the optimizer to improve a prompt against a dataset. An optimizer model drafts `draft_count` rewrites of the version behind `alias` (default "production"), then every rewrite and that baseline run over the dataset on each of `models`, and `judge_model` scores every answer. Returns the run id as soon as the run is queued \u2014 drafting and scoring take minutes, so poll get_optimize_report. Nothing is promoted automatically: a rewrite only ships when promote_optimize_candidate is called. The dataset\'s examples need `criteria`, or the built-in judge has nothing to grade against. `judge_prompt` grades with one of your prompts instead, and the optimizer reads it too, so the rewrites aim at that rubric. Every model named here must be registered in the gateway (see list_models).',
|
|
885
|
+
inputSchema: {
|
|
886
|
+
prompt: import_zod2.z.string().min(1).describe("Prompt name or UUID to improve."),
|
|
887
|
+
dataset_id: import_zod2.z.string().uuid().describe("Dataset whose examples the rewrites are scored on."),
|
|
888
|
+
models: import_zod2.z.array(import_zod2.z.string().min(1)).min(1).describe("Gateway model public names to test each rewrite and the baseline on."),
|
|
889
|
+
judge_model: import_zod2.z.string().min(1).describe("Gateway model that scores every answer. Required."),
|
|
890
|
+
optimizer_model: import_zod2.z.string().min(1).optional().describe("Gateway model that writes the rewrites. Default: the first entry of `models`."),
|
|
891
|
+
draft_count: import_zod2.z.number().int().min(1).max(6).optional().describe("How many rewrites to draft, 1 to 6. Default 3."),
|
|
892
|
+
alias: import_zod2.z.string().min(1).optional().describe('Alias whose version is the baseline to beat. Default "production".'),
|
|
893
|
+
optimizer_prompt: import_zod2.z.string().optional().describe("Name or UUID of a team prompt to use as the optimizer's instructions."),
|
|
894
|
+
judge_prompt: import_zod2.z.string().min(1).optional().describe(
|
|
895
|
+
"Name or UUID of a team prompt that grades instead of the built-in judge. The run uses its production version (else its latest). Default: the built-in judge."
|
|
896
|
+
)
|
|
897
|
+
},
|
|
898
|
+
async handler(args, { client, config }) {
|
|
899
|
+
const promptId = await resolvePromptRef(client, args.prompt);
|
|
900
|
+
const body = {
|
|
901
|
+
dataset_id: args.dataset_id,
|
|
902
|
+
models: args.models,
|
|
903
|
+
judge_model: args.judge_model
|
|
904
|
+
};
|
|
905
|
+
if (args.optimizer_model !== void 0) body.optimizer_model = args.optimizer_model;
|
|
906
|
+
if (args.draft_count !== void 0) body.draft_count = args.draft_count;
|
|
907
|
+
if (args.alias !== void 0) body.alias = args.alias;
|
|
908
|
+
if (args.optimizer_prompt !== void 0) {
|
|
909
|
+
body.optimizer_prompt_id = await resolvePromptRef(client, args.optimizer_prompt);
|
|
910
|
+
}
|
|
911
|
+
if (args.judge_prompt !== void 0) {
|
|
912
|
+
body.judge_prompt_id = await resolvePromptRef(client, args.judge_prompt);
|
|
913
|
+
}
|
|
914
|
+
const result = await client.post(`/prompts/${promptId}/optimize`, body);
|
|
915
|
+
const warning = result.prompt_mismatch_warning ? " Warning: some dataset examples were built from a different prompt \u2014 see prompt_mismatch_warning." : "";
|
|
916
|
+
return okResult(
|
|
917
|
+
`Optimize run ${result.run_id} is ${result.status}; ${result.optimizer_model} drafts the rewrites, then they are scored against the ${args.alias ?? "production"} baseline. Poll get_optimize_report with this run id.${warning}`,
|
|
918
|
+
result,
|
|
919
|
+
config.maxTextBytes
|
|
920
|
+
);
|
|
921
|
+
}
|
|
922
|
+
}),
|
|
923
|
+
defineTool({
|
|
924
|
+
name: "get_optimize_report",
|
|
925
|
+
title: "Read an optimizer run's candidates and scores",
|
|
926
|
+
toolset: "evaluations",
|
|
927
|
+
readOnlyHint: true,
|
|
928
|
+
description: "Read an optimize run started by optimize_prompt. While it is queued or running this returns only its status \u2014 call again later. Once it has succeeded it lists every drafted rewrite with its candidate id, the optimizer's rationale, the rewritten messages, and its average score next to the baseline's on each model. `beatsBaseline` is true only for a rewrite that improved on at least one model and was improved or flat on every other; `recommended` is the one among those with the largest gain, or null when no rewrite beat the baseline. When the rewrites are too long to fit in one result, their messages are left out (`messagesOmitted`); pass `candidate_id` to read one rewrite in full.",
|
|
929
|
+
inputSchema: {
|
|
930
|
+
run_id: import_zod2.z.string().uuid().describe("Optimize run UUID, from optimize_prompt."),
|
|
931
|
+
candidate_id: import_zod2.z.string().uuid().optional().describe("Return only this candidate, with its full messages.")
|
|
932
|
+
},
|
|
933
|
+
async handler({ run_id, candidate_id }, { client, config }) {
|
|
934
|
+
const { run, report, candidates } = await loadOptimizeRun(client, run_id);
|
|
935
|
+
if (run.status === "queued" || run.status === "running") {
|
|
936
|
+
return okResult(
|
|
937
|
+
`Optimize run ${run_id} is ${run.status}. No scores yet \u2014 call get_optimize_report again shortly.`,
|
|
938
|
+
{ runId: run_id, status: run.status },
|
|
939
|
+
config.maxTextBytes
|
|
940
|
+
);
|
|
941
|
+
}
|
|
942
|
+
if (run.status === "failed") {
|
|
943
|
+
return okResult(
|
|
944
|
+
`Optimize run ${run_id} failed: ${run.error ?? "no reason recorded"}. Nothing to promote.`,
|
|
945
|
+
{ runId: run_id, status: run.status, error: run.error },
|
|
946
|
+
config.maxTextBytes
|
|
947
|
+
);
|
|
948
|
+
}
|
|
949
|
+
if (candidate_id !== void 0) {
|
|
950
|
+
const candidate = candidates.find((row) => row.candidateId === candidate_id);
|
|
951
|
+
if (!candidate) {
|
|
952
|
+
throw new Error(`Candidate ${candidate_id} is not a rewrite from run ${run_id}.`);
|
|
953
|
+
}
|
|
954
|
+
const detail = await client.get(
|
|
955
|
+
`/runs/${run_id}/candidates/${candidate_id}`
|
|
956
|
+
);
|
|
957
|
+
return okResult(
|
|
958
|
+
`${candidate.label} (candidate_id ${candidate_id}) from optimize run ${run_id}; beatsBaseline: ${candidate.beatsBaseline}.`,
|
|
959
|
+
{
|
|
960
|
+
runId: run_id,
|
|
961
|
+
candidate: { ...candidate, rationale: detail.rationale, messages: detail.messages }
|
|
962
|
+
},
|
|
963
|
+
config.maxTextBytes
|
|
964
|
+
);
|
|
965
|
+
}
|
|
966
|
+
const reads = await Promise.allSettled(
|
|
967
|
+
candidates.map(
|
|
968
|
+
(candidate) => client.get(`/runs/${run_id}/candidates/${candidate.candidateId}`)
|
|
969
|
+
)
|
|
970
|
+
);
|
|
971
|
+
const withContent = candidates.map((candidate, index) => {
|
|
972
|
+
const read = reads[index];
|
|
973
|
+
return read.status === "fulfilled" ? { ...candidate, rationale: read.value.rationale, messages: read.value.messages } : {
|
|
974
|
+
...candidate,
|
|
975
|
+
rationale: null,
|
|
976
|
+
messages: null,
|
|
977
|
+
readError: read.reason instanceof Error ? read.reason.message : String(read.reason)
|
|
978
|
+
};
|
|
979
|
+
});
|
|
980
|
+
const best = bestCandidate(candidates);
|
|
981
|
+
const winners = candidates.filter((candidate) => candidate.beatsBaseline).length;
|
|
982
|
+
const rejected = report?.optimizerRejections ?? [];
|
|
983
|
+
const rejectedNote = rejected.length > 0 ? ` ${rejected.length} more rewrite(s) were dropped before scoring: ${rejected.join(" | ")}.` : "";
|
|
984
|
+
const summary = (best ? `Optimize run ${run_id} succeeded. ${winners} of ${candidates.length} rewrite(s) beat the baseline. Recommended: ${best.label} (candidate_id ${best.candidateId}), ` + best.scores.map((score) => `${formatDelta(score.scoreDelta)} on ${score.model}`).join(", ") + ". Promote it with promote_optimize_candidate only if the user agrees." : `Optimize run ${run_id} succeeded, but none of its ${candidates.length} rewrite(s) beat the baseline. Nothing is worth promoting; keep the current version.`) + rejectedNote;
|
|
985
|
+
const payload = {
|
|
986
|
+
runId: run_id,
|
|
987
|
+
status: run.status,
|
|
988
|
+
judgeModel: report?.judge?.model ?? null,
|
|
989
|
+
// Null means the built-in judge graded the run.
|
|
990
|
+
judgePromptVersionId: report?.judge?.promptVersionId ?? null,
|
|
991
|
+
recommended: best?.candidateId ?? null,
|
|
992
|
+
rejectedRewrites: rejected,
|
|
993
|
+
baseline: (report?.cells ?? []).filter((cell) => cell.isProductionBaseline).map((cell) => ({
|
|
994
|
+
label: cell.variantLabel,
|
|
995
|
+
model: cell.model,
|
|
996
|
+
avgScore: cell.avgScore,
|
|
997
|
+
passRate: cell.passRate
|
|
998
|
+
})),
|
|
999
|
+
candidates: withContent
|
|
1000
|
+
};
|
|
1001
|
+
const full = okResult(summary, payload, config.maxTextBytes);
|
|
1002
|
+
if (!isTruncated(full)) return full;
|
|
1003
|
+
return okResult(
|
|
1004
|
+
`${summary} The rewrites are too long to show together, so their messages are left out; call get_optimize_report again with candidate_id to read one.`,
|
|
1005
|
+
{
|
|
1006
|
+
...payload,
|
|
1007
|
+
messagesOmitted: true,
|
|
1008
|
+
candidates: withContent.map(({ messages: _messages, ...rest }) => rest)
|
|
1009
|
+
},
|
|
1010
|
+
config.maxTextBytes
|
|
1011
|
+
);
|
|
1012
|
+
}
|
|
1013
|
+
}),
|
|
1014
|
+
defineTool({
|
|
1015
|
+
name: "promote_optimize_candidate",
|
|
1016
|
+
title: "Ship an optimizer rewrite as a new prompt version",
|
|
1017
|
+
toolset: "evaluations",
|
|
1018
|
+
readOnlyHint: false,
|
|
1019
|
+
destructiveHint: true,
|
|
1020
|
+
description: "Commit one rewrite from a succeeded optimize run as a new prompt version and move `alias` to it, so live traffic on that alias starts using it. `alias` defaults to the alias the run was scored against (production when the run used the latest version). Take `candidate_id` from get_optimize_report. Refuses a rewrite that did not beat the baseline, and refuses when the alias now points at a version other than the one the run scored against; set `promote_without_improvement` only when the user has explicitly asked to ship that rewrite anyway. Undo by moving the alias back with promote_prompt_alias.",
|
|
1021
|
+
inputSchema: {
|
|
1022
|
+
run_id: import_zod2.z.string().uuid().describe("Optimize run UUID."),
|
|
1023
|
+
candidate_id: import_zod2.z.string().uuid().describe("Candidate id from get_optimize_report."),
|
|
1024
|
+
alias: import_zod2.z.string().min(1).optional().describe("Alias to move. Default: the alias the run was scored against."),
|
|
1025
|
+
promote_without_improvement: import_zod2.z.boolean().optional().describe(
|
|
1026
|
+
"Ship a rewrite that did not beat the baseline, or replace a version the run never scored. Only on the user's explicit say-so."
|
|
1027
|
+
)
|
|
1028
|
+
},
|
|
1029
|
+
async handler(args, { client, config }) {
|
|
1030
|
+
const { run, candidates } = await loadOptimizeRun(client, args.run_id);
|
|
1031
|
+
if (run.status !== "succeeded") {
|
|
1032
|
+
throw new Error(
|
|
1033
|
+
`Optimize run ${args.run_id} is ${run.status}, so its rewrites are not fully scored. Promote only from a succeeded run.`
|
|
1034
|
+
);
|
|
1035
|
+
}
|
|
1036
|
+
const candidate = candidates.find((row) => row.candidateId === args.candidate_id);
|
|
1037
|
+
if (!candidate) {
|
|
1038
|
+
throw new Error(
|
|
1039
|
+
`Candidate ${args.candidate_id} is not a rewrite from run ${args.run_id}. Take the id from get_optimize_report.`
|
|
1040
|
+
);
|
|
1041
|
+
}
|
|
1042
|
+
if (!candidate.beatsBaseline && !args.promote_without_improvement) {
|
|
1043
|
+
const scores = candidate.scores.map((score) => `${formatDelta(score.scoreDelta)} (${score.verdict}) on ${score.model}`).join(", ");
|
|
1044
|
+
throw new Error(
|
|
1045
|
+
`Refused: ${candidate.label} did not beat the baseline (${scores}). Promoting it would ship a prompt that scored no better than the current one. Set promote_without_improvement to true only if the user explicitly asked for this rewrite.`
|
|
1046
|
+
);
|
|
1047
|
+
}
|
|
1048
|
+
const baseline = baselineOf(run);
|
|
1049
|
+
const alias = args.alias ?? baseline.alias ?? "production";
|
|
1050
|
+
const detail = await client.get(
|
|
1051
|
+
`/runs/${args.run_id}/candidates/${args.candidate_id}`
|
|
1052
|
+
);
|
|
1053
|
+
const aliases = await client.get(`/prompts/${detail.promptId}/aliases`);
|
|
1054
|
+
const live = aliases.find((row) => row.alias === alias);
|
|
1055
|
+
if (live && live.versionId !== baseline.versionId && !args.promote_without_improvement) {
|
|
1056
|
+
throw new Error(
|
|
1057
|
+
`Refused: alias "${alias}" now points at version ${live.versionNumber}, but run ${args.run_id} scored its rewrites against a different version. Nothing compared ${candidate.label} with version ${live.versionNumber}. Start a new optimize run against this alias, or set promote_without_improvement to true only if the user explicitly asked to replace it anyway.`
|
|
1058
|
+
);
|
|
1059
|
+
}
|
|
1060
|
+
const result = await client.post(`/runs/${args.run_id}/promote`, { prompt_candidate_id: args.candidate_id, alias });
|
|
1061
|
+
return okResult(
|
|
1062
|
+
`Promoted ${candidate.label} as version ${result.version.versionNumber}; alias "${result.alias.alias}" now points at it.`,
|
|
1063
|
+
result,
|
|
1064
|
+
config.maxTextBytes
|
|
1065
|
+
);
|
|
1066
|
+
}
|
|
786
1067
|
})
|
|
787
1068
|
];
|
|
788
1069
|
|
|
@@ -809,6 +1090,9 @@ function gatewayMeta(headers) {
|
|
|
809
1090
|
}
|
|
810
1091
|
return meta;
|
|
811
1092
|
}
|
|
1093
|
+
function traceSuffix(meta) {
|
|
1094
|
+
return meta["trace-id"] ? ` Open it with get_trace, trace_id ${meta["trace-id"]}.` : "";
|
|
1095
|
+
}
|
|
812
1096
|
function firstChoiceText(body) {
|
|
813
1097
|
const choice = body?.choices?.[0];
|
|
814
1098
|
const content = choice?.message?.content;
|
|
@@ -853,9 +1137,7 @@ var gatewayTools = [
|
|
|
853
1137
|
});
|
|
854
1138
|
const meta = gatewayMeta(response.headers);
|
|
855
1139
|
return okResult(
|
|
856
|
-
`Ran "${name}"@${args.alias ?? "production"} on ${meta.model ?? "the resolved model"}${meta["cost-usd"] ? ` for $${meta["cost-usd"]}` : ""}
|
|
857
|
-
response.data
|
|
858
|
-
).slice(0, 400)}`,
|
|
1140
|
+
`Ran "${name}"@${args.alias ?? "production"} on ${meta.model ?? "the resolved model"}${meta["cost-usd"] ? ` for $${meta["cost-usd"]}` : ""}.${traceSuffix(meta)} Reply: ` + firstChoiceText(response.data).slice(0, 400),
|
|
859
1141
|
{ completion: response.data, gateway: meta },
|
|
860
1142
|
config.maxTextBytes
|
|
861
1143
|
);
|
|
@@ -896,9 +1178,7 @@ var gatewayTools = [
|
|
|
896
1178
|
});
|
|
897
1179
|
const meta = gatewayMeta(response.headers);
|
|
898
1180
|
return okResult(
|
|
899
|
-
`Completion from ${meta.model ?? args.model}${meta["cost-usd"] ? ` ($${meta["cost-usd"]})` : ""}:
|
|
900
|
-
response.data
|
|
901
|
-
).slice(0, 400)}`,
|
|
1181
|
+
`Completion from ${meta.model ?? args.model}${meta["cost-usd"] ? ` ($${meta["cost-usd"]})` : ""}.${traceSuffix(meta)} Reply: ` + firstChoiceText(response.data).slice(0, 400),
|
|
902
1182
|
{ completion: response.data, gateway: meta },
|
|
903
1183
|
config.maxTextBytes
|
|
904
1184
|
);
|
|
@@ -938,6 +1218,37 @@ var gatewayTools = [
|
|
|
938
1218
|
config.maxTextBytes
|
|
939
1219
|
);
|
|
940
1220
|
}
|
|
1221
|
+
}),
|
|
1222
|
+
defineTool({
|
|
1223
|
+
name: "get_spending_limits",
|
|
1224
|
+
title: "Get budgets and virtual-key limits",
|
|
1225
|
+
toolset: "gateway",
|
|
1226
|
+
readOnlyHint: true,
|
|
1227
|
+
description: "Every spend budget on the team \u2014 team-wide or on one virtual key \u2014 with its period, limit, live spend, what is left and when it resets, plus each virtual key's rate limits and model/provider allow-lists. Use this after a gateway call is refused with HTTP 402 (budget spent) or 429 (rate limit) to see which limit caused it. Read-only: budgets and keys are changed in the dashboard.",
|
|
1228
|
+
inputSchema: {},
|
|
1229
|
+
async handler(_args, { client, config }) {
|
|
1230
|
+
const [budgets, keys] = await Promise.all([
|
|
1231
|
+
client.get("/gateway/budgets"),
|
|
1232
|
+
client.get("/gateway/keys")
|
|
1233
|
+
]);
|
|
1234
|
+
const keyNames = new Map(keys.map((k) => [k.id, k.name]));
|
|
1235
|
+
const limits = budgets.map((b) => ({
|
|
1236
|
+
id: b.id,
|
|
1237
|
+
scope: b.virtualKeyId === null ? "team" : "virtual_key",
|
|
1238
|
+
virtualKeyId: b.virtualKeyId,
|
|
1239
|
+
virtualKeyName: b.virtualKeyId === null ? null : keyNames.get(b.virtualKeyId) ?? null,
|
|
1240
|
+
period: b.period,
|
|
1241
|
+
limitUsd: b.limitUsd,
|
|
1242
|
+
spendUsd: b.spendUsd,
|
|
1243
|
+
remainingUsd: Math.max(0, b.limitUsd - b.spendUsd),
|
|
1244
|
+
exhausted: b.spendUsd >= b.limitUsd,
|
|
1245
|
+
resetsAt: b.resetsAt
|
|
1246
|
+
}));
|
|
1247
|
+
const summary = limits.length === 0 ? `No budgets on this team, so the gateway never refuses a call for spend. ${keys.length} virtual key(s).` : `${limits.length} budget(s), ${keys.length} virtual key(s). ` + limits.map(
|
|
1248
|
+
(l) => `${l.scope === "team" ? "Team" : `Key "${l.virtualKeyName ?? l.virtualKeyId}"`} ${l.period}: $${l.spendUsd} of $${l.limitUsd} spent` + (l.exhausted ? " (EXHAUSTED)" : "") + (l.resetsAt ? `, resets ${l.resetsAt}` : "")
|
|
1249
|
+
).join("; ") + ".";
|
|
1250
|
+
return okResult(summary, { budgets: limits, virtualKeys: keys }, config.maxTextBytes);
|
|
1251
|
+
}
|
|
941
1252
|
})
|
|
942
1253
|
];
|
|
943
1254
|
|
|
@@ -1303,7 +1614,7 @@ function selectTools(config) {
|
|
|
1303
1614
|
}
|
|
1304
1615
|
|
|
1305
1616
|
// src/server.ts
|
|
1306
|
-
var SERVER_VERSION = "0.
|
|
1617
|
+
var SERVER_VERSION = "0.3.0";
|
|
1307
1618
|
function buildInstructions(toolNames, readOnly) {
|
|
1308
1619
|
const parts = [
|
|
1309
1620
|
`AcruxCore is an LLM-ops platform: a prompt registry, a versioned tool catalog, an AI gateway, tracing, and evaluation.
|