@mutagent/cli 0.1.114 → 0.1.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cli.js +898 -480
- package/dist/bin/cli.js.map +11 -8
- package/dist/index.js +66 -7
- package/dist/index.js.map +3 -3
- package/package.json +1 -1
package/dist/bin/cli.js
CHANGED
|
@@ -756,15 +756,74 @@ class SDKClientWrapper {
|
|
|
756
756
|
this.handleError(error);
|
|
757
757
|
}
|
|
758
758
|
}
|
|
759
|
-
async listExperiments() {
|
|
759
|
+
async listExperiments(filters) {
|
|
760
760
|
try {
|
|
761
|
-
const
|
|
761
|
+
const params = new URLSearchParams;
|
|
762
|
+
if (filters?.promptId !== undefined)
|
|
763
|
+
params.set("promptId", String(filters.promptId));
|
|
764
|
+
if (filters?.status)
|
|
765
|
+
params.set("status", filters.status);
|
|
766
|
+
if (filters?.limit !== undefined)
|
|
767
|
+
params.set("limit", String(filters.limit));
|
|
768
|
+
if (filters?.offset !== undefined)
|
|
769
|
+
params.set("offset", String(filters.offset));
|
|
770
|
+
const qs = params.toString();
|
|
771
|
+
const response = await this.request(`/api/prompts/experiments${qs ? `?${qs}` : ""}`);
|
|
762
772
|
if (Array.isArray(response))
|
|
763
|
-
return response;
|
|
773
|
+
return { experiments: response, total: response.length };
|
|
764
774
|
const r = response;
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
775
|
+
const experiments = r.experiments ?? r.data ?? [];
|
|
776
|
+
return { experiments, total: r.total ?? experiments.length };
|
|
777
|
+
} catch (error) {
|
|
778
|
+
this.handleError(error);
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
async createExperiment(data) {
|
|
782
|
+
try {
|
|
783
|
+
return await this.request("/api/prompts/experiments", {
|
|
784
|
+
method: "POST",
|
|
785
|
+
body: JSON.stringify(data)
|
|
786
|
+
});
|
|
787
|
+
} catch (error) {
|
|
788
|
+
this.handleError(error);
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
async getExperiment(id) {
|
|
792
|
+
try {
|
|
793
|
+
return await this.request(`/api/prompts/experiments/${id}`);
|
|
794
|
+
} catch (error) {
|
|
795
|
+
this.handleError(error);
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
async deleteExperiment(id) {
|
|
799
|
+
try {
|
|
800
|
+
await this.request(`/api/prompts/experiments/${id}`, { method: "DELETE" });
|
|
801
|
+
} catch (error) {
|
|
802
|
+
this.handleError(error);
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
async executeExperiment(id) {
|
|
806
|
+
try {
|
|
807
|
+
return await this.request(`/api/prompts/experiments/${id}/execute`, { method: "POST", body: JSON.stringify({}) });
|
|
808
|
+
} catch (error) {
|
|
809
|
+
this.handleError(error);
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
async playgroundEval(promptId, data) {
|
|
813
|
+
try {
|
|
814
|
+
return await this.request(`/api/prompt/${promptId}/playground/eval`, {
|
|
815
|
+
method: "POST",
|
|
816
|
+
body: JSON.stringify(data)
|
|
817
|
+
});
|
|
818
|
+
} catch (error) {
|
|
819
|
+
this.handleError(error);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
async cancelOptimization(jobId) {
|
|
823
|
+
try {
|
|
824
|
+
return await this.request(`/api/optimization/${jobId}/cancel`, {
|
|
825
|
+
method: "POST"
|
|
826
|
+
});
|
|
768
827
|
} catch (error) {
|
|
769
828
|
this.handleError(error);
|
|
770
829
|
}
|
|
@@ -1181,8 +1240,8 @@ var init_sdk_client = __esm(() => {
|
|
|
1181
1240
|
});
|
|
1182
1241
|
|
|
1183
1242
|
// src/bin/cli.ts
|
|
1184
|
-
import { Command as
|
|
1185
|
-
import
|
|
1243
|
+
import { Command as Command21 } from "commander";
|
|
1244
|
+
import chalk38 from "chalk";
|
|
1186
1245
|
import { readFileSync as readFileSync12 } from "fs";
|
|
1187
1246
|
import { join as join10, dirname as dirname2 } from "path";
|
|
1188
1247
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
@@ -2569,8 +2628,8 @@ ${chalk5.dim("Note: `mutagent auth login` is a back-compat alias and behaves ide
|
|
|
2569
2628
|
// src/commands/prompts/index.ts
|
|
2570
2629
|
init_errors();
|
|
2571
2630
|
init_sdk_client();
|
|
2572
|
-
import { Command as
|
|
2573
|
-
import
|
|
2631
|
+
import { Command as Command7 } from "commander";
|
|
2632
|
+
import chalk18 from "chalk";
|
|
2574
2633
|
import { readFileSync as readFileSync4, existsSync as existsSync4 } from "fs";
|
|
2575
2634
|
|
|
2576
2635
|
// src/lib/ui-links.ts
|
|
@@ -3621,6 +3680,26 @@ ${chalk8.red("Required: --data must be provided.")}
|
|
|
3621
3680
|
throw new MutagentError("INVALID_JSON", "Invalid JSON in -d/--data flag", `Run: mutagent prompts dataset add --help
|
|
3622
3681
|
` + `Provide a valid JSON array, e.g., '[{"input": {...}, "expectedOutput": {...}}]'`);
|
|
3623
3682
|
}
|
|
3683
|
+
const parsedItems = JSON.parse(content);
|
|
3684
|
+
if (parsedItems.length === 0) {
|
|
3685
|
+
throw new MutagentError("VALIDATION_ERROR", "Dataset cannot be empty", `Run: mutagent prompts dataset add --help
|
|
3686
|
+
Provide at least one item in the array.`);
|
|
3687
|
+
}
|
|
3688
|
+
const warnings = [];
|
|
3689
|
+
for (let i = 0;i < parsedItems.length; i++) {
|
|
3690
|
+
const item = parsedItems[i];
|
|
3691
|
+
if (!("input" in item)) {
|
|
3692
|
+
throw new MutagentError("VALIDATION_ERROR", `Dataset item at index ${i} is missing required field "input"`, `Run: mutagent prompts dataset add --help
|
|
3693
|
+
Each item must have: {"input": {...}, "expectedOutput": {...}}`);
|
|
3694
|
+
}
|
|
3695
|
+
if (typeof item.input !== "object" || item.input === null || Array.isArray(item.input)) {
|
|
3696
|
+
throw new MutagentError("VALIDATION_ERROR", `Dataset item at index ${i}: "input" must be an object, got ${Array.isArray(item.input) ? "array" : typeof item.input}`, `Run: mutagent prompts dataset add --help
|
|
3697
|
+
"input" must be a JSON object: {"input": {"field": "value"}}`);
|
|
3698
|
+
}
|
|
3699
|
+
if (!("expectedOutput" in item)) {
|
|
3700
|
+
warnings.push(`Item at index ${i} is missing "expectedOutput" (recommended for evaluation scoring)`);
|
|
3701
|
+
}
|
|
3702
|
+
}
|
|
3624
3703
|
const datasetName = options.name;
|
|
3625
3704
|
if (!datasetName) {
|
|
3626
3705
|
throw new MutagentError("MISSING_ARGUMENTS", "Dataset name is required", `Run: mutagent prompts dataset add --help
|
|
@@ -3643,6 +3722,9 @@ ${chalk8.red("Required: --data must be provided.")}
|
|
|
3643
3722
|
});
|
|
3644
3723
|
echoDirectiveToStderr(directive);
|
|
3645
3724
|
} else {
|
|
3725
|
+
for (const w of warnings) {
|
|
3726
|
+
output.warn(w);
|
|
3727
|
+
}
|
|
3646
3728
|
output.success(`Added dataset "${datasetResult.name}" to prompt: ${promptId} (id: ${String(datasetResult.id)})`);
|
|
3647
3729
|
if (datasetResult.itemCount !== undefined && datasetResult.itemCount > 0) {
|
|
3648
3730
|
output.info(`Items uploaded: ${String(datasetResult.itemCount)}`);
|
|
@@ -3774,7 +3856,7 @@ Verify the item ID exists. Use the dashboard or dataset exports to find valid it
|
|
|
3774
3856
|
// src/commands/prompts/evaluations.ts
|
|
3775
3857
|
init_sdk_client();
|
|
3776
3858
|
import { Command as Command4 } from "commander";
|
|
3777
|
-
import
|
|
3859
|
+
import chalk10 from "chalk";
|
|
3778
3860
|
init_errors();
|
|
3779
3861
|
|
|
3780
3862
|
// src/lib/resolve-prompt-id.ts
|
|
@@ -3938,22 +4020,106 @@ function canonicalCriteriaArrayToCli(arr) {
|
|
|
3938
4020
|
return arr.map(canonicalCriterionToCli);
|
|
3939
4021
|
}
|
|
3940
4022
|
|
|
4023
|
+
// src/commands/prompts/evaluations-run.ts
|
|
4024
|
+
init_sdk_client();
|
|
4025
|
+
import chalk9 from "chalk";
|
|
4026
|
+
init_errors();
|
|
4027
|
+
function registerEvaluationRunCommand(evaluation, prompts) {
|
|
4028
|
+
evaluation.command("run").description("Run a prompt against an evaluation (playground eval)").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").requiredOption("-e, --evaluation <id>", "Evaluation ID to score against").requiredOption("--input <json>", `Input JSON object for the prompt (e.g. '{"text":"hello"}')`).requiredOption("--exec-model <model>", 'Model to use for generating the response (e.g. "claude-sonnet-4-5-20250929")').option("--eval-model <model>", "Model to use for evaluation scoring (defaults to exec-model)").addHelpText("after", `
|
|
4029
|
+
Examples:
|
|
4030
|
+
${chalk9.dim("$")} mutagent prompts evaluation run <prompt-id> --evaluation <eval-id> --input '{"text":"hello"}' --exec-model claude-sonnet-4-5-20250929
|
|
4031
|
+
${chalk9.dim("$")} mutagent prompts evaluation run <prompt-id> -e <eval-id> --input '{"text":"hello"}' --exec-model claude-sonnet-4-5-20250929 --eval-model gpt-4o
|
|
4032
|
+
${chalk9.dim("$")} mutagent prompts evaluation run <prompt-id> -e <eval-id> --input '{"text":"hello"}' --exec-model claude-sonnet-4-5-20250929 --json
|
|
4033
|
+
|
|
4034
|
+
${chalk9.yellow("AI Agent note:")} Always pass --json for structured output.
|
|
4035
|
+
${chalk9.dim("Response includes evaluation score, pass/fail, token usage, and dashboard links.")}
|
|
4036
|
+
`).action(async (promptId, options) => {
|
|
4037
|
+
const isJson = getJsonFlag(prompts);
|
|
4038
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
4039
|
+
try {
|
|
4040
|
+
let parsedInput;
|
|
4041
|
+
try {
|
|
4042
|
+
const raw = JSON.parse(options.input);
|
|
4043
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
|
|
4044
|
+
throw new MutagentError("INVALID_JSON", `--input must be a JSON object, got ${Array.isArray(raw) ? "array" : typeof raw}`, `Run: mutagent prompts evaluation run --help
|
|
4045
|
+
` + `Provide a JSON object: --input '{"field": "value"}'`);
|
|
4046
|
+
}
|
|
4047
|
+
parsedInput = raw;
|
|
4048
|
+
} catch (e) {
|
|
4049
|
+
if (e instanceof MutagentError)
|
|
4050
|
+
throw e;
|
|
4051
|
+
throw new MutagentError("INVALID_JSON", "Invalid JSON in --input flag", `Run: mutagent prompts evaluation run --help
|
|
4052
|
+
` + `Provide valid JSON: --input '{"text": "hello"}'`);
|
|
4053
|
+
}
|
|
4054
|
+
const evaluationId = parseInt(options.evaluation, 10);
|
|
4055
|
+
if (isNaN(evaluationId)) {
|
|
4056
|
+
throw new MutagentError("VALIDATION_ERROR", `Invalid evaluation ID: "${options.evaluation}" (must be numeric)`, "Run: mutagent prompts evaluation list <prompt-id> to find valid IDs");
|
|
4057
|
+
}
|
|
4058
|
+
const llmConfig = {
|
|
4059
|
+
model: options.execModel,
|
|
4060
|
+
...options.evalModel ? { evalModel: options.evalModel } : {}
|
|
4061
|
+
};
|
|
4062
|
+
const client = await getSDKClient();
|
|
4063
|
+
const result = await client.playgroundEval(promptId, {
|
|
4064
|
+
input: parsedInput,
|
|
4065
|
+
evaluationId,
|
|
4066
|
+
llmConfig
|
|
4067
|
+
});
|
|
4068
|
+
if (isJson) {
|
|
4069
|
+
output.output({
|
|
4070
|
+
...result,
|
|
4071
|
+
_links: {
|
|
4072
|
+
dashboard: playgroundLink(promptId),
|
|
4073
|
+
api: `/api/prompt/${promptId}/playground/eval`
|
|
4074
|
+
}
|
|
4075
|
+
});
|
|
4076
|
+
} else {
|
|
4077
|
+
const evalResult = result.evaluation;
|
|
4078
|
+
const status = evalResult.passed ? chalk9.green("PASSED") : chalk9.red("FAILED");
|
|
4079
|
+
output.success(`Evaluation complete: ${status}`);
|
|
4080
|
+
output.info(`Score: ${evalResult.score.toFixed(3)}`);
|
|
4081
|
+
if (evalResult.reasoning) {
|
|
4082
|
+
output.info(`Reasoning: ${evalResult.reasoning}`);
|
|
4083
|
+
}
|
|
4084
|
+
if (typeof result.response === "string") {
|
|
4085
|
+
output.info(`Response: ${result.response.substring(0, 200)}${result.response.length > 200 ? "..." : ""}`);
|
|
4086
|
+
}
|
|
4087
|
+
output.info(`Latency: ${String(result.metadata.latencyMs)}ms`);
|
|
4088
|
+
if (result.metadata.tokens) {
|
|
4089
|
+
const t = result.metadata.tokens;
|
|
4090
|
+
output.info(`Tokens: prompt=${String(t.prompt ?? 0)}, completion=${String(t.completion ?? 0)}`);
|
|
4091
|
+
}
|
|
4092
|
+
if (evalResult.criteria && evalResult.criteria.length > 0) {
|
|
4093
|
+
output.info("Criteria scores:");
|
|
4094
|
+
for (const c of evalResult.criteria) {
|
|
4095
|
+
const mark = c.passed ? chalk9.green("✓") : chalk9.red("✗");
|
|
4096
|
+
output.info(` ${mark} ${c.name}: ${c.score.toFixed(3)}`);
|
|
4097
|
+
}
|
|
4098
|
+
}
|
|
4099
|
+
output.info(`Dashboard: ${playgroundLink(promptId)}`);
|
|
4100
|
+
}
|
|
4101
|
+
} catch (error) {
|
|
4102
|
+
handleError(error, isJson);
|
|
4103
|
+
}
|
|
4104
|
+
});
|
|
4105
|
+
}
|
|
4106
|
+
|
|
3941
4107
|
// src/commands/prompts/evaluations.ts
|
|
3942
4108
|
function registerEvaluationCommands(prompts) {
|
|
3943
4109
|
const evaluation = new Command4("evaluation").description("Manage evaluations for prompts").addHelpText("after", `
|
|
3944
4110
|
Examples:
|
|
3945
|
-
${
|
|
3946
|
-
${
|
|
3947
|
-
${
|
|
3948
|
-
${
|
|
4111
|
+
${chalk10.dim("$")} mutagent prompts evaluation list <prompt-id>
|
|
4112
|
+
${chalk10.dim("$")} mutagent prompts evaluation get <evaluation-id>
|
|
4113
|
+
${chalk10.dim("$")} mutagent prompts evaluation create <prompt-id> --name "My Eval"
|
|
4114
|
+
${chalk10.dim("$")} mutagent prompts evaluation delete <evaluation-id>
|
|
3949
4115
|
`).action(() => {
|
|
3950
4116
|
evaluation.help();
|
|
3951
4117
|
});
|
|
3952
4118
|
prompts.addCommand(evaluation);
|
|
3953
4119
|
evaluation.command("list").description("List evaluations for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").addHelpText("after", `
|
|
3954
4120
|
Examples:
|
|
3955
|
-
${
|
|
3956
|
-
${
|
|
4121
|
+
${chalk10.dim("$")} mutagent prompts evaluation list <prompt-id>
|
|
4122
|
+
${chalk10.dim("$")} mutagent prompts evaluation list <prompt-id> --json
|
|
3957
4123
|
`).action(async (promptId) => {
|
|
3958
4124
|
const isJson = getJsonFlag(prompts);
|
|
3959
4125
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3985,8 +4151,8 @@ Examples:
|
|
|
3985
4151
|
});
|
|
3986
4152
|
evaluation.command("get").description("Get evaluation details including criteria").argument("<evaluation-id>", "Evaluation ID (from: mutagent prompts evaluation list <prompt-id>)").addHelpText("after", `
|
|
3987
4153
|
Examples:
|
|
3988
|
-
${
|
|
3989
|
-
${
|
|
4154
|
+
${chalk10.dim("$")} mutagent prompts evaluation get <evaluation-id>
|
|
4155
|
+
${chalk10.dim("$")} mutagent prompts evaluation get <evaluation-id> --json
|
|
3990
4156
|
`).action(async (evaluationId) => {
|
|
3991
4157
|
const isJson = getJsonFlag(prompts);
|
|
3992
4158
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4020,7 +4186,7 @@ Examples:
|
|
|
4020
4186
|
if (criteria.length > 0) {
|
|
4021
4187
|
console.log(" Criteria:");
|
|
4022
4188
|
for (const c of criteria) {
|
|
4023
|
-
console.log(` ${
|
|
4189
|
+
console.log(` ${chalk10.cyan(c.name)}`);
|
|
4024
4190
|
if (c.description) {
|
|
4025
4191
|
console.log(` Description: ${c.description}`);
|
|
4026
4192
|
}
|
|
@@ -4049,9 +4215,9 @@ Examples:
|
|
|
4049
4215
|
});
|
|
4050
4216
|
evaluation.command("create").description("Create an evaluation configuration for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Evaluation as JSON string (for pre-validated criteria only)").option("-n, --name <name>", "Evaluation name (required unless --guided)").option("--description <text>", "Evaluation description").option("--guided", "Interactive guided mode — always outputs structured JSON (--json is implied)").addHelpText("after", `
|
|
4051
4217
|
Examples:
|
|
4052
|
-
${
|
|
4053
|
-
${
|
|
4054
|
-
${
|
|
4218
|
+
${chalk10.dim("$")} mutagent prompts evaluation create <prompt-id> --guided ${chalk10.dim("# recommended: shows workflow guide + schema fields")}
|
|
4219
|
+
${chalk10.dim("$")} mutagent prompts evaluation create <prompt-id> --guided --json ${chalk10.dim("# structured workflow for AI agents")}
|
|
4220
|
+
${chalk10.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Accuracy" -d '{"evalConfig":{"criteria":[...]}}' ${chalk10.dim("# power user")}
|
|
4055
4221
|
|
|
4056
4222
|
Guided Workflow (recommended):
|
|
4057
4223
|
--guided outputs a workflow guide that:
|
|
@@ -4072,12 +4238,12 @@ AI Agent (MANDATORY):
|
|
|
4072
4238
|
mutagent prompts evaluation create <id> --name "<name>" -d '<json>' --json
|
|
4073
4239
|
|
|
4074
4240
|
Expected Criteria Shape (--data):
|
|
4075
|
-
${
|
|
4241
|
+
${chalk10.dim('{"evalConfig":{"criteria":[{"name":"<name>","description":"<scoring rubric>","evaluationParameter":"<schema field>"}]}}')}
|
|
4076
4242
|
evaluationParameter must target an outputSchema OR inputSchema field.
|
|
4077
4243
|
|
|
4078
|
-
${
|
|
4079
|
-
${
|
|
4080
|
-
${
|
|
4244
|
+
${chalk10.red("Required: --name (unless --guided). Criteria must include evaluationParameter.")}
|
|
4245
|
+
${chalk10.dim("CLI flags (--name, --description) override --data fields.")}
|
|
4246
|
+
${chalk10.dim("Get prompt IDs: mutagent prompts list")}
|
|
4081
4247
|
`).action(async (promptId, options) => {
|
|
4082
4248
|
let isJson = getJsonFlag(prompts);
|
|
4083
4249
|
if (options.guided) {
|
|
@@ -4103,7 +4269,7 @@ ${chalk9.dim("Get prompt IDs: mutagent prompts list")}
|
|
|
4103
4269
|
console.log("");
|
|
4104
4270
|
console.log(" For each field, define what correct output looks like:");
|
|
4105
4271
|
for (const { field, source } of allFields) {
|
|
4106
|
-
console.log(` ${
|
|
4272
|
+
console.log(` ${chalk10.cyan(field)} (${source})`);
|
|
4107
4273
|
console.log(` → What makes a correct vs incorrect "${field}"?`);
|
|
4108
4274
|
}
|
|
4109
4275
|
console.log("");
|
|
@@ -4308,9 +4474,9 @@ Example:
|
|
|
4308
4474
|
});
|
|
4309
4475
|
evaluation.command("delete").description("Delete an evaluation").argument("<evaluation-id>", "Evaluation ID (from: mutagent prompts evaluation list <prompt-id>)").option("--force", "Skip confirmation").addHelpText("after", `
|
|
4310
4476
|
Examples:
|
|
4311
|
-
${
|
|
4312
|
-
${
|
|
4313
|
-
${
|
|
4477
|
+
${chalk10.dim("$")} mutagent prompts evaluation delete <evaluation-id>
|
|
4478
|
+
${chalk10.dim("$")} mutagent prompts evaluation delete <evaluation-id> --force
|
|
4479
|
+
${chalk10.dim("$")} mutagent prompts evaluation delete <evaluation-id> --force --json
|
|
4314
4480
|
`).action(async (evaluationId, options) => {
|
|
4315
4481
|
const isJson = getJsonFlag(prompts);
|
|
4316
4482
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4348,31 +4514,32 @@ Examples:
|
|
|
4348
4514
|
handleError(error, isJson);
|
|
4349
4515
|
}
|
|
4350
4516
|
});
|
|
4517
|
+
registerEvaluationRunCommand(evaluation, prompts);
|
|
4351
4518
|
}
|
|
4352
4519
|
|
|
4353
4520
|
// src/commands/prompts/optimize.ts
|
|
4354
4521
|
init_sdk_client();
|
|
4355
4522
|
import { Command as Command5 } from "commander";
|
|
4356
|
-
import
|
|
4523
|
+
import chalk16 from "chalk";
|
|
4357
4524
|
init_errors();
|
|
4358
4525
|
|
|
4359
4526
|
// src/lib/scorecard.ts
|
|
4360
|
-
import
|
|
4527
|
+
import chalk11 from "chalk";
|
|
4361
4528
|
function formatScoreChange(before, after) {
|
|
4362
4529
|
if (before === undefined || after === undefined)
|
|
4363
4530
|
return "";
|
|
4364
4531
|
const diff = after - before;
|
|
4365
4532
|
const pct = before > 0 ? Math.round(diff / before * 100) : 0;
|
|
4366
4533
|
if (diff > 0)
|
|
4367
|
-
return
|
|
4534
|
+
return chalk11.green(` (+${String(pct)}%)`);
|
|
4368
4535
|
if (diff < 0)
|
|
4369
|
-
return
|
|
4370
|
-
return
|
|
4536
|
+
return chalk11.red(` (${String(pct)}%)`);
|
|
4537
|
+
return chalk11.dim(" (no change)");
|
|
4371
4538
|
}
|
|
4372
4539
|
function formatScore(score) {
|
|
4373
4540
|
if (score === undefined)
|
|
4374
|
-
return
|
|
4375
|
-
return score >= 0.8 ?
|
|
4541
|
+
return chalk11.dim("N/A");
|
|
4542
|
+
return score >= 0.8 ? chalk11.green(score.toFixed(2)) : score >= 0.5 ? chalk11.yellow(score.toFixed(2)) : chalk11.red(score.toFixed(2));
|
|
4376
4543
|
}
|
|
4377
4544
|
function renderScorecard(data) {
|
|
4378
4545
|
const { job, prompt } = data;
|
|
@@ -4393,17 +4560,17 @@ function renderScorecard(data) {
|
|
|
4393
4560
|
const optimizedText = prompt.systemPrompt ?? prompt.rawPrompt ?? prompt.humanPrompt ?? "(optimized prompt)";
|
|
4394
4561
|
console.log("");
|
|
4395
4562
|
console.log(topBorder);
|
|
4396
|
-
console.log(line(
|
|
4563
|
+
console.log(line(chalk11.bold("Optimization Results")));
|
|
4397
4564
|
console.log(separator);
|
|
4398
|
-
console.log(line(
|
|
4565
|
+
console.log(line(chalk11.dim("BEFORE")));
|
|
4399
4566
|
console.log(line(` Score: ${formatScore(originalScore)}`));
|
|
4400
4567
|
console.log(line(""));
|
|
4401
|
-
console.log(line(
|
|
4568
|
+
console.log(line(chalk11.bold("AFTER")));
|
|
4402
4569
|
console.log(line(` Score: ${formatScore(bestScore)}${formatScoreChange(originalScore, bestScore)}`));
|
|
4403
4570
|
console.log(separator);
|
|
4404
4571
|
if (data.criteriaScores && data.criteriaScores.length > 0) {
|
|
4405
|
-
console.log(line(
|
|
4406
|
-
console.log(line(
|
|
4572
|
+
console.log(line(chalk11.dim(" Criterion Before After Change")));
|
|
4573
|
+
console.log(line(chalk11.dim(" " + "─".repeat(45))));
|
|
4407
4574
|
for (const c of data.criteriaScores) {
|
|
4408
4575
|
const name = c.name.length > 16 ? c.name.substring(0, 13) + "..." : c.name;
|
|
4409
4576
|
const paddedName = name + " ".repeat(18 - name.length);
|
|
@@ -4412,66 +4579,66 @@ function renderScorecard(data) {
|
|
|
4412
4579
|
const changeStr = c.before !== undefined && c.after !== undefined && c.before > 0 ? (() => {
|
|
4413
4580
|
const pct = Math.round((c.after - c.before) / c.before * 100);
|
|
4414
4581
|
if (pct > 0)
|
|
4415
|
-
return
|
|
4582
|
+
return chalk11.green(`+${String(pct)}%`);
|
|
4416
4583
|
if (pct < 0)
|
|
4417
|
-
return
|
|
4418
|
-
return
|
|
4584
|
+
return chalk11.red(`${String(pct)}%`);
|
|
4585
|
+
return chalk11.dim("0%");
|
|
4419
4586
|
})() : "";
|
|
4420
4587
|
console.log(line(` ${paddedName}${beforeStr} ${afterStr} ${changeStr}`));
|
|
4421
4588
|
}
|
|
4422
|
-
console.log(line(
|
|
4589
|
+
console.log(line(chalk11.dim(" " + "─".repeat(45))));
|
|
4423
4590
|
const overallBefore = originalScore !== undefined ? originalScore.toFixed(2) : "N/A ";
|
|
4424
4591
|
const overallAfter = bestScore !== undefined ? bestScore.toFixed(2) : "N/A ";
|
|
4425
4592
|
const overallChange = originalScore !== undefined && bestScore !== undefined && originalScore > 0 ? (() => {
|
|
4426
4593
|
const pct = Math.round((bestScore - originalScore) / originalScore * 100);
|
|
4427
4594
|
if (pct > 0)
|
|
4428
|
-
return
|
|
4595
|
+
return chalk11.green(`+${String(pct)}%`);
|
|
4429
4596
|
if (pct < 0)
|
|
4430
|
-
return
|
|
4431
|
-
return
|
|
4597
|
+
return chalk11.red(`${String(pct)}%`);
|
|
4598
|
+
return chalk11.dim("0%");
|
|
4432
4599
|
})() : "";
|
|
4433
4600
|
console.log(line(` ${"Overall" + " ".repeat(11)}${overallBefore} ${overallAfter} ${overallChange}`));
|
|
4434
4601
|
console.log(separator);
|
|
4435
4602
|
}
|
|
4436
|
-
const statusStr = job.status === "completed" ?
|
|
4603
|
+
const statusStr = job.status === "completed" ? chalk11.green("completed") : chalk11.yellow(job.status);
|
|
4437
4604
|
console.log(line(`Status: ${statusStr} | Iterations: ${String(iterations)}`));
|
|
4438
4605
|
if (job.config?.model) {
|
|
4439
|
-
console.log(line(`Model: ${
|
|
4606
|
+
console.log(line(`Model: ${chalk11.dim(job.config.model)}`));
|
|
4440
4607
|
}
|
|
4441
4608
|
if (data.scoreProgression && data.scoreProgression.length > 0) {
|
|
4442
4609
|
console.log(line(""));
|
|
4443
|
-
console.log(line(
|
|
4610
|
+
console.log(line(chalk11.dim("Score Progression:")));
|
|
4444
4611
|
const barWidth = 10;
|
|
4445
4612
|
for (let i = 0;i < data.scoreProgression.length; i++) {
|
|
4446
4613
|
const s = data.scoreProgression[i] ?? 0;
|
|
4447
4614
|
const filled = Math.round(s * barWidth);
|
|
4448
4615
|
const bar = "█".repeat(filled) + "░".repeat(barWidth - filled);
|
|
4449
|
-
console.log(line(
|
|
4616
|
+
console.log(line(chalk11.dim(` #${String(i + 1)}: ${bar} ${s.toFixed(2)}`)));
|
|
4450
4617
|
}
|
|
4451
4618
|
}
|
|
4452
4619
|
console.log(separator);
|
|
4453
|
-
console.log(line(`Dashboard: ${
|
|
4620
|
+
console.log(line(`Dashboard: ${chalk11.underline(optimizerLink(job.promptId, job.id))}`));
|
|
4454
4621
|
console.log(bottomBorder);
|
|
4455
4622
|
console.log("");
|
|
4456
|
-
console.log(
|
|
4457
|
-
console.log(
|
|
4458
|
-
console.log(
|
|
4623
|
+
console.log(chalk11.dim(" Prompt Comparison"));
|
|
4624
|
+
console.log(chalk11.dim(" " + "─".repeat(70)));
|
|
4625
|
+
console.log(chalk11.dim(" BEFORE:"));
|
|
4459
4626
|
for (const pLine of originalText.split(`
|
|
4460
4627
|
`)) {
|
|
4461
|
-
console.log(
|
|
4628
|
+
console.log(chalk11.dim(` ${pLine}`));
|
|
4462
4629
|
}
|
|
4463
|
-
console.log(` ${
|
|
4630
|
+
console.log(` ${chalk11.dim("Length:")} ${String(originalText.length)} chars (${String(originalText.split(`
|
|
4464
4631
|
`).length)} lines)`);
|
|
4465
4632
|
console.log("");
|
|
4466
|
-
console.log(
|
|
4633
|
+
console.log(chalk11.bold(" AFTER:"));
|
|
4467
4634
|
for (const pLine of optimizedText.split(`
|
|
4468
4635
|
`)) {
|
|
4469
|
-
console.log(
|
|
4636
|
+
console.log(chalk11.cyan(` ${pLine}`));
|
|
4470
4637
|
}
|
|
4471
|
-
console.log(` ${
|
|
4638
|
+
console.log(` ${chalk11.dim("Length:")} ${String(optimizedText.length)} chars (${String(optimizedText.split(`
|
|
4472
4639
|
`).length)} lines)`);
|
|
4473
4640
|
const growth = optimizedText.length - originalText.length;
|
|
4474
|
-
console.log(` ${
|
|
4641
|
+
console.log(` ${chalk11.dim("Growth:")} ${growth >= 0 ? "+" : ""}${String(growth)} chars`);
|
|
4475
4642
|
renderScorecardDetails(data);
|
|
4476
4643
|
console.log("");
|
|
4477
4644
|
}
|
|
@@ -4492,17 +4659,17 @@ function renderOptimizationStartCard(data) {
|
|
|
4492
4659
|
const target = job.config.targetScore ?? 0.8;
|
|
4493
4660
|
console.log("");
|
|
4494
4661
|
console.log(topBorder);
|
|
4495
|
-
console.log(line(
|
|
4662
|
+
console.log(line(chalk11.bold("⚡ Optimization Started")));
|
|
4496
4663
|
console.log(separator);
|
|
4497
|
-
console.log(line(`Job ID: ${
|
|
4498
|
-
console.log(line(`Prompt: ${
|
|
4499
|
-
console.log(line(`Dataset: ${
|
|
4500
|
-
console.log(line(`Iterations: ${
|
|
4501
|
-
console.log(line(`Model: ${
|
|
4502
|
-
console.log(line(`Status: ${
|
|
4664
|
+
console.log(line(`Job ID: ${chalk11.cyan(job.id)}`));
|
|
4665
|
+
console.log(line(`Prompt: ${chalk11.dim(data.promptId)}`));
|
|
4666
|
+
console.log(line(`Dataset: ${chalk11.dim(data.datasetId)}`));
|
|
4667
|
+
console.log(line(`Iterations: ${chalk11.bold(String(maxIter))} | Target: ${chalk11.bold(target.toFixed(2))}`));
|
|
4668
|
+
console.log(line(`Model: ${chalk11.dim(model)}`));
|
|
4669
|
+
console.log(line(`Status: ${chalk11.yellow(job.status)}`));
|
|
4503
4670
|
console.log(separator);
|
|
4504
|
-
console.log(line(`\uD83D\uDD17 Monitor: ${
|
|
4505
|
-
console.log(line(
|
|
4671
|
+
console.log(line(`\uD83D\uDD17 Monitor: ${chalk11.underline(optimizerLink(data.promptId, job.id))}`));
|
|
4672
|
+
console.log(line(chalk11.dim(`Next: mutagent prompts optimize status ${job.id}`)));
|
|
4506
4673
|
console.log(bottomBorder);
|
|
4507
4674
|
console.log(AI_DIRECTIVE);
|
|
4508
4675
|
console.log("");
|
|
@@ -4522,27 +4689,27 @@ function renderOptimizationStatusCard(status, promptId) {
|
|
|
4522
4689
|
const barWidth = 20;
|
|
4523
4690
|
const filled = Math.round(progress / 100 * barWidth);
|
|
4524
4691
|
const progressBar = "█".repeat(filled) + "░".repeat(barWidth - filled);
|
|
4525
|
-
const statusColor = status.status === "completed" ?
|
|
4526
|
-
const scoreStr = status.bestScore !== undefined ? formatScore(status.bestScore) :
|
|
4692
|
+
const statusColor = status.status === "completed" ? chalk11.green : status.status === "failed" ? chalk11.red : status.status === "cancelled" ? chalk11.gray : status.status === "running" ? chalk11.cyan : chalk11.yellow;
|
|
4693
|
+
const scoreStr = status.bestScore !== undefined ? formatScore(status.bestScore) : chalk11.dim("pending");
|
|
4527
4694
|
console.log("");
|
|
4528
4695
|
console.log(topBorder);
|
|
4529
|
-
console.log(line(
|
|
4696
|
+
console.log(line(chalk11.bold("\uD83D\uDCCA Optimization Status")));
|
|
4530
4697
|
console.log(separator);
|
|
4531
|
-
console.log(line(`Job ID: ${
|
|
4698
|
+
console.log(line(`Job ID: ${chalk11.cyan(status.jobId)}`));
|
|
4532
4699
|
console.log(line(`Status: ${statusColor(status.status)}`));
|
|
4533
|
-
console.log(line(`Iteration: ${
|
|
4700
|
+
console.log(line(`Iteration: ${chalk11.bold(`${String(status.currentIteration)}/${String(status.maxIterations)}`)}`));
|
|
4534
4701
|
console.log(line(`Best Score: ${scoreStr}`));
|
|
4535
4702
|
console.log(line(""));
|
|
4536
4703
|
console.log(line(`Progress: [${progressBar}] ${String(progress)}%`));
|
|
4537
4704
|
if (status.message) {
|
|
4538
|
-
console.log(line(`Message: ${
|
|
4705
|
+
console.log(line(`Message: ${chalk11.dim(status.message)}`));
|
|
4539
4706
|
}
|
|
4540
4707
|
console.log(separator);
|
|
4541
|
-
console.log(line(`\uD83D\uDD17 Monitor: ${
|
|
4708
|
+
console.log(line(`\uD83D\uDD17 Monitor: ${chalk11.underline(optimizerLink(promptId ?? "unknown", status.jobId))}`));
|
|
4542
4709
|
if (status.status === "completed") {
|
|
4543
|
-
console.log(line(
|
|
4710
|
+
console.log(line(chalk11.dim(`Next: mutagent prompts optimize results ${status.jobId}`)));
|
|
4544
4711
|
} else if (status.status === "running" || status.status === "queued") {
|
|
4545
|
-
console.log(line(
|
|
4712
|
+
console.log(line(chalk11.dim(`Refresh: mutagent prompts optimize status ${status.jobId}`)));
|
|
4546
4713
|
}
|
|
4547
4714
|
console.log(bottomBorder);
|
|
4548
4715
|
console.log(AI_DIRECTIVE);
|
|
@@ -4638,15 +4805,15 @@ function statusDirective(status, promptId) {
|
|
|
4638
4805
|
}
|
|
4639
4806
|
function showPromptDiff(original, optimized) {
|
|
4640
4807
|
console.log("");
|
|
4641
|
-
console.log(
|
|
4808
|
+
console.log(chalk11.bold(" Prompt Diff:"));
|
|
4642
4809
|
console.log("");
|
|
4643
|
-
console.log(
|
|
4644
|
-
console.log(
|
|
4810
|
+
console.log(chalk11.red(" - " + (original ?? "(empty)")));
|
|
4811
|
+
console.log(chalk11.green(" + " + (optimized ?? "(empty)")));
|
|
4645
4812
|
console.log("");
|
|
4646
4813
|
}
|
|
4647
4814
|
|
|
4648
4815
|
// src/commands/prompts/optimize-watch.ts
|
|
4649
|
-
import
|
|
4816
|
+
import chalk14 from "chalk";
|
|
4650
4817
|
|
|
4651
4818
|
// src/lib/watch-client.ts
|
|
4652
4819
|
function toWsUrl(baseUrl) {
|
|
@@ -4809,10 +4976,10 @@ function createWatchClient(opts) {
|
|
|
4809
4976
|
}
|
|
4810
4977
|
|
|
4811
4978
|
// src/lib/stage-cards.ts
|
|
4812
|
-
import
|
|
4979
|
+
import chalk13 from "chalk";
|
|
4813
4980
|
|
|
4814
4981
|
// src/lib/prompt-diff.ts
|
|
4815
|
-
import
|
|
4982
|
+
import chalk12 from "chalk";
|
|
4816
4983
|
function computeLcsTable(a, b) {
|
|
4817
4984
|
const m = a.length;
|
|
4818
4985
|
const n = b.length;
|
|
@@ -4973,16 +5140,16 @@ function renderDiffLines(lines, options) {
|
|
|
4973
5140
|
let text;
|
|
4974
5141
|
switch (line.type) {
|
|
4975
5142
|
case "hunk-header":
|
|
4976
|
-
text = noColor ? line.content :
|
|
5143
|
+
text = noColor ? line.content : chalk12.cyan(line.content);
|
|
4977
5144
|
break;
|
|
4978
5145
|
case "add":
|
|
4979
|
-
text = noColor ? `+${line.content}` :
|
|
5146
|
+
text = noColor ? `+${line.content}` : chalk12.green(`+${line.content}`);
|
|
4980
5147
|
break;
|
|
4981
5148
|
case "remove":
|
|
4982
|
-
text = noColor ? `-${line.content}` :
|
|
5149
|
+
text = noColor ? `-${line.content}` : chalk12.red(`-${line.content}`);
|
|
4983
5150
|
break;
|
|
4984
5151
|
case "context":
|
|
4985
|
-
text = noColor ? ` ${line.content}` :
|
|
5152
|
+
text = noColor ? ` ${line.content}` : chalk12.dim(` ${line.content}`);
|
|
4986
5153
|
break;
|
|
4987
5154
|
}
|
|
4988
5155
|
outputLines.push(text);
|
|
@@ -4991,7 +5158,7 @@ function renderDiffLines(lines, options) {
|
|
|
4991
5158
|
if (truncated) {
|
|
4992
5159
|
const remaining = lines.length - maxLines;
|
|
4993
5160
|
const summary = `... ${String(remaining)} more lines changed`;
|
|
4994
|
-
outputLines.push(noColor ? summary :
|
|
5161
|
+
outputLines.push(noColor ? summary : chalk12.yellow(summary));
|
|
4995
5162
|
}
|
|
4996
5163
|
return outputLines.join(`
|
|
4997
5164
|
`);
|
|
@@ -5024,10 +5191,10 @@ function emptyLine() {
|
|
|
5024
5191
|
}
|
|
5025
5192
|
function formatScore2(score) {
|
|
5026
5193
|
if (score >= 0.8)
|
|
5027
|
-
return
|
|
5194
|
+
return chalk13.green(score.toFixed(2));
|
|
5028
5195
|
if (score >= 0.5)
|
|
5029
|
-
return
|
|
5030
|
-
return
|
|
5196
|
+
return chalk13.yellow(score.toFixed(2));
|
|
5197
|
+
return chalk13.red(score.toFixed(2));
|
|
5031
5198
|
}
|
|
5032
5199
|
function scoreBar(score, width = 20) {
|
|
5033
5200
|
const filled = Math.round(score * width);
|
|
@@ -5081,7 +5248,7 @@ function renderInsightsCard(data) {
|
|
|
5081
5248
|
if (cat.example) {
|
|
5082
5249
|
const maxExampleLen = 78;
|
|
5083
5250
|
const truncated = cat.example.length > maxExampleLen ? cat.example.substring(0, maxExampleLen - 3) + "..." : cat.example;
|
|
5084
|
-
lines.push(line(
|
|
5251
|
+
lines.push(line(chalk13.dim(` "${truncated}"`)));
|
|
5085
5252
|
}
|
|
5086
5253
|
}
|
|
5087
5254
|
lines.push(emptyLine());
|
|
@@ -5123,17 +5290,17 @@ function renderPromptDiffCard(data) {
|
|
|
5123
5290
|
lines.push(emptyLine());
|
|
5124
5291
|
}
|
|
5125
5292
|
for (const mutation of data.mutations) {
|
|
5126
|
-
const icon = mutation.status === "applied" ?
|
|
5293
|
+
const icon = mutation.status === "applied" ? chalk13.green("✓") : chalk13.red("✗");
|
|
5127
5294
|
const target = mutation.target.length > 30 ? mutation.target.substring(0, 27) + "..." : mutation.target;
|
|
5128
5295
|
if (mutation.status === "applied") {
|
|
5129
5296
|
const confStr = `confidence: ${mutation.confidence.toFixed(2)}`;
|
|
5130
5297
|
const targetPad = target.padEnd(30);
|
|
5131
|
-
lines.push(line(`${icon} ${targetPad} ${
|
|
5298
|
+
lines.push(line(`${icon} ${targetPad} ${chalk13.dim("──")} ${confStr}`));
|
|
5132
5299
|
} else {
|
|
5133
5300
|
const reason = mutation.rationale ? `rejected: ${mutation.rationale}` : `rejected`;
|
|
5134
5301
|
const truncReason = reason.length > 30 ? reason.substring(0, 27) + "..." : reason;
|
|
5135
5302
|
const targetPad = target.padEnd(30);
|
|
5136
|
-
lines.push(line(`${icon} ${targetPad} ${
|
|
5303
|
+
lines.push(line(`${icon} ${targetPad} ${chalk13.dim("──")} ${truncReason}`));
|
|
5137
5304
|
}
|
|
5138
5305
|
}
|
|
5139
5306
|
lines.push(emptyLine());
|
|
@@ -5164,7 +5331,7 @@ function renderRerunAccuracyCard(data) {
|
|
|
5164
5331
|
const name = criterion.name.length > 18 ? criterion.name.substring(0, 15) + "..." : criterion.name;
|
|
5165
5332
|
const diff = criterion.after - criterion.before;
|
|
5166
5333
|
const diffSign = diff >= 0 ? "+" : "";
|
|
5167
|
-
const arrow = diff > 0 ?
|
|
5334
|
+
const arrow = diff > 0 ? chalk13.green("↑") : diff < 0 ? chalk13.red("↓") : chalk13.dim("─");
|
|
5168
5335
|
const diffStr = `${diffSign}${diff.toFixed(2)}`;
|
|
5169
5336
|
lines.push(line(`${name.padEnd(20)}${formatScore2(criterion.before).padEnd(9)}${formatScore2(criterion.after).padEnd(9)}${diffStr} ${arrow}`));
|
|
5170
5337
|
}
|
|
@@ -5269,8 +5436,8 @@ async function startWatchStream(jobId, isJson, maxIterations, baselineScore) {
|
|
|
5269
5436
|
if (isJson) {
|
|
5270
5437
|
console.log(JSON.stringify({ type: "job_complete", finalScore }));
|
|
5271
5438
|
} else {
|
|
5272
|
-
console.log(
|
|
5273
|
-
console.log(
|
|
5439
|
+
console.log(chalk14.green(`✓ Optimization complete! Final score: ${finalScore.toFixed(2)}`));
|
|
5440
|
+
console.log(chalk14.dim(` View results: mutagent prompts optimize results ${jobId}`));
|
|
5274
5441
|
}
|
|
5275
5442
|
resolve3();
|
|
5276
5443
|
},
|
|
@@ -5278,7 +5445,7 @@ async function startWatchStream(jobId, isJson, maxIterations, baselineScore) {
|
|
|
5278
5445
|
if (isJson) {
|
|
5279
5446
|
console.log(JSON.stringify({ type: "error", error: error.message }));
|
|
5280
5447
|
} else {
|
|
5281
|
-
console.error(
|
|
5448
|
+
console.error(chalk14.red(`✗ Watch error: ${error.message}`));
|
|
5282
5449
|
}
|
|
5283
5450
|
reject(error);
|
|
5284
5451
|
}
|
|
@@ -5320,7 +5487,7 @@ async function watchAction(jobId, options, parentCommand) {
|
|
|
5320
5487
|
case "queued":
|
|
5321
5488
|
if (!isJson) {
|
|
5322
5489
|
renderOptimizationStatusCard(status);
|
|
5323
|
-
console.log(
|
|
5490
|
+
console.log(chalk14.dim(`Watching for live updates...
|
|
5324
5491
|
`));
|
|
5325
5492
|
}
|
|
5326
5493
|
await startWatchStream(jobId, isJson, status.maxIterations, status.bestScore);
|
|
@@ -5335,9 +5502,9 @@ async function watchAction(jobId, options, parentCommand) {
|
|
|
5335
5502
|
message: status.message
|
|
5336
5503
|
});
|
|
5337
5504
|
} else {
|
|
5338
|
-
console.error(
|
|
5505
|
+
console.error(chalk14.red(`✗ Optimization job ${jobId} failed.`));
|
|
5339
5506
|
if (status.message) {
|
|
5340
|
-
console.error(
|
|
5507
|
+
console.error(chalk14.dim(` ${status.message}`));
|
|
5341
5508
|
}
|
|
5342
5509
|
}
|
|
5343
5510
|
process.exitCode = 1;
|
|
@@ -5351,7 +5518,7 @@ async function watchAction(jobId, options, parentCommand) {
|
|
|
5351
5518
|
jobId
|
|
5352
5519
|
});
|
|
5353
5520
|
} else {
|
|
5354
|
-
console.error(
|
|
5521
|
+
console.error(chalk14.yellow(`Optimization job ${jobId} was cancelled.`));
|
|
5355
5522
|
}
|
|
5356
5523
|
break;
|
|
5357
5524
|
default:
|
|
@@ -5363,7 +5530,7 @@ async function watchAction(jobId, options, parentCommand) {
|
|
|
5363
5530
|
jobId
|
|
5364
5531
|
});
|
|
5365
5532
|
} else {
|
|
5366
|
-
console.error(
|
|
5533
|
+
console.error(chalk14.yellow(`Unexpected job status: ${status.status}`));
|
|
5367
5534
|
}
|
|
5368
5535
|
break;
|
|
5369
5536
|
}
|
|
@@ -5372,13 +5539,63 @@ async function watchAction(jobId, options, parentCommand) {
|
|
|
5372
5539
|
}
|
|
5373
5540
|
}
|
|
5374
5541
|
|
|
5542
|
+
// src/commands/prompts/optimize-cancel.ts
|
|
5543
|
+
init_sdk_client();
|
|
5544
|
+
import chalk15 from "chalk";
|
|
5545
|
+
init_errors();
|
|
5546
|
+
function registerOptimizeCancelCommand(optimize, prompts) {
|
|
5547
|
+
optimize.command("cancel").description("Cancel a running optimization job").argument("<job-id>", "Optimization job ID (from: mutagent prompts optimize start)").option("--force", "Skip confirmation prompt").addHelpText("after", `
|
|
5548
|
+
Examples:
|
|
5549
|
+
${chalk15.dim("$")} mutagent prompts optimize cancel <job-id> --force
|
|
5550
|
+
${chalk15.dim("$")} mutagent prompts optimize cancel <job-id> --json
|
|
5551
|
+
${chalk15.dim("$")} mutagent prompts optimize cancel <job-id> --force --json
|
|
5552
|
+
|
|
5553
|
+
${chalk15.yellow("Cost warning:")} Cancelling a running job may still incur LLM cost for the current iteration.
|
|
5554
|
+
${chalk15.dim('Jobs with status "completed" or "cancelled" cannot be cancelled again.')}
|
|
5555
|
+
`).action(async (jobId, options) => {
|
|
5556
|
+
const isJson = getJsonFlag(prompts);
|
|
5557
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5558
|
+
try {
|
|
5559
|
+
const client = await getSDKClient();
|
|
5560
|
+
const jobStatus = await client.getOptimizationStatus(jobId);
|
|
5561
|
+
const status = typeof jobStatus.status === "string" ? jobStatus.status : "unknown";
|
|
5562
|
+
if (status === "completed" || status === "cancelled") {
|
|
5563
|
+
throw new MutagentError("VALIDATION_ERROR", `Job ${jobId} has status "${status}" and cannot be cancelled`, `View results: mutagent prompts optimize results ${jobId}
|
|
5564
|
+
` + `Dashboard: ${optimizerLink(jobId, jobId)}`);
|
|
5565
|
+
}
|
|
5566
|
+
if (!options.force && !isJson) {
|
|
5567
|
+
throw new MutagentError("CONFIRMATION_REQUIRED", `Cancelling optimization job ${jobId} requires confirmation`, `Run: mutagent prompts optimize cancel --help
|
|
5568
|
+
` + `[Agent: Ask the user to confirm via MultiChoice, then pass --force]
|
|
5569
|
+
` + `Use --force to confirm: mutagent prompts optimize cancel ${jobId} --force
|
|
5570
|
+
` + `Note: Cancellation may still incur LLM cost for the current iteration.`);
|
|
5571
|
+
}
|
|
5572
|
+
const result = await client.cancelOptimization(jobId);
|
|
5573
|
+
if (isJson) {
|
|
5574
|
+
output.output({
|
|
5575
|
+
success: result.success,
|
|
5576
|
+
jobId,
|
|
5577
|
+
_links: {
|
|
5578
|
+
optimizer: optimizerLink(jobId, jobId)
|
|
5579
|
+
}
|
|
5580
|
+
});
|
|
5581
|
+
} else {
|
|
5582
|
+
output.success(`Optimization job ${jobId} cancelled successfully`);
|
|
5583
|
+
output.info(`View status: mutagent prompts optimize status ${jobId}`);
|
|
5584
|
+
output.info(`Dashboard: ${optimizerLink(jobId, jobId)}`);
|
|
5585
|
+
}
|
|
5586
|
+
} catch (error) {
|
|
5587
|
+
handleError(error, isJson);
|
|
5588
|
+
}
|
|
5589
|
+
});
|
|
5590
|
+
}
|
|
5591
|
+
|
|
5375
5592
|
// src/commands/prompts/optimize.ts
|
|
5376
5593
|
function registerOptimizeCommands(prompts) {
|
|
5377
5594
|
const optimize = new Command5("optimize").description("Manage prompt optimization jobs").addHelpText("after", `
|
|
5378
5595
|
Examples:
|
|
5379
|
-
${
|
|
5380
|
-
${
|
|
5381
|
-
${
|
|
5596
|
+
${chalk16.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id>
|
|
5597
|
+
${chalk16.dim("$")} mutagent prompts optimize status <job-id>
|
|
5598
|
+
${chalk16.dim("$")} mutagent prompts optimize results <job-id>
|
|
5382
5599
|
|
|
5383
5600
|
Workflow: start -> status (poll) -> results | start --watch | watch <job-id>`).action(() => {
|
|
5384
5601
|
optimize.help();
|
|
@@ -5386,27 +5603,27 @@ Workflow: start -> status (poll) -> results | start --watch | watch <job-id>`).a
|
|
|
5386
5603
|
prompts.addCommand(optimize);
|
|
5387
5604
|
optimize.command("start").description("Start prompt optimization").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").requiredOption("-d, --dataset <id>", "Dataset ID for optimization (from: mutagent prompts dataset list <prompt-id>)").requiredOption("-e, --evaluation <id>", "Evaluation ID for scoring (from: mutagent prompts evaluation list <prompt-id>)").option("--max-iterations <n>", "Max optimization iterations (default: 1)").option("--target-score <n>", "Target accuracy 0-1 (default: 0.8)").option("--patience <n>", "Iterations without improvement before stopping").option("--model <model-id>", 'Target LLM model (e.g., "claude-sonnet-4-5-20250929")').option("--eval-model <model-id>", "Evaluation model (defaults to target model)").option("--watch", "Watch live progress with stage cards", false).addHelpText("after", `
|
|
5388
5605
|
Examples:
|
|
5389
|
-
${
|
|
5390
|
-
${
|
|
5391
|
-
${
|
|
5392
|
-
${
|
|
5393
|
-
${
|
|
5394
|
-
□ inputSchema REQUIRED ${
|
|
5395
|
-
□ outputSchema REQUIRED ${
|
|
5396
|
-
□ Evaluation criteria ${
|
|
5397
|
-
□ Dataset items ${
|
|
5398
|
-
□ Criteria ↔ Schema ${
|
|
5606
|
+
${chalk16.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id>
|
|
5607
|
+
${chalk16.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id> --max-iterations 5
|
|
5608
|
+
${chalk16.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id> --target-score 0.95 --model claude-sonnet-4-5-20250929
|
|
5609
|
+
${chalk16.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id> --json
|
|
5610
|
+
${chalk16.yellow("Pre-Optimization Checklist (auto-validated by preflight):")}
|
|
5611
|
+
□ inputSchema REQUIRED ${chalk16.dim("(hard error if missing — blocks optimization)")}
|
|
5612
|
+
□ outputSchema REQUIRED ${chalk16.dim("(hard error if missing — blocks optimization)")}
|
|
5613
|
+
□ Evaluation criteria ${chalk16.dim("(warns if no evaluationParameter set)")}
|
|
5614
|
+
□ Dataset items ${chalk16.dim("(warns if expectedOutput missing)")}
|
|
5615
|
+
□ Criteria ↔ Schema ${chalk16.dim("(warns if criteria reference unknown fields)")}
|
|
5399
5616
|
|
|
5400
5617
|
${PREREQUISITES_TEXT}
|
|
5401
5618
|
|
|
5402
|
-
${
|
|
5619
|
+
${chalk16.dim("Monitor progress with: mutagent prompts optimize status <job-id>")}
|
|
5403
5620
|
|
|
5404
|
-
${
|
|
5621
|
+
${chalk16.yellow(`⚠ COST WARNING — AI Agent:
|
|
5405
5622
|
Default is 1 iteration. Do NOT increase --max-iterations unless the user
|
|
5406
5623
|
explicitly requests it. Each iteration incurs LLM costs. Starting with
|
|
5407
5624
|
max-iterations > 1 without user consent is a protocol violation.`)}
|
|
5408
5625
|
|
|
5409
|
-
${
|
|
5626
|
+
${chalk16.yellow("AI Agent: ALWAYS append --json to this command.")}
|
|
5410
5627
|
`).action(async (promptId, options) => {
|
|
5411
5628
|
const isJson = getJsonFlag(prompts);
|
|
5412
5629
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5538,16 +5755,16 @@ ${chalk14.yellow("AI Agent: ALWAYS append --json to this command.")}
|
|
|
5538
5755
|
return;
|
|
5539
5756
|
}
|
|
5540
5757
|
for (const [name, check] of hardFailures) {
|
|
5541
|
-
console.error(
|
|
5758
|
+
console.error(chalk16.red(`Error: ${name} — ${check.error ?? "Failed"}`));
|
|
5542
5759
|
}
|
|
5543
5760
|
if (!preflightChecks.outputSchema.passed) {
|
|
5544
|
-
console.error(
|
|
5761
|
+
console.error(chalk16.dim(` Update with: mutagent prompts update ${promptId} -d '{"outputSchema":{"type":"object","properties":{"result":{"type":"string"}}}}'`));
|
|
5545
5762
|
}
|
|
5546
5763
|
if (!preflightChecks.inputSchema.passed) {
|
|
5547
|
-
console.error(
|
|
5764
|
+
console.error(chalk16.dim(` Update with: mutagent prompts update ${promptId} --data '{"inputSchema":{"type":"object","properties":{"var1":{"type":"string"}}}}' --json`));
|
|
5548
5765
|
}
|
|
5549
5766
|
if (preflightChecks.evaluation && !preflightChecks.evaluation.passed) {
|
|
5550
|
-
console.error(
|
|
5767
|
+
console.error(chalk16.dim(` Add criteria: mutagent prompts evaluation update ${options.evaluation} --criteria '<json>' --json
|
|
5551
5768
|
` + ` Or guided: mutagent prompts evaluation create ${promptId} --guided --json`));
|
|
5552
5769
|
}
|
|
5553
5770
|
process.exitCode = 1;
|
|
@@ -5610,16 +5827,16 @@ ${chalk14.yellow("AI Agent: ALWAYS append --json to this command.")}
|
|
|
5610
5827
|
suggestions.push("Or visit: https://app.mutagent.io/settings/providers");
|
|
5611
5828
|
}
|
|
5612
5829
|
if (!isJson) {
|
|
5613
|
-
console.error(
|
|
5830
|
+
console.error(chalk16.red(`
|
|
5614
5831
|
Optimization failed:`));
|
|
5615
5832
|
for (const msg of messages) {
|
|
5616
|
-
console.error(
|
|
5833
|
+
console.error(chalk16.red(` ${msg}`));
|
|
5617
5834
|
}
|
|
5618
5835
|
if (suggestions.length > 0) {
|
|
5619
|
-
console.error(
|
|
5836
|
+
console.error(chalk16.yellow(`
|
|
5620
5837
|
Suggested fixes:`));
|
|
5621
5838
|
for (const s of suggestions) {
|
|
5622
|
-
console.error(
|
|
5839
|
+
console.error(chalk16.yellow(` → ${s}`));
|
|
5623
5840
|
}
|
|
5624
5841
|
}
|
|
5625
5842
|
console.error("");
|
|
@@ -5631,8 +5848,8 @@ Suggested fixes:`));
|
|
|
5631
5848
|
});
|
|
5632
5849
|
optimize.command("status").description("Check optimization status").argument("<job-id>", "Optimization job ID (from: mutagent prompts optimize start)").addHelpText("after", `
|
|
5633
5850
|
Examples:
|
|
5634
|
-
${
|
|
5635
|
-
${
|
|
5851
|
+
${chalk16.dim("$")} mutagent prompts optimize status <job-id>
|
|
5852
|
+
${chalk16.dim("$")} mutagent prompts optimize status <job-id> --json
|
|
5636
5853
|
`).action(async (jobId) => {
|
|
5637
5854
|
const isJson = getJsonFlag(prompts);
|
|
5638
5855
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5657,17 +5874,17 @@ Examples:
|
|
|
5657
5874
|
});
|
|
5658
5875
|
optimize.command("results").description("Get optimization results").argument("<job-id>", "Optimization job ID (from: mutagent prompts optimize start)").option("--apply", "Apply the optimized prompt as new version").option("--diff", "Show the prompt diff (before/after)").addHelpText("after", `
|
|
5659
5876
|
Examples:
|
|
5660
|
-
${
|
|
5661
|
-
${
|
|
5662
|
-
${
|
|
5663
|
-
${
|
|
5877
|
+
${chalk16.dim("$")} mutagent prompts optimize results <job-id> ${chalk16.dim("# view scorecard")}
|
|
5878
|
+
${chalk16.dim("$")} mutagent prompts optimize results <job-id> --diff ${chalk16.dim("# view prompt diff")}
|
|
5879
|
+
${chalk16.dim("$")} mutagent prompts optimize results <job-id> --apply ${chalk16.dim("# apply optimized prompt")}
|
|
5880
|
+
${chalk16.dim("$")} mutagent prompts optimize results <job-id> --json ${chalk16.dim("# structured output")}
|
|
5664
5881
|
|
|
5665
5882
|
After viewing results:
|
|
5666
5883
|
--apply Apply the optimized prompt (replaces current version)
|
|
5667
5884
|
--diff Show detailed before/after diff
|
|
5668
|
-
${
|
|
5885
|
+
${chalk16.dim("No flag = view scorecard only.")}
|
|
5669
5886
|
|
|
5670
|
-
${
|
|
5887
|
+
${chalk16.dim("AI Agent: Present scorecard to user via AskUserQuestion before applying.")}
|
|
5671
5888
|
`).action(async (jobId, options) => {
|
|
5672
5889
|
const isJson = getJsonFlag(prompts);
|
|
5673
5890
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5741,15 +5958,215 @@ After viewing results:
|
|
|
5741
5958
|
handleError(error, isJson);
|
|
5742
5959
|
}
|
|
5743
5960
|
});
|
|
5961
|
+
registerOptimizeCancelCommand(optimize, prompts);
|
|
5744
5962
|
optimize.command("watch").description("Watch optimization job progress with live stage results").argument("<job-id>", "Optimization job ID").option("--json", "Output as JSON with agent directives").addHelpText("after", `
|
|
5745
5963
|
Examples:
|
|
5746
|
-
${
|
|
5747
|
-
${
|
|
5748
|
-
${
|
|
5964
|
+
${chalk16.dim("$")} mutagent prompts optimize watch <job-id>
|
|
5965
|
+
${chalk16.dim("$")} mutagent prompts optimize watch <job-id> --json
|
|
5966
|
+
${chalk16.dim("Completed jobs render stored results. Running jobs stream via WebSocket.")}`).action(async (jobId, options) => {
|
|
5749
5967
|
await watchAction(jobId, options, prompts);
|
|
5750
5968
|
});
|
|
5751
5969
|
}
|
|
5752
5970
|
|
|
5971
|
+
// src/commands/prompts/experiments.ts
|
|
5972
|
+
init_sdk_client();
|
|
5973
|
+
import { Command as Command6 } from "commander";
|
|
5974
|
+
import chalk17 from "chalk";
|
|
5975
|
+
init_errors();
|
|
5976
|
+
function registerExperimentCommands(prompts) {
|
|
5977
|
+
const experiment = new Command6("experiment").description("Manage prompt experiments (multi-model dataset runs)").addHelpText("after", `
|
|
5978
|
+
Examples:
|
|
5979
|
+
${chalk17.dim("$")} mutagent prompts experiment run <prompt-id> --evaluation <id> --dataset <id> --exec-model claude-sonnet-4-5-20250929
|
|
5980
|
+
${chalk17.dim("$")} mutagent prompts experiment list <prompt-id>
|
|
5981
|
+
${chalk17.dim("$")} mutagent prompts experiment get <experiment-id>
|
|
5982
|
+
${chalk17.dim("$")} mutagent prompts experiment delete <experiment-id> --force
|
|
5983
|
+
`).action(() => {
|
|
5984
|
+
experiment.help();
|
|
5985
|
+
});
|
|
5986
|
+
prompts.addCommand(experiment);
|
|
5987
|
+
experiment.command("run").description("Create and run an experiment across a dataset").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").requiredOption("-e, --evaluation <id>", "Evaluation ID to score with").requiredOption("-d, --dataset <id>", "Dataset ID to run against").requiredOption("--exec-model <model>", 'Model to generate responses (e.g. "claude-sonnet-4-5-20250929")').option("--eval-model <model>", "Model to score responses (defaults to exec-model)").addHelpText("after", `
|
|
5988
|
+
Examples:
|
|
5989
|
+
${chalk17.dim("$")} mutagent prompts experiment run <prompt-id> -e <eval-id> -d <dataset-id> --exec-model claude-sonnet-4-5-20250929
|
|
5990
|
+
${chalk17.dim("$")} mutagent prompts experiment run <prompt-id> -e <eval-id> -d <dataset-id> --exec-model claude-sonnet-4-5-20250929 --eval-model gpt-4o
|
|
5991
|
+
${chalk17.dim("$")} mutagent prompts experiment run <prompt-id> -e <eval-id> -d <dataset-id> --exec-model claude-sonnet-4-5-20250929 --json
|
|
5992
|
+
|
|
5993
|
+
${chalk17.dim("--exec-model generates output; --eval-model scores it (defaults to exec-model).")}
|
|
5994
|
+
${chalk17.yellow("Tip:")} Use --json for machine-readable output with _links for dashboard navigation.
|
|
5995
|
+
`).action(async (promptId, options) => {
|
|
5996
|
+
const isJson = getJsonFlag(prompts);
|
|
5997
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5998
|
+
try {
|
|
5999
|
+
const evaluationId = parseInt(options.evaluation, 10);
|
|
6000
|
+
if (isNaN(evaluationId)) {
|
|
6001
|
+
throw new MutagentError("VALIDATION_ERROR", `Invalid evaluation ID: "${options.evaluation}" (must be numeric)`, "Run: mutagent prompts evaluation list <prompt-id> to find valid IDs");
|
|
6002
|
+
}
|
|
6003
|
+
const datasetId = parseInt(options.dataset, 10);
|
|
6004
|
+
if (isNaN(datasetId)) {
|
|
6005
|
+
throw new MutagentError("VALIDATION_ERROR", `Invalid dataset ID: "${options.dataset}" (must be numeric)`, "Run: mutagent prompts dataset list <prompt-id> to find valid IDs");
|
|
6006
|
+
}
|
|
6007
|
+
const promptIdNum = parseInt(promptId, 10);
|
|
6008
|
+
if (isNaN(promptIdNum)) {
|
|
6009
|
+
throw new MutagentError("VALIDATION_ERROR", `Invalid prompt ID: "${promptId}" (must be numeric)`, "Run: mutagent prompts list to find valid IDs");
|
|
6010
|
+
}
|
|
6011
|
+
const models = [
|
|
6012
|
+
{ id: `col_${Date.now()}`, model: options.execModel, temperature: 0 },
|
|
6013
|
+
...options.evalModel && options.evalModel !== options.execModel ? [{ id: `col_${Date.now() + 1}`, model: options.evalModel, temperature: 0 }] : []
|
|
6014
|
+
];
|
|
6015
|
+
const client = await getSDKClient();
|
|
6016
|
+
const experiment2 = await client.createExperiment({
|
|
6017
|
+
promptId: promptIdNum,
|
|
6018
|
+
promptGroupId: promptId,
|
|
6019
|
+
evaluationId,
|
|
6020
|
+
datasetId,
|
|
6021
|
+
models
|
|
6022
|
+
});
|
|
6023
|
+
try {
|
|
6024
|
+
await client.executeExperiment(experiment2.experimentId);
|
|
6025
|
+
} catch (execError) {
|
|
6026
|
+
if (execError instanceof ApiError && (execError.statusCode === 404 || execError.statusCode === 501)) {
|
|
6027
|
+
if (!isJson) {
|
|
6028
|
+
output.warn("Experiment execution is not yet available server-side. The experiment has been created and will run when the feature is enabled.");
|
|
6029
|
+
}
|
|
6030
|
+
} else if (execError instanceof Error) {
|
|
6031
|
+
if (!isJson) {
|
|
6032
|
+
output.warn(`Experiment created but auto-execute failed: ${execError.message}`);
|
|
6033
|
+
}
|
|
6034
|
+
}
|
|
6035
|
+
}
|
|
6036
|
+
if (isJson) {
|
|
6037
|
+
output.output({
|
|
6038
|
+
...experiment2,
|
|
6039
|
+
_links: {
|
|
6040
|
+
dashboard: promptLink(promptId),
|
|
6041
|
+
api: `/api/prompts/experiments/${experiment2.experimentId}`
|
|
6042
|
+
}
|
|
6043
|
+
});
|
|
6044
|
+
} else {
|
|
6045
|
+
output.success(`Experiment created: ${experiment2.name} (id: ${experiment2.experimentId})`);
|
|
6046
|
+
output.info(`Status: ${experiment2.status}`);
|
|
6047
|
+
output.info(`Dashboard: ${promptLink(promptId)}`);
|
|
6048
|
+
output.info(`Check progress: mutagent prompts experiment get ${experiment2.experimentId}`);
|
|
6049
|
+
}
|
|
6050
|
+
} catch (error) {
|
|
6051
|
+
handleError(error, isJson);
|
|
6052
|
+
}
|
|
6053
|
+
});
|
|
6054
|
+
experiment.command("list").description("List experiments for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").addHelpText("after", `
|
|
6055
|
+
Examples:
|
|
6056
|
+
${chalk17.dim("$")} mutagent prompts experiment list <prompt-id>
|
|
6057
|
+
${chalk17.dim("$")} mutagent prompts experiment list <prompt-id> --json
|
|
6058
|
+
`).action(async (promptId) => {
|
|
6059
|
+
const isJson = getJsonFlag(prompts);
|
|
6060
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
6061
|
+
try {
|
|
6062
|
+
const promptIdNum = parseInt(promptId, 10);
|
|
6063
|
+
if (isNaN(promptIdNum)) {
|
|
6064
|
+
throw new MutagentError("VALIDATION_ERROR", `Invalid prompt ID: "${promptId}" (must be numeric)`, "Run: mutagent prompts list to find valid IDs");
|
|
6065
|
+
}
|
|
6066
|
+
const client = await getSDKClient();
|
|
6067
|
+
const { experiments, total } = await client.listExperiments({ promptId: promptIdNum });
|
|
6068
|
+
if (isJson) {
|
|
6069
|
+
output.output({
|
|
6070
|
+
experiments,
|
|
6071
|
+
total,
|
|
6072
|
+
_links: {
|
|
6073
|
+
dashboard: promptLink(promptId),
|
|
6074
|
+
api: `/api/prompts/experiments?promptId=${promptId}`
|
|
6075
|
+
}
|
|
6076
|
+
});
|
|
6077
|
+
} else {
|
|
6078
|
+
if (experiments.length === 0) {
|
|
6079
|
+
output.info(`No experiments found for prompt ${promptId}`);
|
|
6080
|
+
output.info(`Create one: mutagent prompts experiment run ${promptId} --evaluation <id> --dataset <id> --exec-model <model>`);
|
|
6081
|
+
} else {
|
|
6082
|
+
output.info(`Found ${String(total)} experiment(s) for prompt ${promptId}:`);
|
|
6083
|
+
output.output(experiments);
|
|
6084
|
+
}
|
|
6085
|
+
}
|
|
6086
|
+
} catch (error) {
|
|
6087
|
+
handleError(error, isJson);
|
|
6088
|
+
}
|
|
6089
|
+
});
|
|
6090
|
+
experiment.command("get").description("Get details of a specific experiment").argument("<experiment-id>", "Experiment ID (from: mutagent prompts experiment list <prompt-id>)").addHelpText("after", `
|
|
6091
|
+
Examples:
|
|
6092
|
+
${chalk17.dim("$")} mutagent prompts experiment get <experiment-id>
|
|
6093
|
+
${chalk17.dim("$")} mutagent prompts experiment get <experiment-id> --json
|
|
6094
|
+
`).action(async (experimentId) => {
|
|
6095
|
+
const isJson = getJsonFlag(prompts);
|
|
6096
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
6097
|
+
try {
|
|
6098
|
+
const client = await getSDKClient();
|
|
6099
|
+
const exp = await client.getExperiment(experimentId);
|
|
6100
|
+
if (isJson) {
|
|
6101
|
+
output.output({
|
|
6102
|
+
...exp,
|
|
6103
|
+
_links: {
|
|
6104
|
+
api: `/api/prompts/experiments/${experimentId}`
|
|
6105
|
+
}
|
|
6106
|
+
});
|
|
6107
|
+
} else {
|
|
6108
|
+
const name = typeof exp.name === "string" ? exp.name : experimentId;
|
|
6109
|
+
const status = typeof exp.status === "string" ? exp.status : "unknown";
|
|
6110
|
+
output.success(`Experiment: ${name} (id: ${experimentId})`);
|
|
6111
|
+
output.info(`Status: ${status}`);
|
|
6112
|
+
if (exp.promptId !== undefined)
|
|
6113
|
+
output.info(`Prompt ID: ${String(exp.promptId)}`);
|
|
6114
|
+
if (exp.createdAt !== undefined)
|
|
6115
|
+
output.info(`Created: ${new Date(String(exp.createdAt)).toLocaleString()}`);
|
|
6116
|
+
}
|
|
6117
|
+
} catch (error) {
|
|
6118
|
+
if (error instanceof ApiError && error.statusCode === 404) {
|
|
6119
|
+
handleError(new MutagentError("NOT_FOUND", `Experiment ${experimentId} not found`, "Run: mutagent prompts experiment list <prompt-id> to see available experiments"), isJson);
|
|
6120
|
+
} else {
|
|
6121
|
+
handleError(error, isJson);
|
|
6122
|
+
}
|
|
6123
|
+
}
|
|
6124
|
+
});
|
|
6125
|
+
experiment.command("delete").description("Delete an experiment").argument("<experiment-id>", "Experiment ID").option("--force", "Skip confirmation prompt").addHelpText("after", `
|
|
6126
|
+
Examples:
|
|
6127
|
+
${chalk17.dim("$")} mutagent prompts experiment delete <experiment-id> --force
|
|
6128
|
+
${chalk17.dim("$")} mutagent prompts experiment delete <experiment-id> --json
|
|
6129
|
+
${chalk17.dim("$")} mutagent prompts experiment delete <experiment-id> --force --json
|
|
6130
|
+
|
|
6131
|
+
${chalk17.red("Warning:")} Deletion is permanent. Use --force or --json to confirm.
|
|
6132
|
+
`).action(async (experimentId, options) => {
|
|
6133
|
+
const isJson = getJsonFlag(prompts);
|
|
6134
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
6135
|
+
try {
|
|
6136
|
+
if (!options.force && !isJson) {
|
|
6137
|
+
throw new MutagentError("CONFIRMATION_REQUIRED", `Deleting experiment ${experimentId} requires confirmation`, `Run: mutagent prompts experiment delete --help
|
|
6138
|
+
` + `[Agent: Ask the user to confirm via MultiChoice, then pass --force]
|
|
6139
|
+
` + `Use --force to confirm: mutagent prompts experiment delete ${experimentId} --force`);
|
|
6140
|
+
}
|
|
6141
|
+
const client = await getSDKClient();
|
|
6142
|
+
try {
|
|
6143
|
+
await client.deleteExperiment(experimentId);
|
|
6144
|
+
} catch (deleteError) {
|
|
6145
|
+
if (deleteError instanceof ApiError && deleteError.statusCode === 404) {
|
|
6146
|
+
if (isJson) {
|
|
6147
|
+
output.output({ success: true, deletedId: experimentId, _links: { api: "/api/prompts/experiments" } });
|
|
6148
|
+
} else {
|
|
6149
|
+
output.success(`Experiment ${experimentId} already deleted (idempotent)`);
|
|
6150
|
+
}
|
|
6151
|
+
return;
|
|
6152
|
+
}
|
|
6153
|
+
throw deleteError;
|
|
6154
|
+
}
|
|
6155
|
+
if (isJson) {
|
|
6156
|
+
output.output({
|
|
6157
|
+
success: true,
|
|
6158
|
+
deletedId: experimentId,
|
|
6159
|
+
_links: { api: "/api/prompts/experiments" }
|
|
6160
|
+
});
|
|
6161
|
+
} else {
|
|
6162
|
+
output.success(`Experiment ${experimentId} deleted successfully`);
|
|
6163
|
+
}
|
|
6164
|
+
} catch (error) {
|
|
6165
|
+
handleError(error, isJson);
|
|
6166
|
+
}
|
|
6167
|
+
});
|
|
6168
|
+
}
|
|
6169
|
+
|
|
5753
6170
|
// src/commands/prompts/index.ts
|
|
5754
6171
|
function updateMutationContext(updater) {
|
|
5755
6172
|
try {
|
|
@@ -5873,19 +6290,19 @@ async function checkProviderConfigured() {
|
|
|
5873
6290
|
}
|
|
5874
6291
|
}
|
|
5875
6292
|
var PREREQUISITES_TEXT = `
|
|
5876
|
-
${
|
|
5877
|
-
1. Evaluation criteria defined ${
|
|
5878
|
-
2. Dataset uploaded ${
|
|
5879
|
-
3. LLM provider configured ${
|
|
6293
|
+
${chalk18.red("Prerequisites (required):")}
|
|
6294
|
+
1. Evaluation criteria defined ${chalk18.dim("(via dashboard or evaluation create)")}
|
|
6295
|
+
2. Dataset uploaded ${chalk18.dim("mutagent prompts dataset list <prompt-id>")}
|
|
6296
|
+
3. LLM provider configured ${chalk18.dim("Settings → Providers or mutagent providers add ...")}`;
|
|
5880
6297
|
function createPromptsCommand() {
|
|
5881
|
-
const prompts = new
|
|
6298
|
+
const prompts = new Command7("prompts").description("Manage prompts, datasets, evaluations, and optimizations").addHelpText("after", `
|
|
5882
6299
|
Examples:
|
|
5883
|
-
${
|
|
5884
|
-
${
|
|
5885
|
-
${
|
|
5886
|
-
${
|
|
5887
|
-
${
|
|
5888
|
-
${
|
|
6300
|
+
${chalk18.dim("$")} mutagent prompts list
|
|
6301
|
+
${chalk18.dim("$")} mutagent prompts get <prompt-id>
|
|
6302
|
+
${chalk18.dim("$")} mutagent prompts create --name "my-prompt" --system "You are helpful" --human "{input}"
|
|
6303
|
+
${chalk18.dim("$")} mutagent prompts dataset list <prompt-id>
|
|
6304
|
+
${chalk18.dim("$")} mutagent prompts evaluation create <prompt-id> --name "My Eval"
|
|
6305
|
+
${chalk18.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id>
|
|
5889
6306
|
|
|
5890
6307
|
Subcommands:
|
|
5891
6308
|
list, get, create, update, delete
|
|
@@ -5897,28 +6314,29 @@ Subcommands:
|
|
|
5897
6314
|
registerDatasetCommands(prompts);
|
|
5898
6315
|
registerEvaluationCommands(prompts);
|
|
5899
6316
|
registerOptimizeCommands(prompts);
|
|
6317
|
+
registerExperimentCommands(prompts);
|
|
5900
6318
|
return prompts;
|
|
5901
6319
|
}
|
|
5902
6320
|
|
|
5903
6321
|
// src/commands/traces/index.ts
|
|
5904
6322
|
init_sdk_client();
|
|
5905
|
-
import { Command as
|
|
5906
|
-
import
|
|
6323
|
+
import { Command as Command8 } from "commander";
|
|
6324
|
+
import chalk22 from "chalk";
|
|
5907
6325
|
init_errors();
|
|
5908
6326
|
|
|
5909
6327
|
// src/commands/traces/export.ts
|
|
5910
6328
|
init_sdk_client();
|
|
5911
|
-
import
|
|
6329
|
+
import chalk19 from "chalk";
|
|
5912
6330
|
init_errors();
|
|
5913
6331
|
function registerExportCommand(parent) {
|
|
5914
6332
|
parent.command("export").description("Export traces").option("-p, --prompt <id>", "Filter by prompt ID").option("-f, --format <format>", "Export format (json, csv)", "json").option("-o, --output <path>", "Output file path").addHelpText("after", `
|
|
5915
6333
|
Examples:
|
|
5916
|
-
${
|
|
5917
|
-
${
|
|
5918
|
-
${
|
|
5919
|
-
${
|
|
6334
|
+
${chalk19.dim("$")} mutagent traces export
|
|
6335
|
+
${chalk19.dim("$")} mutagent traces export --format json --output traces.json
|
|
6336
|
+
${chalk19.dim("$")} mutagent traces export --format csv --output traces.csv
|
|
6337
|
+
${chalk19.dim("$")} mutagent traces export --prompt <prompt-id> --format json
|
|
5920
6338
|
|
|
5921
|
-
${
|
|
6339
|
+
${chalk19.dim("Exports to stdout by default. Use --output to save to a file.")}
|
|
5922
6340
|
`).action(async (options) => {
|
|
5923
6341
|
const isJson = getJsonFlag(parent);
|
|
5924
6342
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5959,15 +6377,15 @@ ${chalk16.dim("Exports to stdout by default. Use --output to save to a file.")}
|
|
|
5959
6377
|
}
|
|
5960
6378
|
|
|
5961
6379
|
// src/commands/traces/analyze.ts
|
|
5962
|
-
import
|
|
6380
|
+
import chalk20 from "chalk";
|
|
5963
6381
|
function registerAnalyzeCommand(parent) {
|
|
5964
6382
|
parent.command("analyze").description("Analyze traces for a prompt (coming soon)").argument("<prompt-id>", "Prompt ID").addHelpText("after", `
|
|
5965
6383
|
Examples:
|
|
5966
|
-
${
|
|
5967
|
-
${
|
|
6384
|
+
${chalk20.dim("$")} mutagent traces analyze <prompt-id>
|
|
6385
|
+
${chalk20.dim("$")} mutagent traces analyze <prompt-id> --json
|
|
5968
6386
|
|
|
5969
|
-
${
|
|
5970
|
-
${
|
|
6387
|
+
${chalk20.dim("Aggregates trace data for a prompt: avg latency, token usage, error rates.")}
|
|
6388
|
+
${chalk20.yellow("NOTE: This command is not yet connected to the API.")}
|
|
5971
6389
|
`).action((promptId) => {
|
|
5972
6390
|
const isJson = getJsonFlag(parent);
|
|
5973
6391
|
if (isJson) {
|
|
@@ -5977,26 +6395,26 @@ ${chalk17.yellow("NOTE: This command is not yet connected to the API.")}
|
|
|
5977
6395
|
message: "Per-prompt trace analysis is not yet available. Use the dashboard at /observe/analytics for workspace-level metrics."
|
|
5978
6396
|
}));
|
|
5979
6397
|
} else {
|
|
5980
|
-
console.log(
|
|
6398
|
+
console.log(chalk20.yellow(`
|
|
5981
6399
|
⚠ Per-prompt trace analysis for ${promptId} is not yet available — coming soon.
|
|
5982
6400
|
`));
|
|
5983
|
-
console.log(
|
|
5984
|
-
console.log(
|
|
6401
|
+
console.log(chalk20.dim(" Workspace-level analytics are available on the dashboard:"));
|
|
6402
|
+
console.log(chalk20.dim(` ${getAppBaseUrl()}/observe/analytics
|
|
5985
6403
|
`));
|
|
5986
6404
|
}
|
|
5987
6405
|
});
|
|
5988
6406
|
}
|
|
5989
6407
|
|
|
5990
6408
|
// src/commands/traces/stream.ts
|
|
5991
|
-
import
|
|
6409
|
+
import chalk21 from "chalk";
|
|
5992
6410
|
function registerStreamCommand(parent) {
|
|
5993
6411
|
parent.command("stream").description("Stream traces in real-time (coming soon)").addHelpText("after", `
|
|
5994
6412
|
Examples:
|
|
5995
|
-
${
|
|
5996
|
-
${
|
|
6413
|
+
${chalk21.dim("$")} mutagent traces stream
|
|
6414
|
+
${chalk21.dim("$")} mutagent traces stream --json
|
|
5997
6415
|
|
|
5998
|
-
${
|
|
5999
|
-
${
|
|
6416
|
+
${chalk21.dim("Real-time trace streaming via SSE.")}
|
|
6417
|
+
${chalk21.yellow("NOTE: This command is not yet connected to the API.")}
|
|
6000
6418
|
`).action(() => {
|
|
6001
6419
|
const isJson = getJsonFlag(parent);
|
|
6002
6420
|
if (isJson) {
|
|
@@ -6005,12 +6423,12 @@ ${chalk18.yellow("NOTE: This command is not yet connected to the API.")}
|
|
|
6005
6423
|
message: 'Real-time trace streaming is not yet available. Use "mutagent traces list" to poll, or view live traces on the dashboard.'
|
|
6006
6424
|
}));
|
|
6007
6425
|
} else {
|
|
6008
|
-
console.log(
|
|
6426
|
+
console.log(chalk21.yellow(`
|
|
6009
6427
|
⚠ Real-time trace streaming is not yet available — coming soon.
|
|
6010
6428
|
`));
|
|
6011
|
-
console.log(
|
|
6012
|
-
console.log(
|
|
6013
|
-
console.log(
|
|
6429
|
+
console.log(chalk21.dim(" In the meantime, use:"));
|
|
6430
|
+
console.log(chalk21.dim(" mutagent traces list --json (poll for new traces)"));
|
|
6431
|
+
console.log(chalk21.dim(` ${getAppBaseUrl()}/observe/traces (live dashboard)
|
|
6014
6432
|
`));
|
|
6015
6433
|
}
|
|
6016
6434
|
});
|
|
@@ -6018,22 +6436,22 @@ ${chalk18.yellow("NOTE: This command is not yet connected to the API.")}
|
|
|
6018
6436
|
|
|
6019
6437
|
// src/commands/traces/index.ts
|
|
6020
6438
|
function createTracesCommand() {
|
|
6021
|
-
const traces = new
|
|
6439
|
+
const traces = new Command8("traces").description("View and analyze traces").addHelpText("after", `
|
|
6022
6440
|
Examples:
|
|
6023
|
-
${
|
|
6024
|
-
${
|
|
6025
|
-
${
|
|
6026
|
-
${
|
|
6027
|
-
${
|
|
6441
|
+
${chalk22.dim("$")} mutagent traces list
|
|
6442
|
+
${chalk22.dim("$")} mutagent traces list --prompt <prompt-id>
|
|
6443
|
+
${chalk22.dim("$")} mutagent traces get <trace-id>
|
|
6444
|
+
${chalk22.dim("$")} mutagent traces analyze <prompt-id>
|
|
6445
|
+
${chalk22.dim("$")} mutagent traces export --format json --output traces.json
|
|
6028
6446
|
`);
|
|
6029
6447
|
traces.command("list").description("List traces").option("-p, --prompt <id>", "Filter by prompt ID").option("-s, --source <source>", "Filter by trace source (e.g., claude-code, sdk, langchain)").option("-l, --limit <n>", "Limit results", "50").addHelpText("after", `
|
|
6030
6448
|
Examples:
|
|
6031
|
-
${
|
|
6032
|
-
${
|
|
6033
|
-
${
|
|
6034
|
-
${
|
|
6449
|
+
${chalk22.dim("$")} mutagent traces list
|
|
6450
|
+
${chalk22.dim("$")} mutagent traces list --prompt <prompt-id>
|
|
6451
|
+
${chalk22.dim("$")} mutagent traces list --source claude-code --json
|
|
6452
|
+
${chalk22.dim("$")} mutagent traces list --limit 10 --json
|
|
6035
6453
|
|
|
6036
|
-
${
|
|
6454
|
+
${chalk22.dim("Tip: Filter by prompt to see traces for a specific prompt version.")}
|
|
6037
6455
|
`).action(async (options) => {
|
|
6038
6456
|
const isJson = getJsonFlag(traces);
|
|
6039
6457
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -6073,10 +6491,10 @@ ${chalk19.dim("Tip: Filter by prompt to see traces for a specific prompt version
|
|
|
6073
6491
|
});
|
|
6074
6492
|
traces.command("get").description("Get trace details").argument("<id>", "Trace ID").addHelpText("after", `
|
|
6075
6493
|
Examples:
|
|
6076
|
-
${
|
|
6077
|
-
${
|
|
6494
|
+
${chalk22.dim("$")} mutagent traces get <trace-id>
|
|
6495
|
+
${chalk22.dim("$")} mutagent traces get <trace-id> --json
|
|
6078
6496
|
|
|
6079
|
-
${
|
|
6497
|
+
${chalk22.dim("Returns full trace details including spans, tokens, and latency.")}
|
|
6080
6498
|
`).action(async (id) => {
|
|
6081
6499
|
const isJson = getJsonFlag(traces);
|
|
6082
6500
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -6101,8 +6519,8 @@ ${chalk19.dim("Returns full trace details including spans, tokens, and latency."
|
|
|
6101
6519
|
|
|
6102
6520
|
// src/commands/integrate.ts
|
|
6103
6521
|
init_config();
|
|
6104
|
-
import { Command as
|
|
6105
|
-
import
|
|
6522
|
+
import { Command as Command9 } from "commander";
|
|
6523
|
+
import chalk23 from "chalk";
|
|
6106
6524
|
import { writeFileSync as writeFileSync3, existsSync as existsSync9 } from "fs";
|
|
6107
6525
|
import { execSync } from "child_process";
|
|
6108
6526
|
init_errors();
|
|
@@ -6697,14 +7115,14 @@ AI AGENT: Please determine the framework by:
|
|
|
6697
7115
|
`;
|
|
6698
7116
|
}
|
|
6699
7117
|
function createIntegrateCommand() {
|
|
6700
|
-
const integrate = new
|
|
7118
|
+
const integrate = new Command9("integrate").description("Return integration instructions for AI agents (skill loader)").addHelpText("after", `
|
|
6701
7119
|
Examples:
|
|
6702
|
-
${
|
|
6703
|
-
${
|
|
6704
|
-
${
|
|
6705
|
-
${
|
|
6706
|
-
${
|
|
6707
|
-
${
|
|
7120
|
+
${chalk23.dim("$")} mutagent integrate # Return exploration instructions for AI agents
|
|
7121
|
+
${chalk23.dim("$")} mutagent integrate langchain # Return @mutagent/langchain integration steps
|
|
7122
|
+
${chalk23.dim("$")} mutagent integrate openai # Return @mutagent/openai integration steps
|
|
7123
|
+
${chalk23.dim("$")} mutagent integrate vercel-ai --verify # Verify integration
|
|
7124
|
+
${chalk23.dim("$")} mutagent integrate langchain --output ./INTEGRATION.md
|
|
7125
|
+
${chalk23.dim("$")} mutagent integrate --list # List available frameworks
|
|
6708
7126
|
|
|
6709
7127
|
Frameworks: langchain, langgraph, vercel-ai, openai
|
|
6710
7128
|
|
|
@@ -6819,19 +7237,19 @@ AI AGENT: Execute these steps using Bash for install, Write/Edit for code change
|
|
|
6819
7237
|
}
|
|
6820
7238
|
|
|
6821
7239
|
// src/commands/agents/index.ts
|
|
6822
|
-
import { Command as
|
|
6823
|
-
import
|
|
7240
|
+
import { Command as Command10 } from "commander";
|
|
7241
|
+
import chalk25 from "chalk";
|
|
6824
7242
|
|
|
6825
7243
|
// src/commands/agents/agents-crud.ts
|
|
6826
7244
|
init_sdk_client();
|
|
6827
|
-
import
|
|
7245
|
+
import chalk24 from "chalk";
|
|
6828
7246
|
init_errors();
|
|
6829
7247
|
function registerAgentsCrud(agents) {
|
|
6830
7248
|
agents.command("list").description("List all agents").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").option("-n, --name <name>", "Filter by name").option("-s, --status <status>", "Filter by status (active, paused, archived)").addHelpText("after", `
|
|
6831
7249
|
Examples:
|
|
6832
|
-
${
|
|
6833
|
-
${
|
|
6834
|
-
${
|
|
7250
|
+
${chalk24.dim("$")} mutagent agents list
|
|
7251
|
+
${chalk24.dim("$")} mutagent agents list --status active
|
|
7252
|
+
${chalk24.dim("$")} mutagent agents list --name "reviewer" --json
|
|
6835
7253
|
`).action(async (options) => {
|
|
6836
7254
|
const isJson = getJsonFlag(agents);
|
|
6837
7255
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -6881,8 +7299,8 @@ Examples:
|
|
|
6881
7299
|
});
|
|
6882
7300
|
agents.command("get").description("Get agent details").argument("<id>", "Agent ID").addHelpText("after", `
|
|
6883
7301
|
Examples:
|
|
6884
|
-
${
|
|
6885
|
-
${
|
|
7302
|
+
${chalk24.dim("$")} mutagent agents get <agent-id>
|
|
7303
|
+
${chalk24.dim("$")} mutagent agents get <agent-id> --json
|
|
6886
7304
|
`).action(async (id) => {
|
|
6887
7305
|
const isJson = getJsonFlag(agents);
|
|
6888
7306
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -6912,11 +7330,11 @@ Examples:
|
|
|
6912
7330
|
};
|
|
6913
7331
|
output.output(formatted);
|
|
6914
7332
|
if (agent.systemPrompt) {
|
|
6915
|
-
console.log(
|
|
7333
|
+
console.log(chalk24.bold(`
|
|
6916
7334
|
System Prompt:`));
|
|
6917
|
-
console.log(
|
|
7335
|
+
console.log(chalk24.gray("─".repeat(60)));
|
|
6918
7336
|
console.log(agent.systemPrompt);
|
|
6919
|
-
console.log(
|
|
7337
|
+
console.log(chalk24.gray("─".repeat(60)));
|
|
6920
7338
|
}
|
|
6921
7339
|
}
|
|
6922
7340
|
} catch (error) {
|
|
@@ -6925,17 +7343,17 @@ System Prompt:`));
|
|
|
6925
7343
|
});
|
|
6926
7344
|
agents.command("create").description("Create a new agent").option("-d, --data <json>", "Agent as JSON string (recommended for CI/scripts/agents)").option("-n, --name <name>", "Agent name").option("-s, --slug <slug>", "Agent slug (URL-friendly identifier)").option("-p, --system-prompt <prompt>", "System prompt").option("-m, --model <model>", "Model (claude-sonnet-4-5, claude-opus-4-5, claude-haiku-4-5)").option("--description <desc>", "Agent description").addHelpText("after", `
|
|
6927
7345
|
Examples:
|
|
6928
|
-
${
|
|
6929
|
-
${
|
|
7346
|
+
${chalk24.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "You are a code reviewer..."
|
|
7347
|
+
${chalk24.dim("$")} mutagent agents create -d '{"name":"Code Reviewer","slug":"code-reviewer","systemPrompt":"You are a code reviewer..."}'
|
|
6930
7348
|
|
|
6931
7349
|
Expected JSON (--data):
|
|
6932
|
-
${
|
|
7350
|
+
${chalk24.dim('{"name":"<name>","slug":"<slug>","systemPrompt":"<system prompt>","model":"<model-id>","description":"<description>"}')}
|
|
6933
7351
|
|
|
6934
7352
|
Input Methods (pick one, priority order):
|
|
6935
|
-
--name/--slug/... Individual flags ${
|
|
7353
|
+
--name/--slug/... Individual flags ${chalk24.green("(recommended)")}
|
|
6936
7354
|
-d, --data Inline JSON object (CI/scripts/agents)
|
|
6937
7355
|
|
|
6938
|
-
${
|
|
7356
|
+
${chalk24.red("Required: name, slug, systemPrompt.")} ${chalk24.dim("CLI flags override --data fields.")}
|
|
6939
7357
|
`).action(async (options) => {
|
|
6940
7358
|
const isJson = getJsonFlag(agents);
|
|
6941
7359
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -6981,15 +7399,15 @@ ${chalk21.red("Required: name, slug, systemPrompt.")} ${chalk21.dim("CLI flags o
|
|
|
6981
7399
|
});
|
|
6982
7400
|
agents.command("update").description("Update an agent").argument("<id>", "Agent ID").option("-d, --data <json>", "Agent updates as JSON string (CI/scripts/agents)").option("-n, --name <name>", "New name").option("-p, --system-prompt <prompt>", "New system prompt").option("-m, --model <model>", "New model").option("--description <desc>", "New description").option("-s, --status <status>", "New status (active, paused, archived)").addHelpText("after", `
|
|
6983
7401
|
Examples:
|
|
6984
|
-
${
|
|
6985
|
-
${
|
|
6986
|
-
${
|
|
7402
|
+
${chalk24.dim("$")} mutagent agents update <id> --name "New Name"
|
|
7403
|
+
${chalk24.dim("$")} mutagent agents update <id> --system-prompt "Updated prompt" --status active
|
|
7404
|
+
${chalk24.dim("$")} mutagent agents update <id> -d '{"name":"New Name","systemPrompt":"Updated prompt"}'
|
|
6987
7405
|
|
|
6988
7406
|
Input Methods (pick one, priority order):
|
|
6989
|
-
--name/--system-prompt/... Individual flags ${
|
|
7407
|
+
--name/--system-prompt/... Individual flags ${chalk24.green("(recommended)")}
|
|
6990
7408
|
-d, --data Inline JSON object (CI/scripts/agents)
|
|
6991
7409
|
|
|
6992
|
-
${
|
|
7410
|
+
${chalk24.dim("CLI flags override --data fields.")}
|
|
6993
7411
|
`).action(async (id, options) => {
|
|
6994
7412
|
const isJson = getJsonFlag(agents);
|
|
6995
7413
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7037,11 +7455,11 @@ ${chalk21.dim("CLI flags override --data fields.")}
|
|
|
7037
7455
|
});
|
|
7038
7456
|
agents.command("delete").description("Delete an agent").argument("<id>", "Agent ID").option("--force", "Skip confirmation").addHelpText("after", `
|
|
7039
7457
|
Examples:
|
|
7040
|
-
${
|
|
7041
|
-
${
|
|
7042
|
-
${
|
|
7458
|
+
${chalk24.dim("$")} mutagent agents delete <id>
|
|
7459
|
+
${chalk24.dim("$")} mutagent agents delete <id> --force
|
|
7460
|
+
${chalk24.dim("$")} mutagent agents delete <id> --force --json
|
|
7043
7461
|
|
|
7044
|
-
${
|
|
7462
|
+
${chalk24.dim("Tip: Use --force to skip confirmation (required for non-interactive/CI usage).")}
|
|
7045
7463
|
`).action(async (id, options) => {
|
|
7046
7464
|
const isJson = getJsonFlag(agents);
|
|
7047
7465
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7070,14 +7488,14 @@ ${chalk21.dim("Tip: Use --force to skip confirmation (required for non-interacti
|
|
|
7070
7488
|
|
|
7071
7489
|
// src/commands/agents/index.ts
|
|
7072
7490
|
function createAgentsCommand() {
|
|
7073
|
-
const agents = new
|
|
7491
|
+
const agents = new Command10("agents").description("Manage AI agents").addHelpText("after", `
|
|
7074
7492
|
Examples:
|
|
7075
|
-
${
|
|
7076
|
-
${
|
|
7077
|
-
${
|
|
7078
|
-
${
|
|
7079
|
-
${
|
|
7080
|
-
${
|
|
7493
|
+
${chalk25.dim("$")} mutagent agents list
|
|
7494
|
+
${chalk25.dim("$")} mutagent agents get <agent-id>
|
|
7495
|
+
${chalk25.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "You are a code reviewer..."
|
|
7496
|
+
${chalk25.dim("$")} mutagent agents create -d '{"name":"Code Reviewer","slug":"code-reviewer","systemPrompt":"You are..."}'
|
|
7497
|
+
${chalk25.dim("$")} mutagent agents update <agent-id> --name "Updated Name"
|
|
7498
|
+
${chalk25.dim("$")} mutagent agents delete <agent-id> --force
|
|
7081
7499
|
|
|
7082
7500
|
Subcommands:
|
|
7083
7501
|
list, get, create, update, delete
|
|
@@ -7088,23 +7506,23 @@ Subcommands:
|
|
|
7088
7506
|
|
|
7089
7507
|
// src/commands/config.ts
|
|
7090
7508
|
init_config();
|
|
7091
|
-
import { Command as
|
|
7092
|
-
import
|
|
7509
|
+
import { Command as Command11 } from "commander";
|
|
7510
|
+
import chalk26 from "chalk";
|
|
7093
7511
|
init_errors();
|
|
7094
7512
|
init_sdk_client();
|
|
7095
7513
|
var VALID_CONFIG_KEYS = ["apiKey", "endpoint", "format", "timeout", "defaultWorkspace", "defaultOrganization"];
|
|
7096
7514
|
function createConfigCommand() {
|
|
7097
|
-
const config = new
|
|
7515
|
+
const config = new Command11("config").description("Manage CLI configuration").addHelpText("after", `
|
|
7098
7516
|
Examples:
|
|
7099
|
-
${
|
|
7100
|
-
${
|
|
7101
|
-
${
|
|
7102
|
-
${
|
|
7517
|
+
${chalk26.dim("$")} mutagent config list
|
|
7518
|
+
${chalk26.dim("$")} mutagent config get endpoint
|
|
7519
|
+
${chalk26.dim("$")} mutagent config set workspace <workspace-id>
|
|
7520
|
+
${chalk26.dim("$")} mutagent config set org <org-id>
|
|
7103
7521
|
`);
|
|
7104
7522
|
config.command("list").description("List all configuration").addHelpText("after", `
|
|
7105
7523
|
Examples:
|
|
7106
|
-
${
|
|
7107
|
-
${
|
|
7524
|
+
${chalk26.dim("$")} mutagent config list
|
|
7525
|
+
${chalk26.dim("$")} mutagent config list --json
|
|
7108
7526
|
`).action(() => {
|
|
7109
7527
|
const isJson = getJsonFlag(config);
|
|
7110
7528
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7117,11 +7535,11 @@ Examples:
|
|
|
7117
7535
|
});
|
|
7118
7536
|
config.command("get").description("Get configuration value").argument("<key>", "Configuration key (apiKey, endpoint, format, timeout, defaultWorkspace, defaultOrganization)").addHelpText("after", `
|
|
7119
7537
|
Examples:
|
|
7120
|
-
${
|
|
7121
|
-
${
|
|
7122
|
-
${
|
|
7538
|
+
${chalk26.dim("$")} mutagent config get endpoint
|
|
7539
|
+
${chalk26.dim("$")} mutagent config get defaultWorkspace
|
|
7540
|
+
${chalk26.dim("$")} mutagent config get apiKey --json
|
|
7123
7541
|
|
|
7124
|
-
${
|
|
7542
|
+
${chalk26.dim("Keys: apiKey, endpoint, format, timeout, defaultWorkspace, defaultOrganization")}
|
|
7125
7543
|
`).action((key) => {
|
|
7126
7544
|
const isJson = getJsonFlag(config);
|
|
7127
7545
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7143,14 +7561,14 @@ ${chalk23.dim("Keys: apiKey, endpoint, format, timeout, defaultWorkspace, defaul
|
|
|
7143
7561
|
handleError(error, isJson);
|
|
7144
7562
|
}
|
|
7145
7563
|
});
|
|
7146
|
-
const set = new
|
|
7564
|
+
const set = new Command11("set").description("Set configuration value").action(() => {
|
|
7147
7565
|
set.help();
|
|
7148
7566
|
});
|
|
7149
7567
|
set.command("workspace").description("Set default workspace ID").argument("<id>", "Workspace ID to set as default").addHelpText("after", `
|
|
7150
7568
|
Examples:
|
|
7151
|
-
${
|
|
7569
|
+
${chalk26.dim("$")} mutagent config set workspace <workspace-id>
|
|
7152
7570
|
|
|
7153
|
-
${
|
|
7571
|
+
${chalk26.dim("Persists workspace ID so you don't need to pass headers on every request.")}
|
|
7154
7572
|
`).action((id) => {
|
|
7155
7573
|
const isJson = getJsonFlag(config);
|
|
7156
7574
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7164,9 +7582,9 @@ ${chalk23.dim("Persists workspace ID so you don't need to pass headers on every
|
|
|
7164
7582
|
});
|
|
7165
7583
|
set.command("org").description("Set default organization ID").argument("<id>", "Organization ID to set as default").addHelpText("after", `
|
|
7166
7584
|
Examples:
|
|
7167
|
-
${
|
|
7585
|
+
${chalk26.dim("$")} mutagent config set org <org-id>
|
|
7168
7586
|
|
|
7169
|
-
${
|
|
7587
|
+
${chalk26.dim("Persists organization ID for org-scoped API keys.")}
|
|
7170
7588
|
`).action((id) => {
|
|
7171
7589
|
const isJson = getJsonFlag(config);
|
|
7172
7590
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7184,8 +7602,8 @@ ${chalk23.dim("Persists organization ID for org-scoped API keys.")}
|
|
|
7184
7602
|
|
|
7185
7603
|
// src/commands/playground.ts
|
|
7186
7604
|
init_sdk_client();
|
|
7187
|
-
import { Command as
|
|
7188
|
-
import
|
|
7605
|
+
import { Command as Command12 } from "commander";
|
|
7606
|
+
import chalk27 from "chalk";
|
|
7189
7607
|
init_errors();
|
|
7190
7608
|
function parseSSELine(line3) {
|
|
7191
7609
|
if (!line3 || line3.startsWith(":")) {
|
|
@@ -7210,13 +7628,13 @@ function parsePromptStreamEvent(data) {
|
|
|
7210
7628
|
}
|
|
7211
7629
|
}
|
|
7212
7630
|
function createPlaygroundCommand() {
|
|
7213
|
-
const playground = new
|
|
7631
|
+
const playground = new Command12("playground").description("Execute and test prompts interactively").addHelpText("after", `
|
|
7214
7632
|
Examples:
|
|
7215
|
-
${
|
|
7216
|
-
${
|
|
7217
|
-
${
|
|
7218
|
-
${
|
|
7219
|
-
${
|
|
7633
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> --input '{"name": "John"}'
|
|
7634
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> --input '{}' --stream
|
|
7635
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> -i '{}' --model gpt-4-turbo
|
|
7636
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> --system "You are helpful" --human "Hello"
|
|
7637
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> --messages '[{"role":"user","content":"Hi"}]'
|
|
7220
7638
|
|
|
7221
7639
|
Input Format:
|
|
7222
7640
|
The input must be a valid JSON object matching the prompt's input schema.
|
|
@@ -7232,16 +7650,16 @@ Streaming:
|
|
|
7232
7650
|
`);
|
|
7233
7651
|
playground.command("run").description("Execute a prompt with input variables").argument("<prompt-id>", "Prompt ID to execute (from: mutagent prompts list)").option("-i, --input <json>", "Input variables as JSON").option("-s, --stream", "Stream the response").option("-m, --model <model>", "Override model").option("--system <text>", "Set system prompt text").option("--human <text>", "Set human/user message text").option("--messages <json>", "Pass full messages array as JSON string").addHelpText("after", `
|
|
7234
7652
|
Examples:
|
|
7235
|
-
${
|
|
7236
|
-
${
|
|
7237
|
-
${
|
|
7238
|
-
${
|
|
7653
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> --input '{"name": "John"}'
|
|
7654
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> --input '{}' --stream
|
|
7655
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> --system "You are helpful" --human "Hello"
|
|
7656
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> --input '{}' --model gpt-4-turbo --json
|
|
7239
7657
|
|
|
7240
7658
|
Input Methods (pick one, priority order):
|
|
7241
|
-
--system/--human Quick system + user message ${
|
|
7659
|
+
--system/--human Quick system + user message ${chalk27.green("(recommended)")}
|
|
7242
7660
|
--input '{"key":"value"}' Inline JSON variables
|
|
7243
7661
|
--messages '[...]' Full messages array
|
|
7244
|
-
${
|
|
7662
|
+
${chalk27.dim(`Hint: Test before evaluating: mutagent playground run <id> --input '{"key":"value"}'`)}
|
|
7245
7663
|
`).action(async (promptId, options) => {
|
|
7246
7664
|
const isJson = getJsonFlag(playground);
|
|
7247
7665
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7263,21 +7681,21 @@ ${chalk24.dim(`Hint: Test before evaluating: mutagent playground run <id> --inpu
|
|
|
7263
7681
|
}
|
|
7264
7682
|
});
|
|
7265
7683
|
} else {
|
|
7266
|
-
console.log(
|
|
7684
|
+
console.log(chalk27.bold(`
|
|
7267
7685
|
Execution Result:`));
|
|
7268
|
-
console.log(
|
|
7269
|
-
console.log(
|
|
7686
|
+
console.log(chalk27.gray("─".repeat(50)));
|
|
7687
|
+
console.log(chalk27.cyan("Output:"));
|
|
7270
7688
|
console.log(result.output);
|
|
7271
|
-
console.log(
|
|
7272
|
-
console.log(
|
|
7273
|
-
console.log(
|
|
7689
|
+
console.log(chalk27.gray("─".repeat(50)));
|
|
7690
|
+
console.log(chalk27.dim(`Model: ${result.model}`));
|
|
7691
|
+
console.log(chalk27.dim(`Execution Time: ${String(result.executionTimeMs)}ms`));
|
|
7274
7692
|
if (result.tokens) {
|
|
7275
|
-
console.log(
|
|
7693
|
+
console.log(chalk27.dim(`Tokens: ${String(result.tokens.prompt)} prompt + ${String(result.tokens.completion)} completion = ${String(result.tokens.total)} total`));
|
|
7276
7694
|
}
|
|
7277
7695
|
if (result.cost !== undefined) {
|
|
7278
|
-
console.log(
|
|
7696
|
+
console.log(chalk27.dim(`Cost: $${result.cost.toFixed(6)}`));
|
|
7279
7697
|
}
|
|
7280
|
-
console.log(
|
|
7698
|
+
console.log(chalk27.dim(`Playground: ${playgroundLink(promptId)}`));
|
|
7281
7699
|
console.log();
|
|
7282
7700
|
}
|
|
7283
7701
|
}
|
|
@@ -7366,9 +7784,9 @@ async function executeStreaming(client, promptId, input, model, isJson, output)
|
|
|
7366
7784
|
const decoder = new TextDecoder;
|
|
7367
7785
|
let buffer = "";
|
|
7368
7786
|
if (!isJson) {
|
|
7369
|
-
console.log(
|
|
7787
|
+
console.log(chalk27.bold(`
|
|
7370
7788
|
Streaming Output:`));
|
|
7371
|
-
console.log(
|
|
7789
|
+
console.log(chalk27.gray("─".repeat(50)));
|
|
7372
7790
|
}
|
|
7373
7791
|
try {
|
|
7374
7792
|
for (;; ) {
|
|
@@ -7407,15 +7825,15 @@ Streaming Output:`));
|
|
|
7407
7825
|
console.log(JSON.stringify({ type: "complete", result: event.result }));
|
|
7408
7826
|
} else {
|
|
7409
7827
|
console.log();
|
|
7410
|
-
console.log(
|
|
7828
|
+
console.log(chalk27.gray("─".repeat(50)));
|
|
7411
7829
|
if (event.result) {
|
|
7412
|
-
console.log(
|
|
7413
|
-
console.log(
|
|
7830
|
+
console.log(chalk27.dim(`Model: ${event.result.model}`));
|
|
7831
|
+
console.log(chalk27.dim(`Execution Time: ${String(event.result.executionTimeMs)}ms`));
|
|
7414
7832
|
if (event.result.tokens) {
|
|
7415
|
-
console.log(
|
|
7833
|
+
console.log(chalk27.dim(`Tokens: ${String(event.result.tokens.prompt)} prompt + ${String(event.result.tokens.completion)} completion = ${String(event.result.tokens.total)} total`));
|
|
7416
7834
|
}
|
|
7417
7835
|
if (event.result.cost !== undefined) {
|
|
7418
|
-
console.log(
|
|
7836
|
+
console.log(chalk27.dim(`Cost: $${event.result.cost.toFixed(6)}`));
|
|
7419
7837
|
}
|
|
7420
7838
|
}
|
|
7421
7839
|
console.log();
|
|
@@ -7439,14 +7857,14 @@ Streaming Output:`));
|
|
|
7439
7857
|
|
|
7440
7858
|
// src/commands/workspaces.ts
|
|
7441
7859
|
init_sdk_client();
|
|
7442
|
-
import { Command as
|
|
7443
|
-
import
|
|
7860
|
+
import { Command as Command13 } from "commander";
|
|
7861
|
+
import chalk28 from "chalk";
|
|
7444
7862
|
init_errors();
|
|
7445
7863
|
function createWorkspacesCommand() {
|
|
7446
|
-
const workspaces = new
|
|
7864
|
+
const workspaces = new Command13("workspaces").description("View workspaces (read-only)").addHelpText("after", `
|
|
7447
7865
|
Examples:
|
|
7448
|
-
${
|
|
7449
|
-
${
|
|
7866
|
+
${chalk28.dim("$")} mutagent workspaces list
|
|
7867
|
+
${chalk28.dim("$")} mutagent workspaces get <workspace-id>
|
|
7450
7868
|
|
|
7451
7869
|
Subcommands:
|
|
7452
7870
|
list, get
|
|
@@ -7455,8 +7873,8 @@ Note: Workspace management (create, update, delete) is available in the Admin Pa
|
|
|
7455
7873
|
`);
|
|
7456
7874
|
workspaces.command("list").description("List all workspaces").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").addHelpText("after", `
|
|
7457
7875
|
Examples:
|
|
7458
|
-
${
|
|
7459
|
-
${
|
|
7876
|
+
${chalk28.dim("$")} mutagent workspaces list
|
|
7877
|
+
${chalk28.dim("$")} mutagent workspaces list --limit 10 --json
|
|
7460
7878
|
`).action(async (options) => {
|
|
7461
7879
|
const isJson = getJsonFlag(workspaces);
|
|
7462
7880
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7496,8 +7914,8 @@ Examples:
|
|
|
7496
7914
|
});
|
|
7497
7915
|
workspaces.command("get").description("Get workspace details").argument("<id>", "Workspace ID").addHelpText("after", `
|
|
7498
7916
|
Examples:
|
|
7499
|
-
${
|
|
7500
|
-
${
|
|
7917
|
+
${chalk28.dim("$")} mutagent workspaces get <workspace-id>
|
|
7918
|
+
${chalk28.dim("$")} mutagent workspaces get <workspace-id> --json
|
|
7501
7919
|
`).action(async (id) => {
|
|
7502
7920
|
const isJson = getJsonFlag(workspaces);
|
|
7503
7921
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7529,13 +7947,13 @@ Examples:
|
|
|
7529
7947
|
|
|
7530
7948
|
// src/commands/providers/index.ts
|
|
7531
7949
|
init_sdk_client();
|
|
7532
|
-
import { Command as
|
|
7533
|
-
import
|
|
7950
|
+
import { Command as Command14 } from "commander";
|
|
7951
|
+
import chalk32 from "chalk";
|
|
7534
7952
|
init_errors();
|
|
7535
7953
|
|
|
7536
7954
|
// src/commands/providers/add.ts
|
|
7537
7955
|
init_sdk_client();
|
|
7538
|
-
import
|
|
7956
|
+
import chalk29 from "chalk";
|
|
7539
7957
|
init_errors();
|
|
7540
7958
|
function resolveScope(scopeFlag, client) {
|
|
7541
7959
|
const scope = scopeFlag ?? "workspace";
|
|
@@ -7558,10 +7976,10 @@ function resolveScope(scopeFlag, client) {
|
|
|
7558
7976
|
function registerAddCommand(parent) {
|
|
7559
7977
|
parent.command("add").description("Add a new provider configuration").requiredOption("-p, --provider <type>", "Provider type (openai, anthropic, google, ...)").requiredOption("-n, --name <name>", "Display name for this provider").requiredOption("-k, --api-key <key>", "API key for the provider").option("-s, --scope <scope>", "Scope: workspace (default), org, user", "workspace").option("--base-url <url>", "Custom base URL for the provider API").option("--set-default", "Set as default provider for this scope").addHelpText("after", `
|
|
7560
7978
|
Examples:
|
|
7561
|
-
${
|
|
7562
|
-
${
|
|
7563
|
-
${
|
|
7564
|
-
${
|
|
7979
|
+
${chalk29.dim("$")} mutagent providers add --provider openai --name "My OpenAI" --api-key $OPENAI_API_KEY
|
|
7980
|
+
${chalk29.dim("$")} mutagent providers add --provider anthropic --name "Team Claude" --api-key $KEY --scope org
|
|
7981
|
+
${chalk29.dim("$")} mutagent providers add --provider openai --name "Personal" --api-key $KEY --scope user --set-default
|
|
7982
|
+
${chalk29.dim("$")} mutagent providers add --provider custom --name "Ollama" --api-key none --base-url http://localhost:11434 --json
|
|
7565
7983
|
|
|
7566
7984
|
Scope Resolution:
|
|
7567
7985
|
workspace (default) Uses your configured workspace
|
|
@@ -7607,10 +8025,10 @@ The API key is encrypted server-side and never returned in plain text.
|
|
|
7607
8025
|
console.log(` Scope: ${options.scope ?? "workspace"}`);
|
|
7608
8026
|
console.log(` URL: ${providerLink(created.id)}`);
|
|
7609
8027
|
if (options.setDefault) {
|
|
7610
|
-
console.log(
|
|
8028
|
+
console.log(chalk29.green(" Set as default provider"));
|
|
7611
8029
|
}
|
|
7612
8030
|
console.log("");
|
|
7613
|
-
console.log(
|
|
8031
|
+
console.log(chalk29.dim("API key is encrypted server-side. Use `providers get` to see masked key."));
|
|
7614
8032
|
}
|
|
7615
8033
|
} catch (error) {
|
|
7616
8034
|
handleError(error, isJson);
|
|
@@ -7648,15 +8066,15 @@ function buildProviderCreatedDirective(provider, scope) {
|
|
|
7648
8066
|
|
|
7649
8067
|
// src/commands/providers/update.ts
|
|
7650
8068
|
init_sdk_client();
|
|
7651
|
-
import
|
|
8069
|
+
import chalk30 from "chalk";
|
|
7652
8070
|
init_errors();
|
|
7653
8071
|
function registerUpdateCommand(parent) {
|
|
7654
8072
|
parent.command("update").description("Update an existing provider configuration").argument("<id>", "Provider ID (from: mutagent providers list)").option("-n, --name <name>", "Updated display name").option("-k, --api-key <key>", "Updated API key (will be re-encrypted)").option("--active <bool>", "Activate or deactivate (true|false)").option("--set-default", "Set as default provider for its scope").option("--base-url <url>", 'Updated base URL (use "" to clear)').addHelpText("after", `
|
|
7655
8073
|
Examples:
|
|
7656
|
-
${
|
|
7657
|
-
${
|
|
7658
|
-
${
|
|
7659
|
-
${
|
|
8074
|
+
${chalk30.dim("$")} mutagent providers update <id> --name "New Name"
|
|
8075
|
+
${chalk30.dim("$")} mutagent providers update <id> --api-key $NEW_KEY --json
|
|
8076
|
+
${chalk30.dim("$")} mutagent providers update <id> --active false
|
|
8077
|
+
${chalk30.dim("$")} mutagent providers update <id> --set-default --json
|
|
7660
8078
|
|
|
7661
8079
|
PATCH semantics — only provided fields are updated.
|
|
7662
8080
|
`).action(async (id, options) => {
|
|
@@ -7707,7 +8125,7 @@ PATCH semantics — only provided fields are updated.
|
|
|
7707
8125
|
console.log(` ID: ${String(updated.id ?? id)}`);
|
|
7708
8126
|
console.log(` URL: ${providerLink(updated.id ?? id)}`);
|
|
7709
8127
|
if (options.apiKey) {
|
|
7710
|
-
console.log(
|
|
8128
|
+
console.log(chalk30.dim(" API key re-encrypted server-side."));
|
|
7711
8129
|
}
|
|
7712
8130
|
}
|
|
7713
8131
|
} catch (error) {
|
|
@@ -7746,17 +8164,17 @@ function buildProviderUpdatedDirective(provider, requestId) {
|
|
|
7746
8164
|
|
|
7747
8165
|
// src/commands/providers/delete.ts
|
|
7748
8166
|
init_sdk_client();
|
|
7749
|
-
import
|
|
8167
|
+
import chalk31 from "chalk";
|
|
7750
8168
|
init_errors();
|
|
7751
8169
|
function registerDeleteCommand(parent) {
|
|
7752
8170
|
parent.command("delete").description("Delete a provider configuration").argument("<id>", "Provider ID (from: mutagent providers list)").option("-f, --force", "Skip confirmation prompt").addHelpText("after", `
|
|
7753
8171
|
Examples:
|
|
7754
|
-
${
|
|
7755
|
-
${
|
|
7756
|
-
${
|
|
8172
|
+
${chalk31.dim("$")} mutagent providers delete <id>
|
|
8173
|
+
${chalk31.dim("$")} mutagent providers delete <id> --force
|
|
8174
|
+
${chalk31.dim("$")} mutagent providers delete <id> --force --json
|
|
7757
8175
|
|
|
7758
|
-
${
|
|
7759
|
-
${
|
|
8176
|
+
${chalk31.dim("Note: --force is required. The CLI is non-interactive — confirm with the user via your native flow, then pass --force.")}
|
|
8177
|
+
${chalk31.dim("Warning: API keys are AES-256-GCM encrypted and irrecoverable after deletion. Agents referencing this provider will lose their model config.")}
|
|
7760
8178
|
`).action(async (id, options) => {
|
|
7761
8179
|
const isJson = getJsonFlag(parent);
|
|
7762
8180
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7844,14 +8262,14 @@ function validateProviderType(type) {
|
|
|
7844
8262
|
return type;
|
|
7845
8263
|
}
|
|
7846
8264
|
function createProvidersCommand() {
|
|
7847
|
-
const providers = new
|
|
8265
|
+
const providers = new Command14("providers").description("Manage LLM provider configurations (BYOK)").addHelpText("after", `
|
|
7848
8266
|
Examples:
|
|
7849
|
-
${
|
|
7850
|
-
${
|
|
7851
|
-
${
|
|
7852
|
-
${
|
|
7853
|
-
${
|
|
7854
|
-
${
|
|
8267
|
+
${chalk32.dim("$")} mutagent providers list
|
|
8268
|
+
${chalk32.dim("$")} mutagent providers get <provider-id>
|
|
8269
|
+
${chalk32.dim("$")} mutagent providers add --provider openai --name "My OpenAI" --api-key $KEY
|
|
8270
|
+
${chalk32.dim("$")} mutagent providers update <id> --name "New Name"
|
|
8271
|
+
${chalk32.dim("$")} mutagent providers delete <id> --force
|
|
8272
|
+
${chalk32.dim("$")} mutagent providers test <provider-id>
|
|
7855
8273
|
|
|
7856
8274
|
Provider Types:
|
|
7857
8275
|
openai, anthropic, google, azure, bedrock, cohere, mistral, groq, together, replicate, custom
|
|
@@ -7861,9 +8279,9 @@ Subcommands:
|
|
|
7861
8279
|
`);
|
|
7862
8280
|
providers.command("list").description("List all providers").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").option("-t, --type <type>", "Filter by provider type").addHelpText("after", `
|
|
7863
8281
|
Examples:
|
|
7864
|
-
${
|
|
7865
|
-
${
|
|
7866
|
-
${
|
|
8282
|
+
${chalk32.dim("$")} mutagent providers list
|
|
8283
|
+
${chalk32.dim("$")} mutagent providers list --type openai
|
|
8284
|
+
${chalk32.dim("$")} mutagent providers list --json
|
|
7867
8285
|
`).action(async (options) => {
|
|
7868
8286
|
const isJson = getJsonFlag(providers);
|
|
7869
8287
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7912,8 +8330,8 @@ Examples:
|
|
|
7912
8330
|
});
|
|
7913
8331
|
providers.command("get").description("Get provider details").argument("<id>", "Provider ID (from: mutagent providers list)").addHelpText("after", `
|
|
7914
8332
|
Examples:
|
|
7915
|
-
${
|
|
7916
|
-
${
|
|
8333
|
+
${chalk32.dim("$")} mutagent providers get <provider-id>
|
|
8334
|
+
${chalk32.dim("$")} mutagent providers get <provider-id> --json
|
|
7917
8335
|
`).action(async (id) => {
|
|
7918
8336
|
const isJson = getJsonFlag(providers);
|
|
7919
8337
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7942,10 +8360,10 @@ Examples:
|
|
|
7942
8360
|
});
|
|
7943
8361
|
providers.command("test").description("Test provider connectivity").argument("<id>", "Provider ID (from: mutagent providers list)").addHelpText("after", `
|
|
7944
8362
|
Examples:
|
|
7945
|
-
${
|
|
7946
|
-
${
|
|
8363
|
+
${chalk32.dim("$")} mutagent providers test <provider-id>
|
|
8364
|
+
${chalk32.dim("$")} mutagent providers test <provider-id> --json
|
|
7947
8365
|
|
|
7948
|
-
${
|
|
8366
|
+
${chalk32.dim("Tests connectivity and lists available models for the provider.")}
|
|
7949
8367
|
`).action(async (id) => {
|
|
7950
8368
|
const isJson = getJsonFlag(providers);
|
|
7951
8369
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7960,9 +8378,9 @@ ${chalk29.dim("Tests connectivity and lists available models for the provider.")
|
|
|
7960
8378
|
} else {
|
|
7961
8379
|
if (result.success) {
|
|
7962
8380
|
output.success(`Provider test passed (${String(result.responseTimeMs)}ms)`);
|
|
7963
|
-
console.log(
|
|
8381
|
+
console.log(chalk32.green(`Message: ${result.message}`));
|
|
7964
8382
|
if (result.availableModels && result.availableModels.length > 0) {
|
|
7965
|
-
console.log(
|
|
8383
|
+
console.log(chalk32.bold(`
|
|
7966
8384
|
Available Models:`));
|
|
7967
8385
|
result.availableModels.forEach((model) => {
|
|
7968
8386
|
console.log(` - ${model}`);
|
|
@@ -7971,7 +8389,7 @@ Available Models:`));
|
|
|
7971
8389
|
} else {
|
|
7972
8390
|
output.error(`Provider test failed: ${result.message}`);
|
|
7973
8391
|
if (result.error) {
|
|
7974
|
-
console.log(
|
|
8392
|
+
console.log(chalk32.red(`Error: ${result.error}`));
|
|
7975
8393
|
}
|
|
7976
8394
|
}
|
|
7977
8395
|
}
|
|
@@ -7987,9 +8405,9 @@ Available Models:`));
|
|
|
7987
8405
|
|
|
7988
8406
|
// src/commands/init.ts
|
|
7989
8407
|
init_config();
|
|
7990
|
-
import { Command as
|
|
8408
|
+
import { Command as Command15 } from "commander";
|
|
7991
8409
|
import inquirer2 from "inquirer";
|
|
7992
|
-
import
|
|
8410
|
+
import chalk33 from "chalk";
|
|
7993
8411
|
import { existsSync as existsSync11, mkdirSync as mkdirSync3, writeFileSync as writeFileSync4 } from "fs";
|
|
7994
8412
|
import { execSync as execSync3 } from "child_process";
|
|
7995
8413
|
import { join as join6 } from "path";
|
|
@@ -8110,15 +8528,15 @@ function writeRcConfig(config, cwd = process.cwd()) {
|
|
|
8110
8528
|
`);
|
|
8111
8529
|
}
|
|
8112
8530
|
function createInitCommand() {
|
|
8113
|
-
const init = new
|
|
8531
|
+
const init = new Command15("init").description("Initialize MutagenT in your project").option("--non-interactive", "Skip interactive prompts (defaults to CLI-only mode)").addHelpText("after", `
|
|
8114
8532
|
Examples:
|
|
8115
|
-
${
|
|
8116
|
-
${
|
|
8533
|
+
${chalk33.dim("$")} mutagent init # Interactive setup wizard
|
|
8534
|
+
${chalk33.dim("$")} mutagent init --non-interactive # CLI-only mode (no prompts)
|
|
8117
8535
|
|
|
8118
8536
|
Modes:
|
|
8119
|
-
${
|
|
8120
|
-
${
|
|
8121
|
-
${
|
|
8537
|
+
${chalk33.bold("Full scaffold")} Install SDK + integration package, create config, setup tracing
|
|
8538
|
+
${chalk33.bold("CLI-only")} Verify auth + create .mutagentrc.json with workspace/endpoint
|
|
8539
|
+
${chalk33.bold("Skip")} Exit without changes
|
|
8122
8540
|
`).action(async (options) => {
|
|
8123
8541
|
const isJson = getJsonFlag(init);
|
|
8124
8542
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -8389,24 +8807,24 @@ Modes:
|
|
|
8389
8807
|
}
|
|
8390
8808
|
|
|
8391
8809
|
// src/commands/explore.ts
|
|
8392
|
-
import { Command as
|
|
8393
|
-
import
|
|
8810
|
+
import { Command as Command16 } from "commander";
|
|
8811
|
+
import chalk34 from "chalk";
|
|
8394
8812
|
import { resolve as resolve3 } from "path";
|
|
8395
8813
|
init_errors();
|
|
8396
8814
|
function createExploreCommand() {
|
|
8397
|
-
const explore = new
|
|
8815
|
+
const explore = new Command16("explore").description("Scan codebase for prompts, datasets, and MutagenT markers").option("-p, --path <dir>", "Directory to scan", ".").option("--depth <n>", "Max directory depth", "10").option("--include <glob>", "Include file pattern", "**/*.{ts,js,py,tsx,jsx}").option("--exclude <dirs>", "Comma-separated directories to exclude", "node_modules,dist,.git,build,.next,__pycache__,venv,.venv").option("--markers-only", "Only find existing MutagenT markers").addHelpText("after", `
|
|
8398
8816
|
Examples:
|
|
8399
|
-
${
|
|
8400
|
-
${
|
|
8401
|
-
${
|
|
8402
|
-
${
|
|
8403
|
-
${
|
|
8817
|
+
${chalk34.dim("$")} mutagent explore
|
|
8818
|
+
${chalk34.dim("$")} mutagent explore --path ./src
|
|
8819
|
+
${chalk34.dim("$")} mutagent explore --include "**/*.{ts,py}" --depth 5
|
|
8820
|
+
${chalk34.dim("$")} mutagent explore --markers-only
|
|
8821
|
+
${chalk34.dim("$")} mutagent explore --json
|
|
8404
8822
|
|
|
8405
8823
|
Detection modes:
|
|
8406
|
-
${
|
|
8407
|
-
${
|
|
8824
|
+
${chalk34.dim("Heuristic")} Template variables ({{var}}), prompt constants, schema definitions
|
|
8825
|
+
${chalk34.dim("Marker")} MutagenT:START/END comment markers from previous uploads
|
|
8408
8826
|
|
|
8409
|
-
${
|
|
8827
|
+
${chalk34.dim("Results are saved to .mutagent/mutation-context.md for use by other commands.")}
|
|
8410
8828
|
`).action((options) => {
|
|
8411
8829
|
const isJson = getJsonFlag(explore);
|
|
8412
8830
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -8424,7 +8842,7 @@ ${chalk31.dim("Results are saved to .mutagent/mutation-context.md for use by oth
|
|
|
8424
8842
|
markersOnly
|
|
8425
8843
|
};
|
|
8426
8844
|
if (!isJson) {
|
|
8427
|
-
console.log(
|
|
8845
|
+
console.log(chalk34.cyan(`
|
|
8428
8846
|
Scanning ${scanPath}...
|
|
8429
8847
|
`));
|
|
8430
8848
|
}
|
|
@@ -8453,41 +8871,41 @@ Scanning ${scanPath}...
|
|
|
8453
8871
|
const totalFindings = result.prompts.length + result.datasets.length + result.markers.length;
|
|
8454
8872
|
if (totalFindings === 0) {
|
|
8455
8873
|
output.info("No prompts, datasets, or markers found.");
|
|
8456
|
-
console.log(
|
|
8874
|
+
console.log(chalk34.dim(`
|
|
8457
8875
|
Tip: Create a prompt with template variables like {{input}} to get started.`));
|
|
8458
8876
|
return;
|
|
8459
8877
|
}
|
|
8460
8878
|
if (result.prompts.length > 0) {
|
|
8461
|
-
console.log(
|
|
8879
|
+
console.log(chalk34.bold(` Prompts Found (${String(result.prompts.length)}):`));
|
|
8462
8880
|
console.log();
|
|
8463
8881
|
for (const p of result.prompts) {
|
|
8464
|
-
const confidenceTag = p.confidence === "high" ?
|
|
8465
|
-
const reasonTag =
|
|
8466
|
-
console.log(` ${confidenceTag} ${
|
|
8467
|
-
console.log(` ${
|
|
8882
|
+
const confidenceTag = p.confidence === "high" ? chalk34.green("[high]") : p.confidence === "medium" ? chalk34.yellow("[medium]") : chalk34.dim("[low]");
|
|
8883
|
+
const reasonTag = chalk34.dim(`[${p.reason}]`);
|
|
8884
|
+
console.log(` ${confidenceTag} ${chalk34.green(p.file)}:${chalk34.yellow(String(p.line))} ${reasonTag}`);
|
|
8885
|
+
console.log(` ${chalk34.dim(p.preview)}`);
|
|
8468
8886
|
}
|
|
8469
8887
|
console.log();
|
|
8470
8888
|
}
|
|
8471
8889
|
if (result.datasets.length > 0) {
|
|
8472
|
-
console.log(
|
|
8890
|
+
console.log(chalk34.bold(` Datasets Found (${String(result.datasets.length)}):`));
|
|
8473
8891
|
console.log();
|
|
8474
8892
|
for (const d of result.datasets) {
|
|
8475
|
-
console.log(` ${
|
|
8893
|
+
console.log(` ${chalk34.green(d.file)} ${chalk34.dim(`(${String(d.items)} items)`)}`);
|
|
8476
8894
|
}
|
|
8477
8895
|
console.log();
|
|
8478
8896
|
}
|
|
8479
8897
|
if (result.markers.length > 0) {
|
|
8480
|
-
console.log(
|
|
8898
|
+
console.log(chalk34.bold(` MutagenT Markers (${String(result.markers.length)}):`));
|
|
8481
8899
|
console.log();
|
|
8482
8900
|
for (const m of result.markers) {
|
|
8483
|
-
const idPart = m.platformId ?
|
|
8484
|
-
console.log(` ${
|
|
8901
|
+
const idPart = m.platformId ? chalk34.cyan(` id=${m.platformId}`) : "";
|
|
8902
|
+
console.log(` ${chalk34.green(m.file)}:${chalk34.yellow(String(m.line))} ${chalk34.magenta(m.type)}${idPart}`);
|
|
8485
8903
|
}
|
|
8486
8904
|
console.log();
|
|
8487
8905
|
}
|
|
8488
|
-
console.log(
|
|
8489
|
-
console.log(` ${
|
|
8490
|
-
console.log(
|
|
8906
|
+
console.log(chalk34.dim(" ─────────────────────────────────"));
|
|
8907
|
+
console.log(` ${chalk34.bold("Summary:")} ${String(result.prompts.length)} prompts, ${String(result.datasets.length)} datasets, ${String(result.markers.length)} markers`);
|
|
8908
|
+
console.log(chalk34.dim(` Saved to .mutagent/mutation-context.md`));
|
|
8491
8909
|
console.log();
|
|
8492
8910
|
}
|
|
8493
8911
|
} catch (error) {
|
|
@@ -8498,8 +8916,8 @@ Scanning ${scanPath}...
|
|
|
8498
8916
|
}
|
|
8499
8917
|
|
|
8500
8918
|
// src/commands/skills.ts
|
|
8501
|
-
import { Command as
|
|
8502
|
-
import
|
|
8919
|
+
import { Command as Command17 } from "commander";
|
|
8920
|
+
import chalk35 from "chalk";
|
|
8503
8921
|
import { existsSync as existsSync12, mkdirSync as mkdirSync4, writeFileSync as writeFileSync5 } from "fs";
|
|
8504
8922
|
import { join as join7 } from "path";
|
|
8505
8923
|
import { execSync as execSync4 } from "child_process";
|
|
@@ -8620,10 +9038,10 @@ Or run: \`mutagent init\` (option 1 installs skill + hooks together)`;
|
|
|
8620
9038
|
var SKILL_DIR = ".claude/skills/mutagent-cli";
|
|
8621
9039
|
var SKILL_FILE = "SKILL.md";
|
|
8622
9040
|
function createSkillsCommand() {
|
|
8623
|
-
const skills = new
|
|
9041
|
+
const skills = new Command17("skills").description("Manage MutagenT CLI skills for coding agents");
|
|
8624
9042
|
skills.command("install").description("Install MutagenT CLI skill for Claude Code").addHelpText("after", `
|
|
8625
9043
|
Examples:
|
|
8626
|
-
${
|
|
9044
|
+
${chalk35.dim("$")} mutagent skills install
|
|
8627
9045
|
|
|
8628
9046
|
This creates a Claude Code skill at .claude/skills/mutagent-cli/SKILL.md
|
|
8629
9047
|
that teaches coding agents how to use the MutagenT CLI effectively.
|
|
@@ -8650,10 +9068,10 @@ ${SKILL_BODY}
|
|
|
8650
9068
|
});
|
|
8651
9069
|
} else {
|
|
8652
9070
|
output.success(`Installed MutagenT CLI skill`);
|
|
8653
|
-
console.log(` ${
|
|
9071
|
+
console.log(` ${chalk35.dim("Path:")} ${skillPath}`);
|
|
8654
9072
|
console.log("");
|
|
8655
|
-
console.log(` ${
|
|
8656
|
-
console.log(` ${
|
|
9073
|
+
console.log(` ${chalk35.dim("This skill teaches coding agents how to use the MutagenT CLI.")}`);
|
|
9074
|
+
console.log(` ${chalk35.dim("It will be automatically loaded by Claude Code when relevant triggers match.")}`);
|
|
8657
9075
|
}
|
|
8658
9076
|
});
|
|
8659
9077
|
return skills;
|
|
@@ -8661,16 +9079,16 @@ ${SKILL_BODY}
|
|
|
8661
9079
|
|
|
8662
9080
|
// src/commands/usage.ts
|
|
8663
9081
|
init_config();
|
|
8664
|
-
import { Command as
|
|
8665
|
-
import
|
|
9082
|
+
import { Command as Command18 } from "commander";
|
|
9083
|
+
import chalk36 from "chalk";
|
|
8666
9084
|
init_errors();
|
|
8667
9085
|
init_sdk_client();
|
|
8668
9086
|
var PROVIDERS_URL = "https://app.mutagent.io/settings/providers";
|
|
8669
9087
|
function createUsageCommand() {
|
|
8670
|
-
const usage = new
|
|
9088
|
+
const usage = new Command18("usage").description("Show resource counts (prompts, datasets, evaluations, optimizations, experiments)").addHelpText("after", `
|
|
8671
9089
|
Examples:
|
|
8672
|
-
${
|
|
8673
|
-
${
|
|
9090
|
+
${chalk36.dim("$")} mutagent usage
|
|
9091
|
+
${chalk36.dim("$")} mutagent usage --json
|
|
8674
9092
|
`);
|
|
8675
9093
|
usage.action(async () => {
|
|
8676
9094
|
const isJson = getJsonFlag(usage);
|
|
@@ -8725,17 +9143,17 @@ Examples:
|
|
|
8725
9143
|
});
|
|
8726
9144
|
} else {
|
|
8727
9145
|
console.log("");
|
|
8728
|
-
console.log(
|
|
8729
|
-
console.log(
|
|
9146
|
+
console.log(chalk36.bold("\uD83D\uDCCA MutagenT Usage"));
|
|
9147
|
+
console.log(chalk36.dim("─".repeat(45)));
|
|
8730
9148
|
console.log("");
|
|
8731
|
-
console.log(
|
|
8732
|
-
console.log(` Prompts: ${
|
|
8733
|
-
console.log(` Datasets: ${
|
|
8734
|
-
console.log(` Evaluations: ${
|
|
8735
|
-
console.log(` Optimizations: ${
|
|
8736
|
-
console.log(` Experiments: ${
|
|
9149
|
+
console.log(chalk36.bold("Resources:"));
|
|
9150
|
+
console.log(` Prompts: ${chalk36.cyan(String(promptCount))}`);
|
|
9151
|
+
console.log(` Datasets: ${chalk36.cyan(String(datasetCount))}`);
|
|
9152
|
+
console.log(` Evaluations: ${chalk36.cyan(String(evaluationCount))}`);
|
|
9153
|
+
console.log(` Optimizations: ${chalk36.cyan(String(optimizationCount))}`);
|
|
9154
|
+
console.log(` Experiments: ${chalk36.cyan(String(experimentCount))}`);
|
|
8737
9155
|
console.log("");
|
|
8738
|
-
console.log(` Providers: ${
|
|
9156
|
+
console.log(` Providers: ${chalk36.underline(PROVIDERS_URL)}`);
|
|
8739
9157
|
console.log("");
|
|
8740
9158
|
}
|
|
8741
9159
|
} catch (error) {
|
|
@@ -8747,7 +9165,7 @@ Examples:
|
|
|
8747
9165
|
|
|
8748
9166
|
// src/commands/hooks.ts
|
|
8749
9167
|
init_config();
|
|
8750
|
-
import { Command as
|
|
9168
|
+
import { Command as Command19 } from "commander";
|
|
8751
9169
|
import { randomUUID } from "crypto";
|
|
8752
9170
|
import { join as join8 } from "path";
|
|
8753
9171
|
import { tmpdir } from "os";
|
|
@@ -8988,7 +9406,7 @@ async function handlePostToolUse() {
|
|
|
8988
9406
|
]);
|
|
8989
9407
|
}
|
|
8990
9408
|
function createHooksCommand() {
|
|
8991
|
-
const hooks = new
|
|
9409
|
+
const hooks = new Command19("hooks").description("Hook handlers for AI coding assistants").addHelpText("after", `
|
|
8992
9410
|
Claude Code Session Telemetry:
|
|
8993
9411
|
Sends lightweight session activity to the MutagenT traces API for observability.
|
|
8994
9412
|
|
|
@@ -9022,8 +9440,8 @@ Claude Code Session Telemetry:
|
|
|
9022
9440
|
}
|
|
9023
9441
|
|
|
9024
9442
|
// src/commands/feedback.ts
|
|
9025
|
-
import { Command as
|
|
9026
|
-
import
|
|
9443
|
+
import { Command as Command20 } from "commander";
|
|
9444
|
+
import chalk37 from "chalk";
|
|
9027
9445
|
init_errors();
|
|
9028
9446
|
init_config();
|
|
9029
9447
|
import { readFileSync as readFileSync11 } from "fs";
|
|
@@ -9076,13 +9494,13 @@ async function postToServer(payload, endpoint, apiKey, workspaceId, organization
|
|
|
9076
9494
|
throw new MutagentError("API_ERROR", errorMessage, `Server responded with status ${String(response.status)}. Check your configuration with: mutagent config show`);
|
|
9077
9495
|
}
|
|
9078
9496
|
function createFeedbackCommand() {
|
|
9079
|
-
const feedback = new
|
|
9080
|
-
${
|
|
9081
|
-
${
|
|
9082
|
-
${
|
|
9083
|
-
${
|
|
9497
|
+
const feedback = new Command20("feedback").description("Send product feedback to MutagenT").addHelpText("after", `
|
|
9498
|
+
${chalk37.bold("Examples:")}
|
|
9499
|
+
${chalk37.cyan('mutagent feedback send -m "Great optimization results!"')}
|
|
9500
|
+
${chalk37.cyan('mutagent feedback send -m "CLI crashed on export" --category bug')}
|
|
9501
|
+
${chalk37.cyan('mutagent feedback send -m "Need batch operations" --category feature --json')}
|
|
9084
9502
|
|
|
9085
|
-
${
|
|
9503
|
+
${chalk37.yellow("AI Agent (MANDATORY):")}
|
|
9086
9504
|
ALWAYS use --json: mutagent feedback send -m "..." --category improvement --json
|
|
9087
9505
|
Use this command to report bugs, request features, or share UX feedback.
|
|
9088
9506
|
`).action(() => {
|
|
@@ -9093,18 +9511,18 @@ ${chalk34.yellow("AI Agent (MANDATORY):")}
|
|
|
9093
9511
|
}
|
|
9094
9512
|
function registerFeedbackSend(feedback) {
|
|
9095
9513
|
feedback.command("send").description("Send feedback about the MutagenT platform").requiredOption("-m, --message <text>", "Feedback message").option("--category <type>", `Feedback category: ${VALID_CATEGORIES.join(", ")}`, "improvement").option("--session <id>", "Link feedback to a specific session").addHelpText("after", `
|
|
9096
|
-
${
|
|
9097
|
-
${
|
|
9098
|
-
${
|
|
9099
|
-
${
|
|
9100
|
-
|
|
9101
|
-
${
|
|
9102
|
-
${
|
|
9103
|
-
${
|
|
9104
|
-
${
|
|
9105
|
-
${
|
|
9106
|
-
|
|
9107
|
-
${
|
|
9514
|
+
${chalk37.bold("Examples:")}
|
|
9515
|
+
${chalk37.dim("$")} mutagent feedback send -m "The optimization UX could show progress better"
|
|
9516
|
+
${chalk37.dim("$")} mutagent feedback send -m "CLI errored on traces export" --category bug
|
|
9517
|
+
${chalk37.dim("$")} mutagent feedback send -m "Love the guided eval!" --category praise --json
|
|
9518
|
+
|
|
9519
|
+
${chalk37.bold("Categories:")}
|
|
9520
|
+
${chalk37.bold("bug")} Something is broken or not working as expected
|
|
9521
|
+
${chalk37.bold("feature")} Request a new capability
|
|
9522
|
+
${chalk37.bold("improvement")} Suggest a UX or workflow enhancement (default)
|
|
9523
|
+
${chalk37.bold("praise")} Share what you love about the platform
|
|
9524
|
+
|
|
9525
|
+
${chalk37.yellow("AI Agent (MANDATORY):")}
|
|
9108
9526
|
ALWAYS use --json: mutagent feedback send -m "..." --json
|
|
9109
9527
|
Auto-captured context (CLI version, platform, node version) is included automatically.
|
|
9110
9528
|
`).action(async (options) => {
|
|
@@ -9167,7 +9585,7 @@ if (process.env.CLI_VERSION) {
|
|
|
9167
9585
|
cliVersion = pkg.version ?? cliVersion;
|
|
9168
9586
|
} catch {}
|
|
9169
9587
|
}
|
|
9170
|
-
var program = new
|
|
9588
|
+
var program = new Command21;
|
|
9171
9589
|
program.name("mutagent").description(`MutagenT CLI - AI-native prompt optimization platform
|
|
9172
9590
|
|
|
9173
9591
|
Documentation: https://docs.mutagent.io/cli
|
|
@@ -9176,90 +9594,90 @@ program.name("mutagent").description(`MutagenT CLI - AI-native prompt optimizati
|
|
|
9176
9594
|
showGlobalOptions: true
|
|
9177
9595
|
});
|
|
9178
9596
|
program.addHelpText("after", `
|
|
9179
|
-
${
|
|
9180
|
-
export MUTAGENT_API_KEY=mt_... ${
|
|
9181
|
-
--json ${
|
|
9182
|
-
|
|
9183
|
-
${
|
|
9184
|
-
mutagent login ${
|
|
9185
|
-
mutagent auth status ${
|
|
9186
|
-
mutagent init ${
|
|
9187
|
-
mutagent explore ${
|
|
9188
|
-
mutagent workspaces list --json ${
|
|
9189
|
-
mutagent config set workspace <id> ${
|
|
9190
|
-
mutagent usage --json ${
|
|
9191
|
-
|
|
9192
|
-
mutagent prompts create --help ${
|
|
9193
|
-
mutagent prompts list --json ${
|
|
9194
|
-
mutagent prompts get <id> --json ${
|
|
9195
|
-
|
|
9196
|
-
mutagent prompts dataset add --help ${
|
|
9197
|
-
mutagent prompts dataset list <id> ${
|
|
9198
|
-
|
|
9199
|
-
mutagent prompts evaluation create --help ${
|
|
9200
|
-
mutagent prompts evaluation create <id> --guided --json ${
|
|
9201
|
-
mutagent prompts evaluation list <id> --json ${
|
|
9202
|
-
|
|
9203
|
-
mutagent prompts optimize start --help ${
|
|
9204
|
-
mutagent prompts optimize status <job-id> ${
|
|
9205
|
-
mutagent prompts optimize results <job-id> ${
|
|
9206
|
-
|
|
9207
|
-
mutagent feedback send -m "..." ${
|
|
9208
|
-
mutagent feedback send -m "..." --category bug ${
|
|
9209
|
-
|
|
9210
|
-
mutagent integrate <framework> ${
|
|
9211
|
-
mutagent hooks --help ${
|
|
9212
|
-
mutagent playground run <id> --input '{...}' ${
|
|
9213
|
-
|
|
9214
|
-
${
|
|
9215
|
-
1. mutagent explore ${
|
|
9216
|
-
2. mutagent integrate <framework> ${
|
|
9217
|
-
3. Apply tracing code to your codebase ${
|
|
9218
|
-
4. mutagent traces list --json ${
|
|
9219
|
-
|
|
9220
|
-
${
|
|
9221
|
-
1. mutagent prompts create --help ${
|
|
9222
|
-
2. mutagent prompts create ... --json ${
|
|
9223
|
-
3. mutagent prompts dataset add --help ${
|
|
9224
|
-
4. mutagent prompts dataset add <id> ... --json ${
|
|
9225
|
-
5. mutagent prompts evaluation create <id> --guided --json ${
|
|
9597
|
+
${chalk38.yellow("Non-Interactive Mode (CI/CD & Coding Agents):")}
|
|
9598
|
+
export MUTAGENT_API_KEY=mt_... ${chalk38.dim("or")} --api-key mt_...
|
|
9599
|
+
--json ${chalk38.dim("for structured output")} --non-interactive ${chalk38.dim("to disable prompts")}
|
|
9600
|
+
|
|
9601
|
+
${chalk38.yellow("Command Navigation:")}
|
|
9602
|
+
mutagent login ${chalk38.dim("Login (browser OAuth — recommended)")}
|
|
9603
|
+
mutagent auth status ${chalk38.dim("Check auth + workspace")}
|
|
9604
|
+
mutagent init ${chalk38.dim("Initialize project (.mutagentrc.json)")}
|
|
9605
|
+
mutagent explore ${chalk38.dim("Discover prompts in codebase")}
|
|
9606
|
+
mutagent workspaces list --json ${chalk38.dim("List workspaces (verify ID)")}
|
|
9607
|
+
mutagent config set workspace <id> ${chalk38.dim("Set active workspace")}
|
|
9608
|
+
mutagent usage --json ${chalk38.dim("Show account usage + provider status")}
|
|
9609
|
+
|
|
9610
|
+
mutagent prompts create --help ${chalk38.dim("Upload prompt (read help first!)")}
|
|
9611
|
+
mutagent prompts list --json ${chalk38.dim("List prompts")}
|
|
9612
|
+
mutagent prompts get <id> --json ${chalk38.dim("Full prompt details + schemas")}
|
|
9613
|
+
|
|
9614
|
+
mutagent prompts dataset add --help ${chalk38.dim("Upload dataset (read help first!)")}
|
|
9615
|
+
mutagent prompts dataset list <id> ${chalk38.dim("List datasets")}
|
|
9616
|
+
|
|
9617
|
+
mutagent prompts evaluation create --help ${chalk38.dim("Create eval (read help first!)")}
|
|
9618
|
+
mutagent prompts evaluation create <id> --guided --json ${chalk38.dim("Guided eval workflow")}
|
|
9619
|
+
mutagent prompts evaluation list <id> --json ${chalk38.dim("List evaluations")}
|
|
9620
|
+
|
|
9621
|
+
mutagent prompts optimize start --help ${chalk38.dim("Run optimization (read help first!)")}
|
|
9622
|
+
mutagent prompts optimize status <job-id> ${chalk38.dim("Poll progress")}
|
|
9623
|
+
mutagent prompts optimize results <job-id> ${chalk38.dim("View scorecard")}
|
|
9624
|
+
|
|
9625
|
+
mutagent feedback send -m "..." ${chalk38.dim("Send product feedback")}
|
|
9626
|
+
mutagent feedback send -m "..." --category bug ${chalk38.dim("Report a bug")}
|
|
9627
|
+
|
|
9628
|
+
mutagent integrate <framework> ${chalk38.dim("Framework integration guide")}
|
|
9629
|
+
mutagent hooks --help ${chalk38.dim("Hook setup for Claude Code telemetry")}
|
|
9630
|
+
mutagent playground run <id> --input '{...}' ${chalk38.dim("Quick test")}
|
|
9631
|
+
|
|
9632
|
+
${chalk38.yellow("★ Workflow: Framework Integration (Tracing):")}
|
|
9633
|
+
1. mutagent explore ${chalk38.dim("← discover prompts/agents in codebase")}
|
|
9634
|
+
2. mutagent integrate <framework> ${chalk38.dim("← get integration instructions")}
|
|
9635
|
+
3. Apply tracing code to your codebase ${chalk38.dim("← follow the guide output")}
|
|
9636
|
+
4. mutagent traces list --json ${chalk38.dim("← verify traces are arriving")}
|
|
9637
|
+
|
|
9638
|
+
${chalk38.yellow("★ Workflow: Evaluate → Optimize:")}
|
|
9639
|
+
1. mutagent prompts create --help ${chalk38.dim("← read help")}
|
|
9640
|
+
2. mutagent prompts create ... --json ${chalk38.dim("← upload prompt with {variables} + inputSchema")}
|
|
9641
|
+
3. mutagent prompts dataset add --help ${chalk38.dim("← read help")}
|
|
9642
|
+
4. mutagent prompts dataset add <id> ... --json ${chalk38.dim("← upload dataset")}
|
|
9643
|
+
5. mutagent prompts evaluation create <id> --guided --json ${chalk38.dim("← guided eval")}
|
|
9226
9644
|
6. mutagent prompts optimize start <id> --dataset <d> --evaluation <e> --json
|
|
9227
9645
|
|
|
9228
|
-
${
|
|
9229
|
-
After ${
|
|
9230
|
-
${
|
|
9231
|
-
${
|
|
9232
|
-
${
|
|
9646
|
+
${chalk38.yellow("Post-Onboarding Decision Tree:")}
|
|
9647
|
+
After ${chalk38.bold("mutagent auth login")}, users land in one of 3 paths:
|
|
9648
|
+
${chalk38.bold("Path A")} (Tracing): explore → integrate <framework> → apply tracing → verify
|
|
9649
|
+
${chalk38.bold("Path B")} (Optimization): explore → prompts create → dataset add → eval create → optimize
|
|
9650
|
+
${chalk38.bold("Path C")} (Manual): Use CLI commands directly — run mutagent <command> --help
|
|
9233
9651
|
|
|
9234
|
-
${
|
|
9652
|
+
${chalk38.yellow("Directive System:")}
|
|
9235
9653
|
Every --json response may include:
|
|
9236
|
-
${
|
|
9237
|
-
${
|
|
9238
|
-
${
|
|
9239
|
-
${
|
|
9654
|
+
${chalk38.bold("_directive.renderedCard")} Pre-formatted card for the user ${chalk38.red("(MUST be shown in chat)")}
|
|
9655
|
+
${chalk38.bold("_directive.instruction")} Next step for the agent
|
|
9656
|
+
${chalk38.bold("_directive.next")} Array of suggested follow-up commands
|
|
9657
|
+
${chalk38.bold("_links")} Dashboard/API URLs (format as markdown links)
|
|
9240
9658
|
|
|
9241
|
-
${
|
|
9242
|
-
Each criterion MUST have: ${
|
|
9659
|
+
${chalk38.yellow("Evaluation Criteria Format:")}
|
|
9660
|
+
Each criterion MUST have: ${chalk38.bold("name")}, ${chalk38.bold("description")} (scoring rubric), ${chalk38.bold("evaluationParameter")}
|
|
9243
9661
|
evaluationParameter MUST match an inputSchema or outputSchema field name
|
|
9244
9662
|
No duplicate evaluationParameter values — each criterion targets a unique field
|
|
9245
9663
|
ALL schema fields must be covered (missing fields = error)
|
|
9246
|
-
Use ${
|
|
9664
|
+
Use ${chalk38.bold("--guided --json")} to generate criteria templates from prompt schemas
|
|
9247
9665
|
|
|
9248
|
-
${
|
|
9249
|
-
Default max-iterations is 1. ${
|
|
9666
|
+
${chalk38.yellow("Optimization Cost Control:")}
|
|
9667
|
+
Default max-iterations is 1. ${chalk38.red("NEVER increase without explicit user request.")}
|
|
9250
9668
|
Each iteration incurs LLM costs — confirm with user before starting >1.
|
|
9251
9669
|
|
|
9252
|
-
${
|
|
9253
|
-
After ${
|
|
9254
|
-
Then offer choices: ${
|
|
9670
|
+
${chalk38.yellow("Post-Optimization:")}
|
|
9671
|
+
After ${chalk38.bold("optimize results")}: ALWAYS show the before/after diff to the user first.
|
|
9672
|
+
Then offer choices: ${chalk38.bold("Apply")} / ${chalk38.bold("Reject")}.
|
|
9255
9673
|
|
|
9256
|
-
${
|
|
9674
|
+
${chalk38.yellow("State Tracking:")}
|
|
9257
9675
|
.mutagent/mutation-context.md — Codebase index of discovered/uploaded prompts
|
|
9258
9676
|
Update after explore, create, and dataset operations
|
|
9259
9677
|
mutagent auth status — Auth + workspace state
|
|
9260
9678
|
Comment markers (// MutagenT:START ... // MutagenT:END) in source files
|
|
9261
9679
|
|
|
9262
|
-
${
|
|
9680
|
+
${chalk38.yellow("AI Agent Rules (MANDATORY for coding agents):")}
|
|
9263
9681
|
1. Login (two paths):
|
|
9264
9682
|
- CI / fully automated: export MUTAGENT_API_KEY=mt_... then mutagent login --json
|
|
9265
9683
|
- Helping a user onboard: mutagent login --browser --json
|
|
@@ -9270,15 +9688,15 @@ ${chalk35.yellow("AI Agent Rules (MANDATORY for coding agents):")}
|
|
|
9270
9688
|
3. Run <command> --help BEFORE first use of any command
|
|
9271
9689
|
4. Use --guided --json for evaluation creation (NEVER --guided alone)
|
|
9272
9690
|
5. Parse _directive.renderedCard and copy it into your CHAT RESPONSE verbatim
|
|
9273
|
-
${
|
|
9691
|
+
${chalk38.red("HARD STOP")}: do NOT run further commands until the card is rendered in chat
|
|
9274
9692
|
6. After mutagent init, verify workspace: mutagent workspaces list --json
|
|
9275
9693
|
7. Use {single_braces} for template variables in prompts
|
|
9276
9694
|
8. Collect evaluation criteria from the user — NEVER auto-generate
|
|
9277
9695
|
9. ALL user interaction via AskUserQuestion — CLI is non-interactive
|
|
9278
9696
|
${!hasCredentials() ? `
|
|
9279
|
-
` +
|
|
9697
|
+
` + chalk38.yellow(" Warning: Not authenticated. Run: mutagent login") + `
|
|
9280
9698
|
` : ""}${!hasRcConfig() ? `
|
|
9281
|
-
` +
|
|
9699
|
+
` + chalk38.green(" Get started: mutagent init") + `
|
|
9282
9700
|
` : ""}`);
|
|
9283
9701
|
var rawArgs = process.argv.slice(2);
|
|
9284
9702
|
if (rawArgs.includes("-v") || rawArgs.includes("--version")) {
|
|
@@ -9319,5 +9737,5 @@ program.addCommand(createHooksCommand());
|
|
|
9319
9737
|
program.addCommand(createFeedbackCommand());
|
|
9320
9738
|
program.parse();
|
|
9321
9739
|
|
|
9322
|
-
//# debugId=
|
|
9740
|
+
//# debugId=C3949CD3A9488F4264756E2164756E21
|
|
9323
9741
|
//# sourceMappingURL=cli.js.map
|