@mutagent/cli 0.1.113 → 0.1.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cli.js +995 -528
- package/dist/bin/cli.js.map +17 -11
- package/dist/index.js +66 -7
- package/dist/index.js.map +3 -3
- package/package.json +1 -1
package/dist/bin/cli.js
CHANGED
|
@@ -756,15 +756,74 @@ class SDKClientWrapper {
|
|
|
756
756
|
this.handleError(error);
|
|
757
757
|
}
|
|
758
758
|
}
|
|
759
|
-
async listExperiments() {
|
|
759
|
+
async listExperiments(filters) {
|
|
760
760
|
try {
|
|
761
|
-
const
|
|
761
|
+
const params = new URLSearchParams;
|
|
762
|
+
if (filters?.promptId !== undefined)
|
|
763
|
+
params.set("promptId", String(filters.promptId));
|
|
764
|
+
if (filters?.status)
|
|
765
|
+
params.set("status", filters.status);
|
|
766
|
+
if (filters?.limit !== undefined)
|
|
767
|
+
params.set("limit", String(filters.limit));
|
|
768
|
+
if (filters?.offset !== undefined)
|
|
769
|
+
params.set("offset", String(filters.offset));
|
|
770
|
+
const qs = params.toString();
|
|
771
|
+
const response = await this.request(`/api/prompts/experiments${qs ? `?${qs}` : ""}`);
|
|
762
772
|
if (Array.isArray(response))
|
|
763
|
-
return response;
|
|
773
|
+
return { experiments: response, total: response.length };
|
|
764
774
|
const r = response;
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
775
|
+
const experiments = r.experiments ?? r.data ?? [];
|
|
776
|
+
return { experiments, total: r.total ?? experiments.length };
|
|
777
|
+
} catch (error) {
|
|
778
|
+
this.handleError(error);
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
async createExperiment(data) {
|
|
782
|
+
try {
|
|
783
|
+
return await this.request("/api/prompts/experiments", {
|
|
784
|
+
method: "POST",
|
|
785
|
+
body: JSON.stringify(data)
|
|
786
|
+
});
|
|
787
|
+
} catch (error) {
|
|
788
|
+
this.handleError(error);
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
async getExperiment(id) {
|
|
792
|
+
try {
|
|
793
|
+
return await this.request(`/api/prompts/experiments/${id}`);
|
|
794
|
+
} catch (error) {
|
|
795
|
+
this.handleError(error);
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
async deleteExperiment(id) {
|
|
799
|
+
try {
|
|
800
|
+
await this.request(`/api/prompts/experiments/${id}`, { method: "DELETE" });
|
|
801
|
+
} catch (error) {
|
|
802
|
+
this.handleError(error);
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
async executeExperiment(id) {
|
|
806
|
+
try {
|
|
807
|
+
return await this.request(`/api/prompts/experiments/${id}/execute`, { method: "POST", body: JSON.stringify({}) });
|
|
808
|
+
} catch (error) {
|
|
809
|
+
this.handleError(error);
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
async playgroundEval(promptId, data) {
|
|
813
|
+
try {
|
|
814
|
+
return await this.request(`/api/prompt/${promptId}/playground/eval`, {
|
|
815
|
+
method: "POST",
|
|
816
|
+
body: JSON.stringify(data)
|
|
817
|
+
});
|
|
818
|
+
} catch (error) {
|
|
819
|
+
this.handleError(error);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
async cancelOptimization(jobId) {
|
|
823
|
+
try {
|
|
824
|
+
return await this.request(`/api/optimization/${jobId}/cancel`, {
|
|
825
|
+
method: "POST"
|
|
826
|
+
});
|
|
768
827
|
} catch (error) {
|
|
769
828
|
this.handleError(error);
|
|
770
829
|
}
|
|
@@ -1181,8 +1240,8 @@ var init_sdk_client = __esm(() => {
|
|
|
1181
1240
|
});
|
|
1182
1241
|
|
|
1183
1242
|
// src/bin/cli.ts
|
|
1184
|
-
import { Command as
|
|
1185
|
-
import
|
|
1243
|
+
import { Command as Command21 } from "commander";
|
|
1244
|
+
import chalk38 from "chalk";
|
|
1186
1245
|
import { readFileSync as readFileSync12 } from "fs";
|
|
1187
1246
|
import { join as join10, dirname as dirname2 } from "path";
|
|
1188
1247
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
@@ -2569,8 +2628,8 @@ ${chalk5.dim("Note: `mutagent auth login` is a back-compat alias and behaves ide
|
|
|
2569
2628
|
// src/commands/prompts/index.ts
|
|
2570
2629
|
init_errors();
|
|
2571
2630
|
init_sdk_client();
|
|
2572
|
-
import { Command as
|
|
2573
|
-
import
|
|
2631
|
+
import { Command as Command7 } from "commander";
|
|
2632
|
+
import chalk18 from "chalk";
|
|
2574
2633
|
import { readFileSync as readFileSync4, existsSync as existsSync4 } from "fs";
|
|
2575
2634
|
|
|
2576
2635
|
// src/lib/ui-links.ts
|
|
@@ -3621,6 +3680,26 @@ ${chalk8.red("Required: --data must be provided.")}
|
|
|
3621
3680
|
throw new MutagentError("INVALID_JSON", "Invalid JSON in -d/--data flag", `Run: mutagent prompts dataset add --help
|
|
3622
3681
|
` + `Provide a valid JSON array, e.g., '[{"input": {...}, "expectedOutput": {...}}]'`);
|
|
3623
3682
|
}
|
|
3683
|
+
const parsedItems = JSON.parse(content);
|
|
3684
|
+
if (parsedItems.length === 0) {
|
|
3685
|
+
throw new MutagentError("VALIDATION_ERROR", "Dataset cannot be empty", `Run: mutagent prompts dataset add --help
|
|
3686
|
+
Provide at least one item in the array.`);
|
|
3687
|
+
}
|
|
3688
|
+
const warnings = [];
|
|
3689
|
+
for (let i = 0;i < parsedItems.length; i++) {
|
|
3690
|
+
const item = parsedItems[i];
|
|
3691
|
+
if (!("input" in item)) {
|
|
3692
|
+
throw new MutagentError("VALIDATION_ERROR", `Dataset item at index ${i} is missing required field "input"`, `Run: mutagent prompts dataset add --help
|
|
3693
|
+
Each item must have: {"input": {...}, "expectedOutput": {...}}`);
|
|
3694
|
+
}
|
|
3695
|
+
if (typeof item.input !== "object" || item.input === null || Array.isArray(item.input)) {
|
|
3696
|
+
throw new MutagentError("VALIDATION_ERROR", `Dataset item at index ${i}: "input" must be an object, got ${Array.isArray(item.input) ? "array" : typeof item.input}`, `Run: mutagent prompts dataset add --help
|
|
3697
|
+
"input" must be a JSON object: {"input": {"field": "value"}}`);
|
|
3698
|
+
}
|
|
3699
|
+
if (!("expectedOutput" in item)) {
|
|
3700
|
+
warnings.push(`Item at index ${i} is missing "expectedOutput" (recommended for evaluation scoring)`);
|
|
3701
|
+
}
|
|
3702
|
+
}
|
|
3624
3703
|
const datasetName = options.name;
|
|
3625
3704
|
if (!datasetName) {
|
|
3626
3705
|
throw new MutagentError("MISSING_ARGUMENTS", "Dataset name is required", `Run: mutagent prompts dataset add --help
|
|
@@ -3643,6 +3722,9 @@ ${chalk8.red("Required: --data must be provided.")}
|
|
|
3643
3722
|
});
|
|
3644
3723
|
echoDirectiveToStderr(directive);
|
|
3645
3724
|
} else {
|
|
3725
|
+
for (const w of warnings) {
|
|
3726
|
+
output.warn(w);
|
|
3727
|
+
}
|
|
3646
3728
|
output.success(`Added dataset "${datasetResult.name}" to prompt: ${promptId} (id: ${String(datasetResult.id)})`);
|
|
3647
3729
|
if (datasetResult.itemCount !== undefined && datasetResult.itemCount > 0) {
|
|
3648
3730
|
output.info(`Items uploaded: ${String(datasetResult.itemCount)}`);
|
|
@@ -3774,7 +3856,7 @@ Verify the item ID exists. Use the dashboard or dataset exports to find valid it
|
|
|
3774
3856
|
// src/commands/prompts/evaluations.ts
|
|
3775
3857
|
init_sdk_client();
|
|
3776
3858
|
import { Command as Command4 } from "commander";
|
|
3777
|
-
import
|
|
3859
|
+
import chalk10 from "chalk";
|
|
3778
3860
|
init_errors();
|
|
3779
3861
|
|
|
3780
3862
|
// src/lib/resolve-prompt-id.ts
|
|
@@ -3938,22 +4020,106 @@ function canonicalCriteriaArrayToCli(arr) {
|
|
|
3938
4020
|
return arr.map(canonicalCriterionToCli);
|
|
3939
4021
|
}
|
|
3940
4022
|
|
|
4023
|
+
// src/commands/prompts/evaluations-run.ts
|
|
4024
|
+
init_sdk_client();
|
|
4025
|
+
import chalk9 from "chalk";
|
|
4026
|
+
init_errors();
|
|
4027
|
+
function registerEvaluationRunCommand(evaluation, prompts) {
|
|
4028
|
+
evaluation.command("run").description("Run a prompt against an evaluation (playground eval)").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").requiredOption("-e, --evaluation <id>", "Evaluation ID to score against").requiredOption("--input <json>", `Input JSON object for the prompt (e.g. '{"text":"hello"}')`).requiredOption("--exec-model <model>", 'Model to use for generating the response (e.g. "claude-sonnet-4-5-20250929")').option("--eval-model <model>", "Model to use for evaluation scoring (defaults to exec-model)").addHelpText("after", `
|
|
4029
|
+
Examples:
|
|
4030
|
+
${chalk9.dim("$")} mutagent prompts evaluation run <prompt-id> --evaluation <eval-id> --input '{"text":"hello"}' --exec-model claude-sonnet-4-5-20250929
|
|
4031
|
+
${chalk9.dim("$")} mutagent prompts evaluation run <prompt-id> -e <eval-id> --input '{"text":"hello"}' --exec-model claude-sonnet-4-5-20250929 --eval-model gpt-4o
|
|
4032
|
+
${chalk9.dim("$")} mutagent prompts evaluation run <prompt-id> -e <eval-id> --input '{"text":"hello"}' --exec-model claude-sonnet-4-5-20250929 --json
|
|
4033
|
+
|
|
4034
|
+
${chalk9.yellow("AI Agent note:")} Always pass --json for structured output.
|
|
4035
|
+
${chalk9.dim("Response includes evaluation score, pass/fail, token usage, and dashboard links.")}
|
|
4036
|
+
`).action(async (promptId, options) => {
|
|
4037
|
+
const isJson = getJsonFlag(prompts);
|
|
4038
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
4039
|
+
try {
|
|
4040
|
+
let parsedInput;
|
|
4041
|
+
try {
|
|
4042
|
+
const raw = JSON.parse(options.input);
|
|
4043
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
|
|
4044
|
+
throw new MutagentError("INVALID_JSON", `--input must be a JSON object, got ${Array.isArray(raw) ? "array" : typeof raw}`, `Run: mutagent prompts evaluation run --help
|
|
4045
|
+
` + `Provide a JSON object: --input '{"field": "value"}'`);
|
|
4046
|
+
}
|
|
4047
|
+
parsedInput = raw;
|
|
4048
|
+
} catch (e) {
|
|
4049
|
+
if (e instanceof MutagentError)
|
|
4050
|
+
throw e;
|
|
4051
|
+
throw new MutagentError("INVALID_JSON", "Invalid JSON in --input flag", `Run: mutagent prompts evaluation run --help
|
|
4052
|
+
` + `Provide valid JSON: --input '{"text": "hello"}'`);
|
|
4053
|
+
}
|
|
4054
|
+
const evaluationId = parseInt(options.evaluation, 10);
|
|
4055
|
+
if (isNaN(evaluationId)) {
|
|
4056
|
+
throw new MutagentError("VALIDATION_ERROR", `Invalid evaluation ID: "${options.evaluation}" (must be numeric)`, "Run: mutagent prompts evaluation list <prompt-id> to find valid IDs");
|
|
4057
|
+
}
|
|
4058
|
+
const llmConfig = {
|
|
4059
|
+
model: options.execModel,
|
|
4060
|
+
...options.evalModel ? { evalModel: options.evalModel } : {}
|
|
4061
|
+
};
|
|
4062
|
+
const client = await getSDKClient();
|
|
4063
|
+
const result = await client.playgroundEval(promptId, {
|
|
4064
|
+
input: parsedInput,
|
|
4065
|
+
evaluationId,
|
|
4066
|
+
llmConfig
|
|
4067
|
+
});
|
|
4068
|
+
if (isJson) {
|
|
4069
|
+
output.output({
|
|
4070
|
+
...result,
|
|
4071
|
+
_links: {
|
|
4072
|
+
dashboard: playgroundLink(promptId),
|
|
4073
|
+
api: `/api/prompt/${promptId}/playground/eval`
|
|
4074
|
+
}
|
|
4075
|
+
});
|
|
4076
|
+
} else {
|
|
4077
|
+
const evalResult = result.evaluation;
|
|
4078
|
+
const status = evalResult.passed ? chalk9.green("PASSED") : chalk9.red("FAILED");
|
|
4079
|
+
output.success(`Evaluation complete: ${status}`);
|
|
4080
|
+
output.info(`Score: ${evalResult.score.toFixed(3)}`);
|
|
4081
|
+
if (evalResult.reasoning) {
|
|
4082
|
+
output.info(`Reasoning: ${evalResult.reasoning}`);
|
|
4083
|
+
}
|
|
4084
|
+
if (typeof result.response === "string") {
|
|
4085
|
+
output.info(`Response: ${result.response.substring(0, 200)}${result.response.length > 200 ? "..." : ""}`);
|
|
4086
|
+
}
|
|
4087
|
+
output.info(`Latency: ${String(result.metadata.latencyMs)}ms`);
|
|
4088
|
+
if (result.metadata.tokens) {
|
|
4089
|
+
const t = result.metadata.tokens;
|
|
4090
|
+
output.info(`Tokens: prompt=${String(t.prompt ?? 0)}, completion=${String(t.completion ?? 0)}`);
|
|
4091
|
+
}
|
|
4092
|
+
if (evalResult.criteria && evalResult.criteria.length > 0) {
|
|
4093
|
+
output.info("Criteria scores:");
|
|
4094
|
+
for (const c of evalResult.criteria) {
|
|
4095
|
+
const mark = c.passed ? chalk9.green("✓") : chalk9.red("✗");
|
|
4096
|
+
output.info(` ${mark} ${c.name}: ${c.score.toFixed(3)}`);
|
|
4097
|
+
}
|
|
4098
|
+
}
|
|
4099
|
+
output.info(`Dashboard: ${playgroundLink(promptId)}`);
|
|
4100
|
+
}
|
|
4101
|
+
} catch (error) {
|
|
4102
|
+
handleError(error, isJson);
|
|
4103
|
+
}
|
|
4104
|
+
});
|
|
4105
|
+
}
|
|
4106
|
+
|
|
3941
4107
|
// src/commands/prompts/evaluations.ts
|
|
3942
4108
|
function registerEvaluationCommands(prompts) {
|
|
3943
4109
|
const evaluation = new Command4("evaluation").description("Manage evaluations for prompts").addHelpText("after", `
|
|
3944
4110
|
Examples:
|
|
3945
|
-
${
|
|
3946
|
-
${
|
|
3947
|
-
${
|
|
3948
|
-
${
|
|
4111
|
+
${chalk10.dim("$")} mutagent prompts evaluation list <prompt-id>
|
|
4112
|
+
${chalk10.dim("$")} mutagent prompts evaluation get <evaluation-id>
|
|
4113
|
+
${chalk10.dim("$")} mutagent prompts evaluation create <prompt-id> --name "My Eval"
|
|
4114
|
+
${chalk10.dim("$")} mutagent prompts evaluation delete <evaluation-id>
|
|
3949
4115
|
`).action(() => {
|
|
3950
4116
|
evaluation.help();
|
|
3951
4117
|
});
|
|
3952
4118
|
prompts.addCommand(evaluation);
|
|
3953
4119
|
evaluation.command("list").description("List evaluations for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").addHelpText("after", `
|
|
3954
4120
|
Examples:
|
|
3955
|
-
${
|
|
3956
|
-
${
|
|
4121
|
+
${chalk10.dim("$")} mutagent prompts evaluation list <prompt-id>
|
|
4122
|
+
${chalk10.dim("$")} mutagent prompts evaluation list <prompt-id> --json
|
|
3957
4123
|
`).action(async (promptId) => {
|
|
3958
4124
|
const isJson = getJsonFlag(prompts);
|
|
3959
4125
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3985,8 +4151,8 @@ Examples:
|
|
|
3985
4151
|
});
|
|
3986
4152
|
evaluation.command("get").description("Get evaluation details including criteria").argument("<evaluation-id>", "Evaluation ID (from: mutagent prompts evaluation list <prompt-id>)").addHelpText("after", `
|
|
3987
4153
|
Examples:
|
|
3988
|
-
${
|
|
3989
|
-
${
|
|
4154
|
+
${chalk10.dim("$")} mutagent prompts evaluation get <evaluation-id>
|
|
4155
|
+
${chalk10.dim("$")} mutagent prompts evaluation get <evaluation-id> --json
|
|
3990
4156
|
`).action(async (evaluationId) => {
|
|
3991
4157
|
const isJson = getJsonFlag(prompts);
|
|
3992
4158
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4020,7 +4186,7 @@ Examples:
|
|
|
4020
4186
|
if (criteria.length > 0) {
|
|
4021
4187
|
console.log(" Criteria:");
|
|
4022
4188
|
for (const c of criteria) {
|
|
4023
|
-
console.log(` ${
|
|
4189
|
+
console.log(` ${chalk10.cyan(c.name)}`);
|
|
4024
4190
|
if (c.description) {
|
|
4025
4191
|
console.log(` Description: ${c.description}`);
|
|
4026
4192
|
}
|
|
@@ -4049,9 +4215,9 @@ Examples:
|
|
|
4049
4215
|
});
|
|
4050
4216
|
evaluation.command("create").description("Create an evaluation configuration for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Evaluation as JSON string (for pre-validated criteria only)").option("-n, --name <name>", "Evaluation name (required unless --guided)").option("--description <text>", "Evaluation description").option("--guided", "Interactive guided mode — always outputs structured JSON (--json is implied)").addHelpText("after", `
|
|
4051
4217
|
Examples:
|
|
4052
|
-
${
|
|
4053
|
-
${
|
|
4054
|
-
${
|
|
4218
|
+
${chalk10.dim("$")} mutagent prompts evaluation create <prompt-id> --guided ${chalk10.dim("# recommended: shows workflow guide + schema fields")}
|
|
4219
|
+
${chalk10.dim("$")} mutagent prompts evaluation create <prompt-id> --guided --json ${chalk10.dim("# structured workflow for AI agents")}
|
|
4220
|
+
${chalk10.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Accuracy" -d '{"evalConfig":{"criteria":[...]}}' ${chalk10.dim("# power user")}
|
|
4055
4221
|
|
|
4056
4222
|
Guided Workflow (recommended):
|
|
4057
4223
|
--guided outputs a workflow guide that:
|
|
@@ -4072,12 +4238,12 @@ AI Agent (MANDATORY):
|
|
|
4072
4238
|
mutagent prompts evaluation create <id> --name "<name>" -d '<json>' --json
|
|
4073
4239
|
|
|
4074
4240
|
Expected Criteria Shape (--data):
|
|
4075
|
-
${
|
|
4241
|
+
${chalk10.dim('{"evalConfig":{"criteria":[{"name":"<name>","description":"<scoring rubric>","evaluationParameter":"<schema field>"}]}}')}
|
|
4076
4242
|
evaluationParameter must target an outputSchema OR inputSchema field.
|
|
4077
4243
|
|
|
4078
|
-
${
|
|
4079
|
-
${
|
|
4080
|
-
${
|
|
4244
|
+
${chalk10.red("Required: --name (unless --guided). Criteria must include evaluationParameter.")}
|
|
4245
|
+
${chalk10.dim("CLI flags (--name, --description) override --data fields.")}
|
|
4246
|
+
${chalk10.dim("Get prompt IDs: mutagent prompts list")}
|
|
4081
4247
|
`).action(async (promptId, options) => {
|
|
4082
4248
|
let isJson = getJsonFlag(prompts);
|
|
4083
4249
|
if (options.guided) {
|
|
@@ -4103,7 +4269,7 @@ ${chalk9.dim("Get prompt IDs: mutagent prompts list")}
|
|
|
4103
4269
|
console.log("");
|
|
4104
4270
|
console.log(" For each field, define what correct output looks like:");
|
|
4105
4271
|
for (const { field, source } of allFields) {
|
|
4106
|
-
console.log(` ${
|
|
4272
|
+
console.log(` ${chalk10.cyan(field)} (${source})`);
|
|
4107
4273
|
console.log(` → What makes a correct vs incorrect "${field}"?`);
|
|
4108
4274
|
}
|
|
4109
4275
|
console.log("");
|
|
@@ -4308,9 +4474,9 @@ Example:
|
|
|
4308
4474
|
});
|
|
4309
4475
|
evaluation.command("delete").description("Delete an evaluation").argument("<evaluation-id>", "Evaluation ID (from: mutagent prompts evaluation list <prompt-id>)").option("--force", "Skip confirmation").addHelpText("after", `
|
|
4310
4476
|
Examples:
|
|
4311
|
-
${
|
|
4312
|
-
${
|
|
4313
|
-
${
|
|
4477
|
+
${chalk10.dim("$")} mutagent prompts evaluation delete <evaluation-id>
|
|
4478
|
+
${chalk10.dim("$")} mutagent prompts evaluation delete <evaluation-id> --force
|
|
4479
|
+
${chalk10.dim("$")} mutagent prompts evaluation delete <evaluation-id> --force --json
|
|
4314
4480
|
`).action(async (evaluationId, options) => {
|
|
4315
4481
|
const isJson = getJsonFlag(prompts);
|
|
4316
4482
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4348,31 +4514,32 @@ Examples:
|
|
|
4348
4514
|
handleError(error, isJson);
|
|
4349
4515
|
}
|
|
4350
4516
|
});
|
|
4517
|
+
registerEvaluationRunCommand(evaluation, prompts);
|
|
4351
4518
|
}
|
|
4352
4519
|
|
|
4353
4520
|
// src/commands/prompts/optimize.ts
|
|
4354
4521
|
init_sdk_client();
|
|
4355
4522
|
import { Command as Command5 } from "commander";
|
|
4356
|
-
import
|
|
4523
|
+
import chalk16 from "chalk";
|
|
4357
4524
|
init_errors();
|
|
4358
4525
|
|
|
4359
4526
|
// src/lib/scorecard.ts
|
|
4360
|
-
import
|
|
4527
|
+
import chalk11 from "chalk";
|
|
4361
4528
|
function formatScoreChange(before, after) {
|
|
4362
4529
|
if (before === undefined || after === undefined)
|
|
4363
4530
|
return "";
|
|
4364
4531
|
const diff = after - before;
|
|
4365
4532
|
const pct = before > 0 ? Math.round(diff / before * 100) : 0;
|
|
4366
4533
|
if (diff > 0)
|
|
4367
|
-
return
|
|
4534
|
+
return chalk11.green(` (+${String(pct)}%)`);
|
|
4368
4535
|
if (diff < 0)
|
|
4369
|
-
return
|
|
4370
|
-
return
|
|
4536
|
+
return chalk11.red(` (${String(pct)}%)`);
|
|
4537
|
+
return chalk11.dim(" (no change)");
|
|
4371
4538
|
}
|
|
4372
4539
|
function formatScore(score) {
|
|
4373
4540
|
if (score === undefined)
|
|
4374
|
-
return
|
|
4375
|
-
return score >= 0.8 ?
|
|
4541
|
+
return chalk11.dim("N/A");
|
|
4542
|
+
return score >= 0.8 ? chalk11.green(score.toFixed(2)) : score >= 0.5 ? chalk11.yellow(score.toFixed(2)) : chalk11.red(score.toFixed(2));
|
|
4376
4543
|
}
|
|
4377
4544
|
function renderScorecard(data) {
|
|
4378
4545
|
const { job, prompt } = data;
|
|
@@ -4393,17 +4560,17 @@ function renderScorecard(data) {
|
|
|
4393
4560
|
const optimizedText = prompt.systemPrompt ?? prompt.rawPrompt ?? prompt.humanPrompt ?? "(optimized prompt)";
|
|
4394
4561
|
console.log("");
|
|
4395
4562
|
console.log(topBorder);
|
|
4396
|
-
console.log(line(
|
|
4563
|
+
console.log(line(chalk11.bold("Optimization Results")));
|
|
4397
4564
|
console.log(separator);
|
|
4398
|
-
console.log(line(
|
|
4565
|
+
console.log(line(chalk11.dim("BEFORE")));
|
|
4399
4566
|
console.log(line(` Score: ${formatScore(originalScore)}`));
|
|
4400
4567
|
console.log(line(""));
|
|
4401
|
-
console.log(line(
|
|
4568
|
+
console.log(line(chalk11.bold("AFTER")));
|
|
4402
4569
|
console.log(line(` Score: ${formatScore(bestScore)}${formatScoreChange(originalScore, bestScore)}`));
|
|
4403
4570
|
console.log(separator);
|
|
4404
4571
|
if (data.criteriaScores && data.criteriaScores.length > 0) {
|
|
4405
|
-
console.log(line(
|
|
4406
|
-
console.log(line(
|
|
4572
|
+
console.log(line(chalk11.dim(" Criterion Before After Change")));
|
|
4573
|
+
console.log(line(chalk11.dim(" " + "─".repeat(45))));
|
|
4407
4574
|
for (const c of data.criteriaScores) {
|
|
4408
4575
|
const name = c.name.length > 16 ? c.name.substring(0, 13) + "..." : c.name;
|
|
4409
4576
|
const paddedName = name + " ".repeat(18 - name.length);
|
|
@@ -4412,66 +4579,66 @@ function renderScorecard(data) {
|
|
|
4412
4579
|
const changeStr = c.before !== undefined && c.after !== undefined && c.before > 0 ? (() => {
|
|
4413
4580
|
const pct = Math.round((c.after - c.before) / c.before * 100);
|
|
4414
4581
|
if (pct > 0)
|
|
4415
|
-
return
|
|
4582
|
+
return chalk11.green(`+${String(pct)}%`);
|
|
4416
4583
|
if (pct < 0)
|
|
4417
|
-
return
|
|
4418
|
-
return
|
|
4584
|
+
return chalk11.red(`${String(pct)}%`);
|
|
4585
|
+
return chalk11.dim("0%");
|
|
4419
4586
|
})() : "";
|
|
4420
4587
|
console.log(line(` ${paddedName}${beforeStr} ${afterStr} ${changeStr}`));
|
|
4421
4588
|
}
|
|
4422
|
-
console.log(line(
|
|
4589
|
+
console.log(line(chalk11.dim(" " + "─".repeat(45))));
|
|
4423
4590
|
const overallBefore = originalScore !== undefined ? originalScore.toFixed(2) : "N/A ";
|
|
4424
4591
|
const overallAfter = bestScore !== undefined ? bestScore.toFixed(2) : "N/A ";
|
|
4425
4592
|
const overallChange = originalScore !== undefined && bestScore !== undefined && originalScore > 0 ? (() => {
|
|
4426
4593
|
const pct = Math.round((bestScore - originalScore) / originalScore * 100);
|
|
4427
4594
|
if (pct > 0)
|
|
4428
|
-
return
|
|
4595
|
+
return chalk11.green(`+${String(pct)}%`);
|
|
4429
4596
|
if (pct < 0)
|
|
4430
|
-
return
|
|
4431
|
-
return
|
|
4597
|
+
return chalk11.red(`${String(pct)}%`);
|
|
4598
|
+
return chalk11.dim("0%");
|
|
4432
4599
|
})() : "";
|
|
4433
4600
|
console.log(line(` ${"Overall" + " ".repeat(11)}${overallBefore} ${overallAfter} ${overallChange}`));
|
|
4434
4601
|
console.log(separator);
|
|
4435
4602
|
}
|
|
4436
|
-
const statusStr = job.status === "completed" ?
|
|
4603
|
+
const statusStr = job.status === "completed" ? chalk11.green("completed") : chalk11.yellow(job.status);
|
|
4437
4604
|
console.log(line(`Status: ${statusStr} | Iterations: ${String(iterations)}`));
|
|
4438
4605
|
if (job.config?.model) {
|
|
4439
|
-
console.log(line(`Model: ${
|
|
4606
|
+
console.log(line(`Model: ${chalk11.dim(job.config.model)}`));
|
|
4440
4607
|
}
|
|
4441
4608
|
if (data.scoreProgression && data.scoreProgression.length > 0) {
|
|
4442
4609
|
console.log(line(""));
|
|
4443
|
-
console.log(line(
|
|
4610
|
+
console.log(line(chalk11.dim("Score Progression:")));
|
|
4444
4611
|
const barWidth = 10;
|
|
4445
4612
|
for (let i = 0;i < data.scoreProgression.length; i++) {
|
|
4446
4613
|
const s = data.scoreProgression[i] ?? 0;
|
|
4447
4614
|
const filled = Math.round(s * barWidth);
|
|
4448
4615
|
const bar = "█".repeat(filled) + "░".repeat(barWidth - filled);
|
|
4449
|
-
console.log(line(
|
|
4616
|
+
console.log(line(chalk11.dim(` #${String(i + 1)}: ${bar} ${s.toFixed(2)}`)));
|
|
4450
4617
|
}
|
|
4451
4618
|
}
|
|
4452
4619
|
console.log(separator);
|
|
4453
|
-
console.log(line(`Dashboard: ${
|
|
4620
|
+
console.log(line(`Dashboard: ${chalk11.underline(optimizerLink(job.promptId, job.id))}`));
|
|
4454
4621
|
console.log(bottomBorder);
|
|
4455
4622
|
console.log("");
|
|
4456
|
-
console.log(
|
|
4457
|
-
console.log(
|
|
4458
|
-
console.log(
|
|
4623
|
+
console.log(chalk11.dim(" Prompt Comparison"));
|
|
4624
|
+
console.log(chalk11.dim(" " + "─".repeat(70)));
|
|
4625
|
+
console.log(chalk11.dim(" BEFORE:"));
|
|
4459
4626
|
for (const pLine of originalText.split(`
|
|
4460
4627
|
`)) {
|
|
4461
|
-
console.log(
|
|
4628
|
+
console.log(chalk11.dim(` ${pLine}`));
|
|
4462
4629
|
}
|
|
4463
|
-
console.log(` ${
|
|
4630
|
+
console.log(` ${chalk11.dim("Length:")} ${String(originalText.length)} chars (${String(originalText.split(`
|
|
4464
4631
|
`).length)} lines)`);
|
|
4465
4632
|
console.log("");
|
|
4466
|
-
console.log(
|
|
4633
|
+
console.log(chalk11.bold(" AFTER:"));
|
|
4467
4634
|
for (const pLine of optimizedText.split(`
|
|
4468
4635
|
`)) {
|
|
4469
|
-
console.log(
|
|
4636
|
+
console.log(chalk11.cyan(` ${pLine}`));
|
|
4470
4637
|
}
|
|
4471
|
-
console.log(` ${
|
|
4638
|
+
console.log(` ${chalk11.dim("Length:")} ${String(optimizedText.length)} chars (${String(optimizedText.split(`
|
|
4472
4639
|
`).length)} lines)`);
|
|
4473
4640
|
const growth = optimizedText.length - originalText.length;
|
|
4474
|
-
console.log(` ${
|
|
4641
|
+
console.log(` ${chalk11.dim("Growth:")} ${growth >= 0 ? "+" : ""}${String(growth)} chars`);
|
|
4475
4642
|
renderScorecardDetails(data);
|
|
4476
4643
|
console.log("");
|
|
4477
4644
|
}
|
|
@@ -4492,17 +4659,17 @@ function renderOptimizationStartCard(data) {
|
|
|
4492
4659
|
const target = job.config.targetScore ?? 0.8;
|
|
4493
4660
|
console.log("");
|
|
4494
4661
|
console.log(topBorder);
|
|
4495
|
-
console.log(line(
|
|
4662
|
+
console.log(line(chalk11.bold("⚡ Optimization Started")));
|
|
4496
4663
|
console.log(separator);
|
|
4497
|
-
console.log(line(`Job ID: ${
|
|
4498
|
-
console.log(line(`Prompt: ${
|
|
4499
|
-
console.log(line(`Dataset: ${
|
|
4500
|
-
console.log(line(`Iterations: ${
|
|
4501
|
-
console.log(line(`Model: ${
|
|
4502
|
-
console.log(line(`Status: ${
|
|
4664
|
+
console.log(line(`Job ID: ${chalk11.cyan(job.id)}`));
|
|
4665
|
+
console.log(line(`Prompt: ${chalk11.dim(data.promptId)}`));
|
|
4666
|
+
console.log(line(`Dataset: ${chalk11.dim(data.datasetId)}`));
|
|
4667
|
+
console.log(line(`Iterations: ${chalk11.bold(String(maxIter))} | Target: ${chalk11.bold(target.toFixed(2))}`));
|
|
4668
|
+
console.log(line(`Model: ${chalk11.dim(model)}`));
|
|
4669
|
+
console.log(line(`Status: ${chalk11.yellow(job.status)}`));
|
|
4503
4670
|
console.log(separator);
|
|
4504
|
-
console.log(line(`\uD83D\uDD17 Monitor: ${
|
|
4505
|
-
console.log(line(
|
|
4671
|
+
console.log(line(`\uD83D\uDD17 Monitor: ${chalk11.underline(optimizerLink(data.promptId, job.id))}`));
|
|
4672
|
+
console.log(line(chalk11.dim(`Next: mutagent prompts optimize status ${job.id}`)));
|
|
4506
4673
|
console.log(bottomBorder);
|
|
4507
4674
|
console.log(AI_DIRECTIVE);
|
|
4508
4675
|
console.log("");
|
|
@@ -4522,27 +4689,27 @@ function renderOptimizationStatusCard(status, promptId) {
|
|
|
4522
4689
|
const barWidth = 20;
|
|
4523
4690
|
const filled = Math.round(progress / 100 * barWidth);
|
|
4524
4691
|
const progressBar = "█".repeat(filled) + "░".repeat(barWidth - filled);
|
|
4525
|
-
const statusColor = status.status === "completed" ?
|
|
4526
|
-
const scoreStr = status.bestScore !== undefined ? formatScore(status.bestScore) :
|
|
4692
|
+
const statusColor = status.status === "completed" ? chalk11.green : status.status === "failed" ? chalk11.red : status.status === "cancelled" ? chalk11.gray : status.status === "running" ? chalk11.cyan : chalk11.yellow;
|
|
4693
|
+
const scoreStr = status.bestScore !== undefined ? formatScore(status.bestScore) : chalk11.dim("pending");
|
|
4527
4694
|
console.log("");
|
|
4528
4695
|
console.log(topBorder);
|
|
4529
|
-
console.log(line(
|
|
4696
|
+
console.log(line(chalk11.bold("\uD83D\uDCCA Optimization Status")));
|
|
4530
4697
|
console.log(separator);
|
|
4531
|
-
console.log(line(`Job ID: ${
|
|
4698
|
+
console.log(line(`Job ID: ${chalk11.cyan(status.jobId)}`));
|
|
4532
4699
|
console.log(line(`Status: ${statusColor(status.status)}`));
|
|
4533
|
-
console.log(line(`Iteration: ${
|
|
4700
|
+
console.log(line(`Iteration: ${chalk11.bold(`${String(status.currentIteration)}/${String(status.maxIterations)}`)}`));
|
|
4534
4701
|
console.log(line(`Best Score: ${scoreStr}`));
|
|
4535
4702
|
console.log(line(""));
|
|
4536
4703
|
console.log(line(`Progress: [${progressBar}] ${String(progress)}%`));
|
|
4537
4704
|
if (status.message) {
|
|
4538
|
-
console.log(line(`Message: ${
|
|
4705
|
+
console.log(line(`Message: ${chalk11.dim(status.message)}`));
|
|
4539
4706
|
}
|
|
4540
4707
|
console.log(separator);
|
|
4541
|
-
console.log(line(`\uD83D\uDD17 Monitor: ${
|
|
4708
|
+
console.log(line(`\uD83D\uDD17 Monitor: ${chalk11.underline(optimizerLink(promptId ?? "unknown", status.jobId))}`));
|
|
4542
4709
|
if (status.status === "completed") {
|
|
4543
|
-
console.log(line(
|
|
4710
|
+
console.log(line(chalk11.dim(`Next: mutagent prompts optimize results ${status.jobId}`)));
|
|
4544
4711
|
} else if (status.status === "running" || status.status === "queued") {
|
|
4545
|
-
console.log(line(
|
|
4712
|
+
console.log(line(chalk11.dim(`Refresh: mutagent prompts optimize status ${status.jobId}`)));
|
|
4546
4713
|
}
|
|
4547
4714
|
console.log(bottomBorder);
|
|
4548
4715
|
console.log(AI_DIRECTIVE);
|
|
@@ -4638,15 +4805,15 @@ function statusDirective(status, promptId) {
|
|
|
4638
4805
|
}
|
|
4639
4806
|
function showPromptDiff(original, optimized) {
|
|
4640
4807
|
console.log("");
|
|
4641
|
-
console.log(
|
|
4808
|
+
console.log(chalk11.bold(" Prompt Diff:"));
|
|
4642
4809
|
console.log("");
|
|
4643
|
-
console.log(
|
|
4644
|
-
console.log(
|
|
4810
|
+
console.log(chalk11.red(" - " + (original ?? "(empty)")));
|
|
4811
|
+
console.log(chalk11.green(" + " + (optimized ?? "(empty)")));
|
|
4645
4812
|
console.log("");
|
|
4646
4813
|
}
|
|
4647
4814
|
|
|
4648
4815
|
// src/commands/prompts/optimize-watch.ts
|
|
4649
|
-
import
|
|
4816
|
+
import chalk14 from "chalk";
|
|
4650
4817
|
|
|
4651
4818
|
// src/lib/watch-client.ts
|
|
4652
4819
|
function toWsUrl(baseUrl) {
|
|
@@ -4809,10 +4976,10 @@ function createWatchClient(opts) {
|
|
|
4809
4976
|
}
|
|
4810
4977
|
|
|
4811
4978
|
// src/lib/stage-cards.ts
|
|
4812
|
-
import
|
|
4979
|
+
import chalk13 from "chalk";
|
|
4813
4980
|
|
|
4814
4981
|
// src/lib/prompt-diff.ts
|
|
4815
|
-
import
|
|
4982
|
+
import chalk12 from "chalk";
|
|
4816
4983
|
function computeLcsTable(a, b) {
|
|
4817
4984
|
const m = a.length;
|
|
4818
4985
|
const n = b.length;
|
|
@@ -4973,16 +5140,16 @@ function renderDiffLines(lines, options) {
|
|
|
4973
5140
|
let text;
|
|
4974
5141
|
switch (line.type) {
|
|
4975
5142
|
case "hunk-header":
|
|
4976
|
-
text = noColor ? line.content :
|
|
5143
|
+
text = noColor ? line.content : chalk12.cyan(line.content);
|
|
4977
5144
|
break;
|
|
4978
5145
|
case "add":
|
|
4979
|
-
text = noColor ? `+${line.content}` :
|
|
5146
|
+
text = noColor ? `+${line.content}` : chalk12.green(`+${line.content}`);
|
|
4980
5147
|
break;
|
|
4981
5148
|
case "remove":
|
|
4982
|
-
text = noColor ? `-${line.content}` :
|
|
5149
|
+
text = noColor ? `-${line.content}` : chalk12.red(`-${line.content}`);
|
|
4983
5150
|
break;
|
|
4984
5151
|
case "context":
|
|
4985
|
-
text = noColor ? ` ${line.content}` :
|
|
5152
|
+
text = noColor ? ` ${line.content}` : chalk12.dim(` ${line.content}`);
|
|
4986
5153
|
break;
|
|
4987
5154
|
}
|
|
4988
5155
|
outputLines.push(text);
|
|
@@ -4991,7 +5158,7 @@ function renderDiffLines(lines, options) {
|
|
|
4991
5158
|
if (truncated) {
|
|
4992
5159
|
const remaining = lines.length - maxLines;
|
|
4993
5160
|
const summary = `... ${String(remaining)} more lines changed`;
|
|
4994
|
-
outputLines.push(noColor ? summary :
|
|
5161
|
+
outputLines.push(noColor ? summary : chalk12.yellow(summary));
|
|
4995
5162
|
}
|
|
4996
5163
|
return outputLines.join(`
|
|
4997
5164
|
`);
|
|
@@ -5024,10 +5191,10 @@ function emptyLine() {
|
|
|
5024
5191
|
}
|
|
5025
5192
|
function formatScore2(score) {
|
|
5026
5193
|
if (score >= 0.8)
|
|
5027
|
-
return
|
|
5194
|
+
return chalk13.green(score.toFixed(2));
|
|
5028
5195
|
if (score >= 0.5)
|
|
5029
|
-
return
|
|
5030
|
-
return
|
|
5196
|
+
return chalk13.yellow(score.toFixed(2));
|
|
5197
|
+
return chalk13.red(score.toFixed(2));
|
|
5031
5198
|
}
|
|
5032
5199
|
function scoreBar(score, width = 20) {
|
|
5033
5200
|
const filled = Math.round(score * width);
|
|
@@ -5081,7 +5248,7 @@ function renderInsightsCard(data) {
|
|
|
5081
5248
|
if (cat.example) {
|
|
5082
5249
|
const maxExampleLen = 78;
|
|
5083
5250
|
const truncated = cat.example.length > maxExampleLen ? cat.example.substring(0, maxExampleLen - 3) + "..." : cat.example;
|
|
5084
|
-
lines.push(line(
|
|
5251
|
+
lines.push(line(chalk13.dim(` "${truncated}"`)));
|
|
5085
5252
|
}
|
|
5086
5253
|
}
|
|
5087
5254
|
lines.push(emptyLine());
|
|
@@ -5123,17 +5290,17 @@ function renderPromptDiffCard(data) {
|
|
|
5123
5290
|
lines.push(emptyLine());
|
|
5124
5291
|
}
|
|
5125
5292
|
for (const mutation of data.mutations) {
|
|
5126
|
-
const icon = mutation.status === "applied" ?
|
|
5293
|
+
const icon = mutation.status === "applied" ? chalk13.green("✓") : chalk13.red("✗");
|
|
5127
5294
|
const target = mutation.target.length > 30 ? mutation.target.substring(0, 27) + "..." : mutation.target;
|
|
5128
5295
|
if (mutation.status === "applied") {
|
|
5129
5296
|
const confStr = `confidence: ${mutation.confidence.toFixed(2)}`;
|
|
5130
5297
|
const targetPad = target.padEnd(30);
|
|
5131
|
-
lines.push(line(`${icon} ${targetPad} ${
|
|
5298
|
+
lines.push(line(`${icon} ${targetPad} ${chalk13.dim("──")} ${confStr}`));
|
|
5132
5299
|
} else {
|
|
5133
5300
|
const reason = mutation.rationale ? `rejected: ${mutation.rationale}` : `rejected`;
|
|
5134
5301
|
const truncReason = reason.length > 30 ? reason.substring(0, 27) + "..." : reason;
|
|
5135
5302
|
const targetPad = target.padEnd(30);
|
|
5136
|
-
lines.push(line(`${icon} ${targetPad} ${
|
|
5303
|
+
lines.push(line(`${icon} ${targetPad} ${chalk13.dim("──")} ${truncReason}`));
|
|
5137
5304
|
}
|
|
5138
5305
|
}
|
|
5139
5306
|
lines.push(emptyLine());
|
|
@@ -5164,7 +5331,7 @@ function renderRerunAccuracyCard(data) {
|
|
|
5164
5331
|
const name = criterion.name.length > 18 ? criterion.name.substring(0, 15) + "..." : criterion.name;
|
|
5165
5332
|
const diff = criterion.after - criterion.before;
|
|
5166
5333
|
const diffSign = diff >= 0 ? "+" : "";
|
|
5167
|
-
const arrow = diff > 0 ?
|
|
5334
|
+
const arrow = diff > 0 ? chalk13.green("↑") : diff < 0 ? chalk13.red("↓") : chalk13.dim("─");
|
|
5168
5335
|
const diffStr = `${diffSign}${diff.toFixed(2)}`;
|
|
5169
5336
|
lines.push(line(`${name.padEnd(20)}${formatScore2(criterion.before).padEnd(9)}${formatScore2(criterion.after).padEnd(9)}${diffStr} ${arrow}`));
|
|
5170
5337
|
}
|
|
@@ -5269,8 +5436,8 @@ async function startWatchStream(jobId, isJson, maxIterations, baselineScore) {
|
|
|
5269
5436
|
if (isJson) {
|
|
5270
5437
|
console.log(JSON.stringify({ type: "job_complete", finalScore }));
|
|
5271
5438
|
} else {
|
|
5272
|
-
console.log(
|
|
5273
|
-
console.log(
|
|
5439
|
+
console.log(chalk14.green(`✓ Optimization complete! Final score: ${finalScore.toFixed(2)}`));
|
|
5440
|
+
console.log(chalk14.dim(` View results: mutagent prompts optimize results ${jobId}`));
|
|
5274
5441
|
}
|
|
5275
5442
|
resolve3();
|
|
5276
5443
|
},
|
|
@@ -5278,7 +5445,7 @@ async function startWatchStream(jobId, isJson, maxIterations, baselineScore) {
|
|
|
5278
5445
|
if (isJson) {
|
|
5279
5446
|
console.log(JSON.stringify({ type: "error", error: error.message }));
|
|
5280
5447
|
} else {
|
|
5281
|
-
console.error(
|
|
5448
|
+
console.error(chalk14.red(`✗ Watch error: ${error.message}`));
|
|
5282
5449
|
}
|
|
5283
5450
|
reject(error);
|
|
5284
5451
|
}
|
|
@@ -5320,7 +5487,7 @@ async function watchAction(jobId, options, parentCommand) {
|
|
|
5320
5487
|
case "queued":
|
|
5321
5488
|
if (!isJson) {
|
|
5322
5489
|
renderOptimizationStatusCard(status);
|
|
5323
|
-
console.log(
|
|
5490
|
+
console.log(chalk14.dim(`Watching for live updates...
|
|
5324
5491
|
`));
|
|
5325
5492
|
}
|
|
5326
5493
|
await startWatchStream(jobId, isJson, status.maxIterations, status.bestScore);
|
|
@@ -5335,9 +5502,9 @@ async function watchAction(jobId, options, parentCommand) {
|
|
|
5335
5502
|
message: status.message
|
|
5336
5503
|
});
|
|
5337
5504
|
} else {
|
|
5338
|
-
console.error(
|
|
5505
|
+
console.error(chalk14.red(`✗ Optimization job ${jobId} failed.`));
|
|
5339
5506
|
if (status.message) {
|
|
5340
|
-
console.error(
|
|
5507
|
+
console.error(chalk14.dim(` ${status.message}`));
|
|
5341
5508
|
}
|
|
5342
5509
|
}
|
|
5343
5510
|
process.exitCode = 1;
|
|
@@ -5351,7 +5518,7 @@ async function watchAction(jobId, options, parentCommand) {
|
|
|
5351
5518
|
jobId
|
|
5352
5519
|
});
|
|
5353
5520
|
} else {
|
|
5354
|
-
console.error(
|
|
5521
|
+
console.error(chalk14.yellow(`Optimization job ${jobId} was cancelled.`));
|
|
5355
5522
|
}
|
|
5356
5523
|
break;
|
|
5357
5524
|
default:
|
|
@@ -5363,7 +5530,7 @@ async function watchAction(jobId, options, parentCommand) {
|
|
|
5363
5530
|
jobId
|
|
5364
5531
|
});
|
|
5365
5532
|
} else {
|
|
5366
|
-
console.error(
|
|
5533
|
+
console.error(chalk14.yellow(`Unexpected job status: ${status.status}`));
|
|
5367
5534
|
}
|
|
5368
5535
|
break;
|
|
5369
5536
|
}
|
|
@@ -5372,13 +5539,63 @@ async function watchAction(jobId, options, parentCommand) {
|
|
|
5372
5539
|
}
|
|
5373
5540
|
}
|
|
5374
5541
|
|
|
5542
|
+
// src/commands/prompts/optimize-cancel.ts
|
|
5543
|
+
init_sdk_client();
|
|
5544
|
+
import chalk15 from "chalk";
|
|
5545
|
+
init_errors();
|
|
5546
|
+
function registerOptimizeCancelCommand(optimize, prompts) {
|
|
5547
|
+
optimize.command("cancel").description("Cancel a running optimization job").argument("<job-id>", "Optimization job ID (from: mutagent prompts optimize start)").option("--force", "Skip confirmation prompt").addHelpText("after", `
|
|
5548
|
+
Examples:
|
|
5549
|
+
${chalk15.dim("$")} mutagent prompts optimize cancel <job-id> --force
|
|
5550
|
+
${chalk15.dim("$")} mutagent prompts optimize cancel <job-id> --json
|
|
5551
|
+
${chalk15.dim("$")} mutagent prompts optimize cancel <job-id> --force --json
|
|
5552
|
+
|
|
5553
|
+
${chalk15.yellow("Cost warning:")} Cancelling a running job may still incur LLM cost for the current iteration.
|
|
5554
|
+
${chalk15.dim('Jobs with status "completed" or "cancelled" cannot be cancelled again.')}
|
|
5555
|
+
`).action(async (jobId, options) => {
|
|
5556
|
+
const isJson = getJsonFlag(prompts);
|
|
5557
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5558
|
+
try {
|
|
5559
|
+
const client = await getSDKClient();
|
|
5560
|
+
const jobStatus = await client.getOptimizationStatus(jobId);
|
|
5561
|
+
const status = typeof jobStatus.status === "string" ? jobStatus.status : "unknown";
|
|
5562
|
+
if (status === "completed" || status === "cancelled") {
|
|
5563
|
+
throw new MutagentError("VALIDATION_ERROR", `Job ${jobId} has status "${status}" and cannot be cancelled`, `View results: mutagent prompts optimize results ${jobId}
|
|
5564
|
+
` + `Dashboard: ${optimizerLink(jobId, jobId)}`);
|
|
5565
|
+
}
|
|
5566
|
+
if (!options.force && !isJson) {
|
|
5567
|
+
throw new MutagentError("CONFIRMATION_REQUIRED", `Cancelling optimization job ${jobId} requires confirmation`, `Run: mutagent prompts optimize cancel --help
|
|
5568
|
+
` + `[Agent: Ask the user to confirm via MultiChoice, then pass --force]
|
|
5569
|
+
` + `Use --force to confirm: mutagent prompts optimize cancel ${jobId} --force
|
|
5570
|
+
` + `Note: Cancellation may still incur LLM cost for the current iteration.`);
|
|
5571
|
+
}
|
|
5572
|
+
const result = await client.cancelOptimization(jobId);
|
|
5573
|
+
if (isJson) {
|
|
5574
|
+
output.output({
|
|
5575
|
+
success: result.success,
|
|
5576
|
+
jobId,
|
|
5577
|
+
_links: {
|
|
5578
|
+
optimizer: optimizerLink(jobId, jobId)
|
|
5579
|
+
}
|
|
5580
|
+
});
|
|
5581
|
+
} else {
|
|
5582
|
+
output.success(`Optimization job ${jobId} cancelled successfully`);
|
|
5583
|
+
output.info(`View status: mutagent prompts optimize status ${jobId}`);
|
|
5584
|
+
output.info(`Dashboard: ${optimizerLink(jobId, jobId)}`);
|
|
5585
|
+
}
|
|
5586
|
+
} catch (error) {
|
|
5587
|
+
handleError(error, isJson);
|
|
5588
|
+
}
|
|
5589
|
+
});
|
|
5590
|
+
}
|
|
5591
|
+
|
|
5375
5592
|
// src/commands/prompts/optimize.ts
|
|
5376
5593
|
function registerOptimizeCommands(prompts) {
|
|
5377
5594
|
const optimize = new Command5("optimize").description("Manage prompt optimization jobs").addHelpText("after", `
|
|
5378
5595
|
Examples:
|
|
5379
|
-
${
|
|
5380
|
-
${
|
|
5381
|
-
${
|
|
5596
|
+
${chalk16.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id>
|
|
5597
|
+
${chalk16.dim("$")} mutagent prompts optimize status <job-id>
|
|
5598
|
+
${chalk16.dim("$")} mutagent prompts optimize results <job-id>
|
|
5382
5599
|
|
|
5383
5600
|
Workflow: start -> status (poll) -> results | start --watch | watch <job-id>`).action(() => {
|
|
5384
5601
|
optimize.help();
|
|
@@ -5386,27 +5603,27 @@ Workflow: start -> status (poll) -> results | start --watch | watch <job-id>`).a
|
|
|
5386
5603
|
prompts.addCommand(optimize);
|
|
5387
5604
|
optimize.command("start").description("Start prompt optimization").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").requiredOption("-d, --dataset <id>", "Dataset ID for optimization (from: mutagent prompts dataset list <prompt-id>)").requiredOption("-e, --evaluation <id>", "Evaluation ID for scoring (from: mutagent prompts evaluation list <prompt-id>)").option("--max-iterations <n>", "Max optimization iterations (default: 1)").option("--target-score <n>", "Target accuracy 0-1 (default: 0.8)").option("--patience <n>", "Iterations without improvement before stopping").option("--model <model-id>", 'Target LLM model (e.g., "claude-sonnet-4-5-20250929")').option("--eval-model <model-id>", "Evaluation model (defaults to target model)").option("--watch", "Watch live progress with stage cards", false).addHelpText("after", `
|
|
5388
5605
|
Examples:
|
|
5389
|
-
${
|
|
5390
|
-
${
|
|
5391
|
-
${
|
|
5392
|
-
${
|
|
5393
|
-
${
|
|
5394
|
-
□ inputSchema REQUIRED ${
|
|
5395
|
-
□ outputSchema REQUIRED ${
|
|
5396
|
-
□ Evaluation criteria ${
|
|
5397
|
-
□ Dataset items ${
|
|
5398
|
-
□ Criteria ↔ Schema ${
|
|
5606
|
+
${chalk16.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id>
|
|
5607
|
+
${chalk16.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id> --max-iterations 5
|
|
5608
|
+
${chalk16.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id> --target-score 0.95 --model claude-sonnet-4-5-20250929
|
|
5609
|
+
${chalk16.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id> --json
|
|
5610
|
+
${chalk16.yellow("Pre-Optimization Checklist (auto-validated by preflight):")}
|
|
5611
|
+
□ inputSchema REQUIRED ${chalk16.dim("(hard error if missing — blocks optimization)")}
|
|
5612
|
+
□ outputSchema REQUIRED ${chalk16.dim("(hard error if missing — blocks optimization)")}
|
|
5613
|
+
□ Evaluation criteria ${chalk16.dim("(warns if no evaluationParameter set)")}
|
|
5614
|
+
□ Dataset items ${chalk16.dim("(warns if expectedOutput missing)")}
|
|
5615
|
+
□ Criteria ↔ Schema ${chalk16.dim("(warns if criteria reference unknown fields)")}
|
|
5399
5616
|
|
|
5400
5617
|
${PREREQUISITES_TEXT}
|
|
5401
5618
|
|
|
5402
|
-
${
|
|
5619
|
+
${chalk16.dim("Monitor progress with: mutagent prompts optimize status <job-id>")}
|
|
5403
5620
|
|
|
5404
|
-
${
|
|
5621
|
+
${chalk16.yellow(`⚠ COST WARNING — AI Agent:
|
|
5405
5622
|
Default is 1 iteration. Do NOT increase --max-iterations unless the user
|
|
5406
5623
|
explicitly requests it. Each iteration incurs LLM costs. Starting with
|
|
5407
5624
|
max-iterations > 1 without user consent is a protocol violation.`)}
|
|
5408
5625
|
|
|
5409
|
-
${
|
|
5626
|
+
${chalk16.yellow("AI Agent: ALWAYS append --json to this command.")}
|
|
5410
5627
|
`).action(async (promptId, options) => {
|
|
5411
5628
|
const isJson = getJsonFlag(prompts);
|
|
5412
5629
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5538,16 +5755,16 @@ ${chalk14.yellow("AI Agent: ALWAYS append --json to this command.")}
|
|
|
5538
5755
|
return;
|
|
5539
5756
|
}
|
|
5540
5757
|
for (const [name, check] of hardFailures) {
|
|
5541
|
-
console.error(
|
|
5758
|
+
console.error(chalk16.red(`Error: ${name} — ${check.error ?? "Failed"}`));
|
|
5542
5759
|
}
|
|
5543
5760
|
if (!preflightChecks.outputSchema.passed) {
|
|
5544
|
-
console.error(
|
|
5761
|
+
console.error(chalk16.dim(` Update with: mutagent prompts update ${promptId} -d '{"outputSchema":{"type":"object","properties":{"result":{"type":"string"}}}}'`));
|
|
5545
5762
|
}
|
|
5546
5763
|
if (!preflightChecks.inputSchema.passed) {
|
|
5547
|
-
console.error(
|
|
5764
|
+
console.error(chalk16.dim(` Update with: mutagent prompts update ${promptId} --data '{"inputSchema":{"type":"object","properties":{"var1":{"type":"string"}}}}' --json`));
|
|
5548
5765
|
}
|
|
5549
5766
|
if (preflightChecks.evaluation && !preflightChecks.evaluation.passed) {
|
|
5550
|
-
console.error(
|
|
5767
|
+
console.error(chalk16.dim(` Add criteria: mutagent prompts evaluation update ${options.evaluation} --criteria '<json>' --json
|
|
5551
5768
|
` + ` Or guided: mutagent prompts evaluation create ${promptId} --guided --json`));
|
|
5552
5769
|
}
|
|
5553
5770
|
process.exitCode = 1;
|
|
@@ -5610,16 +5827,16 @@ ${chalk14.yellow("AI Agent: ALWAYS append --json to this command.")}
|
|
|
5610
5827
|
suggestions.push("Or visit: https://app.mutagent.io/settings/providers");
|
|
5611
5828
|
}
|
|
5612
5829
|
if (!isJson) {
|
|
5613
|
-
console.error(
|
|
5830
|
+
console.error(chalk16.red(`
|
|
5614
5831
|
Optimization failed:`));
|
|
5615
5832
|
for (const msg of messages) {
|
|
5616
|
-
console.error(
|
|
5833
|
+
console.error(chalk16.red(` ${msg}`));
|
|
5617
5834
|
}
|
|
5618
5835
|
if (suggestions.length > 0) {
|
|
5619
|
-
console.error(
|
|
5836
|
+
console.error(chalk16.yellow(`
|
|
5620
5837
|
Suggested fixes:`));
|
|
5621
5838
|
for (const s of suggestions) {
|
|
5622
|
-
console.error(
|
|
5839
|
+
console.error(chalk16.yellow(` → ${s}`));
|
|
5623
5840
|
}
|
|
5624
5841
|
}
|
|
5625
5842
|
console.error("");
|
|
@@ -5631,8 +5848,8 @@ Suggested fixes:`));
|
|
|
5631
5848
|
});
|
|
5632
5849
|
optimize.command("status").description("Check optimization status").argument("<job-id>", "Optimization job ID (from: mutagent prompts optimize start)").addHelpText("after", `
|
|
5633
5850
|
Examples:
|
|
5634
|
-
${
|
|
5635
|
-
${
|
|
5851
|
+
${chalk16.dim("$")} mutagent prompts optimize status <job-id>
|
|
5852
|
+
${chalk16.dim("$")} mutagent prompts optimize status <job-id> --json
|
|
5636
5853
|
`).action(async (jobId) => {
|
|
5637
5854
|
const isJson = getJsonFlag(prompts);
|
|
5638
5855
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5657,17 +5874,17 @@ Examples:
|
|
|
5657
5874
|
});
|
|
5658
5875
|
optimize.command("results").description("Get optimization results").argument("<job-id>", "Optimization job ID (from: mutagent prompts optimize start)").option("--apply", "Apply the optimized prompt as new version").option("--diff", "Show the prompt diff (before/after)").addHelpText("after", `
|
|
5659
5876
|
Examples:
|
|
5660
|
-
${
|
|
5661
|
-
${
|
|
5662
|
-
${
|
|
5663
|
-
${
|
|
5877
|
+
${chalk16.dim("$")} mutagent prompts optimize results <job-id> ${chalk16.dim("# view scorecard")}
|
|
5878
|
+
${chalk16.dim("$")} mutagent prompts optimize results <job-id> --diff ${chalk16.dim("# view prompt diff")}
|
|
5879
|
+
${chalk16.dim("$")} mutagent prompts optimize results <job-id> --apply ${chalk16.dim("# apply optimized prompt")}
|
|
5880
|
+
${chalk16.dim("$")} mutagent prompts optimize results <job-id> --json ${chalk16.dim("# structured output")}
|
|
5664
5881
|
|
|
5665
5882
|
After viewing results:
|
|
5666
5883
|
--apply Apply the optimized prompt (replaces current version)
|
|
5667
5884
|
--diff Show detailed before/after diff
|
|
5668
|
-
${
|
|
5885
|
+
${chalk16.dim("No flag = view scorecard only.")}
|
|
5669
5886
|
|
|
5670
|
-
${
|
|
5887
|
+
${chalk16.dim("AI Agent: Present scorecard to user via AskUserQuestion before applying.")}
|
|
5671
5888
|
`).action(async (jobId, options) => {
|
|
5672
5889
|
const isJson = getJsonFlag(prompts);
|
|
5673
5890
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5741,15 +5958,215 @@ After viewing results:
|
|
|
5741
5958
|
handleError(error, isJson);
|
|
5742
5959
|
}
|
|
5743
5960
|
});
|
|
5961
|
+
registerOptimizeCancelCommand(optimize, prompts);
|
|
5744
5962
|
optimize.command("watch").description("Watch optimization job progress with live stage results").argument("<job-id>", "Optimization job ID").option("--json", "Output as JSON with agent directives").addHelpText("after", `
|
|
5745
5963
|
Examples:
|
|
5746
|
-
${
|
|
5747
|
-
${
|
|
5748
|
-
${
|
|
5964
|
+
${chalk16.dim("$")} mutagent prompts optimize watch <job-id>
|
|
5965
|
+
${chalk16.dim("$")} mutagent prompts optimize watch <job-id> --json
|
|
5966
|
+
${chalk16.dim("Completed jobs render stored results. Running jobs stream via WebSocket.")}`).action(async (jobId, options) => {
|
|
5749
5967
|
await watchAction(jobId, options, prompts);
|
|
5750
5968
|
});
|
|
5751
5969
|
}
|
|
5752
5970
|
|
|
5971
|
+
// src/commands/prompts/experiments.ts
|
|
5972
|
+
init_sdk_client();
|
|
5973
|
+
import { Command as Command6 } from "commander";
|
|
5974
|
+
import chalk17 from "chalk";
|
|
5975
|
+
init_errors();
|
|
5976
|
+
function registerExperimentCommands(prompts) {
|
|
5977
|
+
const experiment = new Command6("experiment").description("Manage prompt experiments (multi-model dataset runs)").addHelpText("after", `
|
|
5978
|
+
Examples:
|
|
5979
|
+
${chalk17.dim("$")} mutagent prompts experiment run <prompt-id> --evaluation <id> --dataset <id> --exec-model claude-sonnet-4-5-20250929
|
|
5980
|
+
${chalk17.dim("$")} mutagent prompts experiment list <prompt-id>
|
|
5981
|
+
${chalk17.dim("$")} mutagent prompts experiment get <experiment-id>
|
|
5982
|
+
${chalk17.dim("$")} mutagent prompts experiment delete <experiment-id> --force
|
|
5983
|
+
`).action(() => {
|
|
5984
|
+
experiment.help();
|
|
5985
|
+
});
|
|
5986
|
+
prompts.addCommand(experiment);
|
|
5987
|
+
experiment.command("run").description("Create and run an experiment across a dataset").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").requiredOption("-e, --evaluation <id>", "Evaluation ID to score with").requiredOption("-d, --dataset <id>", "Dataset ID to run against").requiredOption("--exec-model <model>", 'Model to generate responses (e.g. "claude-sonnet-4-5-20250929")').option("--eval-model <model>", "Model to score responses (defaults to exec-model)").addHelpText("after", `
|
|
5988
|
+
Examples:
|
|
5989
|
+
${chalk17.dim("$")} mutagent prompts experiment run <prompt-id> -e <eval-id> -d <dataset-id> --exec-model claude-sonnet-4-5-20250929
|
|
5990
|
+
${chalk17.dim("$")} mutagent prompts experiment run <prompt-id> -e <eval-id> -d <dataset-id> --exec-model claude-sonnet-4-5-20250929 --eval-model gpt-4o
|
|
5991
|
+
${chalk17.dim("$")} mutagent prompts experiment run <prompt-id> -e <eval-id> -d <dataset-id> --exec-model claude-sonnet-4-5-20250929 --json
|
|
5992
|
+
|
|
5993
|
+
${chalk17.dim("--exec-model generates output; --eval-model scores it (defaults to exec-model).")}
|
|
5994
|
+
${chalk17.yellow("Tip:")} Use --json for machine-readable output with _links for dashboard navigation.
|
|
5995
|
+
`).action(async (promptId, options) => {
|
|
5996
|
+
const isJson = getJsonFlag(prompts);
|
|
5997
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5998
|
+
try {
|
|
5999
|
+
const evaluationId = parseInt(options.evaluation, 10);
|
|
6000
|
+
if (isNaN(evaluationId)) {
|
|
6001
|
+
throw new MutagentError("VALIDATION_ERROR", `Invalid evaluation ID: "${options.evaluation}" (must be numeric)`, "Run: mutagent prompts evaluation list <prompt-id> to find valid IDs");
|
|
6002
|
+
}
|
|
6003
|
+
const datasetId = parseInt(options.dataset, 10);
|
|
6004
|
+
if (isNaN(datasetId)) {
|
|
6005
|
+
throw new MutagentError("VALIDATION_ERROR", `Invalid dataset ID: "${options.dataset}" (must be numeric)`, "Run: mutagent prompts dataset list <prompt-id> to find valid IDs");
|
|
6006
|
+
}
|
|
6007
|
+
const promptIdNum = parseInt(promptId, 10);
|
|
6008
|
+
if (isNaN(promptIdNum)) {
|
|
6009
|
+
throw new MutagentError("VALIDATION_ERROR", `Invalid prompt ID: "${promptId}" (must be numeric)`, "Run: mutagent prompts list to find valid IDs");
|
|
6010
|
+
}
|
|
6011
|
+
const models = [
|
|
6012
|
+
{ id: `col_${Date.now()}`, model: options.execModel, temperature: 0 },
|
|
6013
|
+
...options.evalModel && options.evalModel !== options.execModel ? [{ id: `col_${Date.now() + 1}`, model: options.evalModel, temperature: 0 }] : []
|
|
6014
|
+
];
|
|
6015
|
+
const client = await getSDKClient();
|
|
6016
|
+
const experiment2 = await client.createExperiment({
|
|
6017
|
+
promptId: promptIdNum,
|
|
6018
|
+
promptGroupId: promptId,
|
|
6019
|
+
evaluationId,
|
|
6020
|
+
datasetId,
|
|
6021
|
+
models
|
|
6022
|
+
});
|
|
6023
|
+
try {
|
|
6024
|
+
await client.executeExperiment(experiment2.experimentId);
|
|
6025
|
+
} catch (execError) {
|
|
6026
|
+
if (execError instanceof ApiError && (execError.statusCode === 404 || execError.statusCode === 501)) {
|
|
6027
|
+
if (!isJson) {
|
|
6028
|
+
output.warn("Experiment execution is not yet available server-side. The experiment has been created and will run when the feature is enabled.");
|
|
6029
|
+
}
|
|
6030
|
+
} else if (execError instanceof Error) {
|
|
6031
|
+
if (!isJson) {
|
|
6032
|
+
output.warn(`Experiment created but auto-execute failed: ${execError.message}`);
|
|
6033
|
+
}
|
|
6034
|
+
}
|
|
6035
|
+
}
|
|
6036
|
+
if (isJson) {
|
|
6037
|
+
output.output({
|
|
6038
|
+
...experiment2,
|
|
6039
|
+
_links: {
|
|
6040
|
+
dashboard: promptLink(promptId),
|
|
6041
|
+
api: `/api/prompts/experiments/${experiment2.experimentId}`
|
|
6042
|
+
}
|
|
6043
|
+
});
|
|
6044
|
+
} else {
|
|
6045
|
+
output.success(`Experiment created: ${experiment2.name} (id: ${experiment2.experimentId})`);
|
|
6046
|
+
output.info(`Status: ${experiment2.status}`);
|
|
6047
|
+
output.info(`Dashboard: ${promptLink(promptId)}`);
|
|
6048
|
+
output.info(`Check progress: mutagent prompts experiment get ${experiment2.experimentId}`);
|
|
6049
|
+
}
|
|
6050
|
+
} catch (error) {
|
|
6051
|
+
handleError(error, isJson);
|
|
6052
|
+
}
|
|
6053
|
+
});
|
|
6054
|
+
experiment.command("list").description("List experiments for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").addHelpText("after", `
|
|
6055
|
+
Examples:
|
|
6056
|
+
${chalk17.dim("$")} mutagent prompts experiment list <prompt-id>
|
|
6057
|
+
${chalk17.dim("$")} mutagent prompts experiment list <prompt-id> --json
|
|
6058
|
+
`).action(async (promptId) => {
|
|
6059
|
+
const isJson = getJsonFlag(prompts);
|
|
6060
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
6061
|
+
try {
|
|
6062
|
+
const promptIdNum = parseInt(promptId, 10);
|
|
6063
|
+
if (isNaN(promptIdNum)) {
|
|
6064
|
+
throw new MutagentError("VALIDATION_ERROR", `Invalid prompt ID: "${promptId}" (must be numeric)`, "Run: mutagent prompts list to find valid IDs");
|
|
6065
|
+
}
|
|
6066
|
+
const client = await getSDKClient();
|
|
6067
|
+
const { experiments, total } = await client.listExperiments({ promptId: promptIdNum });
|
|
6068
|
+
if (isJson) {
|
|
6069
|
+
output.output({
|
|
6070
|
+
experiments,
|
|
6071
|
+
total,
|
|
6072
|
+
_links: {
|
|
6073
|
+
dashboard: promptLink(promptId),
|
|
6074
|
+
api: `/api/prompts/experiments?promptId=${promptId}`
|
|
6075
|
+
}
|
|
6076
|
+
});
|
|
6077
|
+
} else {
|
|
6078
|
+
if (experiments.length === 0) {
|
|
6079
|
+
output.info(`No experiments found for prompt ${promptId}`);
|
|
6080
|
+
output.info(`Create one: mutagent prompts experiment run ${promptId} --evaluation <id> --dataset <id> --exec-model <model>`);
|
|
6081
|
+
} else {
|
|
6082
|
+
output.info(`Found ${String(total)} experiment(s) for prompt ${promptId}:`);
|
|
6083
|
+
output.output(experiments);
|
|
6084
|
+
}
|
|
6085
|
+
}
|
|
6086
|
+
} catch (error) {
|
|
6087
|
+
handleError(error, isJson);
|
|
6088
|
+
}
|
|
6089
|
+
});
|
|
6090
|
+
experiment.command("get").description("Get details of a specific experiment").argument("<experiment-id>", "Experiment ID (from: mutagent prompts experiment list <prompt-id>)").addHelpText("after", `
|
|
6091
|
+
Examples:
|
|
6092
|
+
${chalk17.dim("$")} mutagent prompts experiment get <experiment-id>
|
|
6093
|
+
${chalk17.dim("$")} mutagent prompts experiment get <experiment-id> --json
|
|
6094
|
+
`).action(async (experimentId) => {
|
|
6095
|
+
const isJson = getJsonFlag(prompts);
|
|
6096
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
6097
|
+
try {
|
|
6098
|
+
const client = await getSDKClient();
|
|
6099
|
+
const exp = await client.getExperiment(experimentId);
|
|
6100
|
+
if (isJson) {
|
|
6101
|
+
output.output({
|
|
6102
|
+
...exp,
|
|
6103
|
+
_links: {
|
|
6104
|
+
api: `/api/prompts/experiments/${experimentId}`
|
|
6105
|
+
}
|
|
6106
|
+
});
|
|
6107
|
+
} else {
|
|
6108
|
+
const name = typeof exp.name === "string" ? exp.name : experimentId;
|
|
6109
|
+
const status = typeof exp.status === "string" ? exp.status : "unknown";
|
|
6110
|
+
output.success(`Experiment: ${name} (id: ${experimentId})`);
|
|
6111
|
+
output.info(`Status: ${status}`);
|
|
6112
|
+
if (exp.promptId !== undefined)
|
|
6113
|
+
output.info(`Prompt ID: ${String(exp.promptId)}`);
|
|
6114
|
+
if (exp.createdAt !== undefined)
|
|
6115
|
+
output.info(`Created: ${new Date(String(exp.createdAt)).toLocaleString()}`);
|
|
6116
|
+
}
|
|
6117
|
+
} catch (error) {
|
|
6118
|
+
if (error instanceof ApiError && error.statusCode === 404) {
|
|
6119
|
+
handleError(new MutagentError("NOT_FOUND", `Experiment ${experimentId} not found`, "Run: mutagent prompts experiment list <prompt-id> to see available experiments"), isJson);
|
|
6120
|
+
} else {
|
|
6121
|
+
handleError(error, isJson);
|
|
6122
|
+
}
|
|
6123
|
+
}
|
|
6124
|
+
});
|
|
6125
|
+
experiment.command("delete").description("Delete an experiment").argument("<experiment-id>", "Experiment ID").option("--force", "Skip confirmation prompt").addHelpText("after", `
|
|
6126
|
+
Examples:
|
|
6127
|
+
${chalk17.dim("$")} mutagent prompts experiment delete <experiment-id> --force
|
|
6128
|
+
${chalk17.dim("$")} mutagent prompts experiment delete <experiment-id> --json
|
|
6129
|
+
${chalk17.dim("$")} mutagent prompts experiment delete <experiment-id> --force --json
|
|
6130
|
+
|
|
6131
|
+
${chalk17.red("Warning:")} Deletion is permanent. Use --force or --json to confirm.
|
|
6132
|
+
`).action(async (experimentId, options) => {
|
|
6133
|
+
const isJson = getJsonFlag(prompts);
|
|
6134
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
6135
|
+
try {
|
|
6136
|
+
if (!options.force && !isJson) {
|
|
6137
|
+
throw new MutagentError("CONFIRMATION_REQUIRED", `Deleting experiment ${experimentId} requires confirmation`, `Run: mutagent prompts experiment delete --help
|
|
6138
|
+
` + `[Agent: Ask the user to confirm via MultiChoice, then pass --force]
|
|
6139
|
+
` + `Use --force to confirm: mutagent prompts experiment delete ${experimentId} --force`);
|
|
6140
|
+
}
|
|
6141
|
+
const client = await getSDKClient();
|
|
6142
|
+
try {
|
|
6143
|
+
await client.deleteExperiment(experimentId);
|
|
6144
|
+
} catch (deleteError) {
|
|
6145
|
+
if (deleteError instanceof ApiError && deleteError.statusCode === 404) {
|
|
6146
|
+
if (isJson) {
|
|
6147
|
+
output.output({ success: true, deletedId: experimentId, _links: { api: "/api/prompts/experiments" } });
|
|
6148
|
+
} else {
|
|
6149
|
+
output.success(`Experiment ${experimentId} already deleted (idempotent)`);
|
|
6150
|
+
}
|
|
6151
|
+
return;
|
|
6152
|
+
}
|
|
6153
|
+
throw deleteError;
|
|
6154
|
+
}
|
|
6155
|
+
if (isJson) {
|
|
6156
|
+
output.output({
|
|
6157
|
+
success: true,
|
|
6158
|
+
deletedId: experimentId,
|
|
6159
|
+
_links: { api: "/api/prompts/experiments" }
|
|
6160
|
+
});
|
|
6161
|
+
} else {
|
|
6162
|
+
output.success(`Experiment ${experimentId} deleted successfully`);
|
|
6163
|
+
}
|
|
6164
|
+
} catch (error) {
|
|
6165
|
+
handleError(error, isJson);
|
|
6166
|
+
}
|
|
6167
|
+
});
|
|
6168
|
+
}
|
|
6169
|
+
|
|
5753
6170
|
// src/commands/prompts/index.ts
|
|
5754
6171
|
function updateMutationContext(updater) {
|
|
5755
6172
|
try {
|
|
@@ -5873,19 +6290,19 @@ async function checkProviderConfigured() {
|
|
|
5873
6290
|
}
|
|
5874
6291
|
}
|
|
5875
6292
|
var PREREQUISITES_TEXT = `
|
|
5876
|
-
${
|
|
5877
|
-
1. Evaluation criteria defined ${
|
|
5878
|
-
2. Dataset uploaded ${
|
|
5879
|
-
3. LLM provider configured ${
|
|
6293
|
+
${chalk18.red("Prerequisites (required):")}
|
|
6294
|
+
1. Evaluation criteria defined ${chalk18.dim("(via dashboard or evaluation create)")}
|
|
6295
|
+
2. Dataset uploaded ${chalk18.dim("mutagent prompts dataset list <prompt-id>")}
|
|
6296
|
+
3. LLM provider configured ${chalk18.dim("Settings → Providers or mutagent providers add ...")}`;
|
|
5880
6297
|
function createPromptsCommand() {
|
|
5881
|
-
const prompts = new
|
|
6298
|
+
const prompts = new Command7("prompts").description("Manage prompts, datasets, evaluations, and optimizations").addHelpText("after", `
|
|
5882
6299
|
Examples:
|
|
5883
|
-
${
|
|
5884
|
-
${
|
|
5885
|
-
${
|
|
5886
|
-
${
|
|
5887
|
-
${
|
|
5888
|
-
${
|
|
6300
|
+
${chalk18.dim("$")} mutagent prompts list
|
|
6301
|
+
${chalk18.dim("$")} mutagent prompts get <prompt-id>
|
|
6302
|
+
${chalk18.dim("$")} mutagent prompts create --name "my-prompt" --system "You are helpful" --human "{input}"
|
|
6303
|
+
${chalk18.dim("$")} mutagent prompts dataset list <prompt-id>
|
|
6304
|
+
${chalk18.dim("$")} mutagent prompts evaluation create <prompt-id> --name "My Eval"
|
|
6305
|
+
${chalk18.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id>
|
|
5889
6306
|
|
|
5890
6307
|
Subcommands:
|
|
5891
6308
|
list, get, create, update, delete
|
|
@@ -5897,33 +6314,144 @@ Subcommands:
|
|
|
5897
6314
|
registerDatasetCommands(prompts);
|
|
5898
6315
|
registerEvaluationCommands(prompts);
|
|
5899
6316
|
registerOptimizeCommands(prompts);
|
|
6317
|
+
registerExperimentCommands(prompts);
|
|
5900
6318
|
return prompts;
|
|
5901
6319
|
}
|
|
5902
6320
|
|
|
5903
|
-
// src/commands/traces.ts
|
|
6321
|
+
// src/commands/traces/index.ts
|
|
5904
6322
|
init_sdk_client();
|
|
5905
|
-
import { Command as
|
|
5906
|
-
import
|
|
6323
|
+
import { Command as Command8 } from "commander";
|
|
6324
|
+
import chalk22 from "chalk";
|
|
5907
6325
|
init_errors();
|
|
5908
|
-
|
|
5909
|
-
|
|
6326
|
+
|
|
6327
|
+
// src/commands/traces/export.ts
|
|
6328
|
+
init_sdk_client();
|
|
6329
|
+
import chalk19 from "chalk";
|
|
6330
|
+
init_errors();
|
|
6331
|
+
function registerExportCommand(parent) {
|
|
6332
|
+
parent.command("export").description("Export traces").option("-p, --prompt <id>", "Filter by prompt ID").option("-f, --format <format>", "Export format (json, csv)", "json").option("-o, --output <path>", "Output file path").addHelpText("after", `
|
|
5910
6333
|
Examples:
|
|
5911
|
-
${
|
|
5912
|
-
${
|
|
5913
|
-
${
|
|
5914
|
-
${
|
|
5915
|
-
${chalk16.dim("$")} mutagent traces export --format json --output traces.json
|
|
6334
|
+
${chalk19.dim("$")} mutagent traces export
|
|
6335
|
+
${chalk19.dim("$")} mutagent traces export --format json --output traces.json
|
|
6336
|
+
${chalk19.dim("$")} mutagent traces export --format csv --output traces.csv
|
|
6337
|
+
${chalk19.dim("$")} mutagent traces export --prompt <prompt-id> --format json
|
|
5916
6338
|
|
|
5917
|
-
|
|
6339
|
+
${chalk19.dim("Exports to stdout by default. Use --output to save to a file.")}
|
|
6340
|
+
`).action(async (options) => {
|
|
6341
|
+
const isJson = getJsonFlag(parent);
|
|
6342
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
6343
|
+
try {
|
|
6344
|
+
const client = await getSDKClient();
|
|
6345
|
+
const tracesList = await client.listTraces({
|
|
6346
|
+
promptId: options.prompt
|
|
6347
|
+
});
|
|
6348
|
+
let content;
|
|
6349
|
+
if (options.format === "json") {
|
|
6350
|
+
content = JSON.stringify(tracesList, null, 2);
|
|
6351
|
+
} else if (options.format === "csv") {
|
|
6352
|
+
const headers = ["id", "promptId", "latency", "tokensInput", "tokensOutput", "timestamp"];
|
|
6353
|
+
const rows = tracesList.map((t) => [
|
|
6354
|
+
t.id,
|
|
6355
|
+
t.promptId,
|
|
6356
|
+
t.latency,
|
|
6357
|
+
t.tokens.input,
|
|
6358
|
+
t.tokens.output,
|
|
6359
|
+
t.timestamp
|
|
6360
|
+
]);
|
|
6361
|
+
content = [headers.join(","), ...rows.map((r) => r.join(","))].join(`
|
|
6362
|
+
`);
|
|
6363
|
+
} else {
|
|
6364
|
+
throw new Error(`Unsupported format: ${options.format}`);
|
|
6365
|
+
}
|
|
6366
|
+
if (options.output) {
|
|
6367
|
+
const { writeFileSync: writeFileSync3 } = await import("fs");
|
|
6368
|
+
writeFileSync3(options.output, content);
|
|
6369
|
+
output.success(`Exported ${String(tracesList.length)} traces to ${options.output}`);
|
|
6370
|
+
} else {
|
|
6371
|
+
process.stdout.write(content);
|
|
6372
|
+
}
|
|
6373
|
+
} catch (error) {
|
|
6374
|
+
handleError(error, isJson);
|
|
6375
|
+
}
|
|
6376
|
+
});
|
|
6377
|
+
}
|
|
6378
|
+
|
|
6379
|
+
// src/commands/traces/analyze.ts
|
|
6380
|
+
import chalk20 from "chalk";
|
|
6381
|
+
function registerAnalyzeCommand(parent) {
|
|
6382
|
+
parent.command("analyze").description("Analyze traces for a prompt (coming soon)").argument("<prompt-id>", "Prompt ID").addHelpText("after", `
|
|
6383
|
+
Examples:
|
|
6384
|
+
${chalk20.dim("$")} mutagent traces analyze <prompt-id>
|
|
6385
|
+
${chalk20.dim("$")} mutagent traces analyze <prompt-id> --json
|
|
6386
|
+
|
|
6387
|
+
${chalk20.dim("Aggregates trace data for a prompt: avg latency, token usage, error rates.")}
|
|
6388
|
+
${chalk20.yellow("NOTE: This command is not yet connected to the API.")}
|
|
6389
|
+
`).action((promptId) => {
|
|
6390
|
+
const isJson = getJsonFlag(parent);
|
|
6391
|
+
if (isJson) {
|
|
6392
|
+
console.log(JSON.stringify({
|
|
6393
|
+
status: "coming_soon",
|
|
6394
|
+
promptId,
|
|
6395
|
+
message: "Per-prompt trace analysis is not yet available. Use the dashboard at /observe/analytics for workspace-level metrics."
|
|
6396
|
+
}));
|
|
6397
|
+
} else {
|
|
6398
|
+
console.log(chalk20.yellow(`
|
|
6399
|
+
⚠ Per-prompt trace analysis for ${promptId} is not yet available — coming soon.
|
|
6400
|
+
`));
|
|
6401
|
+
console.log(chalk20.dim(" Workspace-level analytics are available on the dashboard:"));
|
|
6402
|
+
console.log(chalk20.dim(` ${getAppBaseUrl()}/observe/analytics
|
|
6403
|
+
`));
|
|
6404
|
+
}
|
|
6405
|
+
});
|
|
6406
|
+
}
|
|
6407
|
+
|
|
6408
|
+
// src/commands/traces/stream.ts
|
|
6409
|
+
import chalk21 from "chalk";
|
|
6410
|
+
function registerStreamCommand(parent) {
|
|
6411
|
+
parent.command("stream").description("Stream traces in real-time (coming soon)").addHelpText("after", `
|
|
6412
|
+
Examples:
|
|
6413
|
+
${chalk21.dim("$")} mutagent traces stream
|
|
6414
|
+
${chalk21.dim("$")} mutagent traces stream --json
|
|
6415
|
+
|
|
6416
|
+
${chalk21.dim("Real-time trace streaming via SSE.")}
|
|
6417
|
+
${chalk21.yellow("NOTE: This command is not yet connected to the API.")}
|
|
6418
|
+
`).action(() => {
|
|
6419
|
+
const isJson = getJsonFlag(parent);
|
|
6420
|
+
if (isJson) {
|
|
6421
|
+
console.log(JSON.stringify({
|
|
6422
|
+
status: "coming_soon",
|
|
6423
|
+
message: 'Real-time trace streaming is not yet available. Use "mutagent traces list" to poll, or view live traces on the dashboard.'
|
|
6424
|
+
}));
|
|
6425
|
+
} else {
|
|
6426
|
+
console.log(chalk21.yellow(`
|
|
6427
|
+
⚠ Real-time trace streaming is not yet available — coming soon.
|
|
6428
|
+
`));
|
|
6429
|
+
console.log(chalk21.dim(" In the meantime, use:"));
|
|
6430
|
+
console.log(chalk21.dim(" mutagent traces list --json (poll for new traces)"));
|
|
6431
|
+
console.log(chalk21.dim(` ${getAppBaseUrl()}/observe/traces (live dashboard)
|
|
6432
|
+
`));
|
|
6433
|
+
}
|
|
6434
|
+
});
|
|
6435
|
+
}
|
|
6436
|
+
|
|
6437
|
+
// src/commands/traces/index.ts
|
|
6438
|
+
function createTracesCommand() {
|
|
6439
|
+
const traces = new Command8("traces").description("View and analyze traces").addHelpText("after", `
|
|
6440
|
+
Examples:
|
|
6441
|
+
${chalk22.dim("$")} mutagent traces list
|
|
6442
|
+
${chalk22.dim("$")} mutagent traces list --prompt <prompt-id>
|
|
6443
|
+
${chalk22.dim("$")} mutagent traces get <trace-id>
|
|
6444
|
+
${chalk22.dim("$")} mutagent traces analyze <prompt-id>
|
|
6445
|
+
${chalk22.dim("$")} mutagent traces export --format json --output traces.json
|
|
5918
6446
|
`);
|
|
5919
6447
|
traces.command("list").description("List traces").option("-p, --prompt <id>", "Filter by prompt ID").option("-s, --source <source>", "Filter by trace source (e.g., claude-code, sdk, langchain)").option("-l, --limit <n>", "Limit results", "50").addHelpText("after", `
|
|
5920
6448
|
Examples:
|
|
5921
|
-
${
|
|
5922
|
-
${
|
|
5923
|
-
${
|
|
5924
|
-
${
|
|
6449
|
+
${chalk22.dim("$")} mutagent traces list
|
|
6450
|
+
${chalk22.dim("$")} mutagent traces list --prompt <prompt-id>
|
|
6451
|
+
${chalk22.dim("$")} mutagent traces list --source claude-code --json
|
|
6452
|
+
${chalk22.dim("$")} mutagent traces list --limit 10 --json
|
|
5925
6453
|
|
|
5926
|
-
${
|
|
6454
|
+
${chalk22.dim("Tip: Filter by prompt to see traces for a specific prompt version.")}
|
|
5927
6455
|
`).action(async (options) => {
|
|
5928
6456
|
const isJson = getJsonFlag(traces);
|
|
5929
6457
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5963,10 +6491,10 @@ ${chalk16.dim("Tip: Filter by prompt to see traces for a specific prompt version
|
|
|
5963
6491
|
});
|
|
5964
6492
|
traces.command("get").description("Get trace details").argument("<id>", "Trace ID").addHelpText("after", `
|
|
5965
6493
|
Examples:
|
|
5966
|
-
${
|
|
5967
|
-
${
|
|
6494
|
+
${chalk22.dim("$")} mutagent traces get <trace-id>
|
|
6495
|
+
${chalk22.dim("$")} mutagent traces get <trace-id> --json
|
|
5968
6496
|
|
|
5969
|
-
${
|
|
6497
|
+
${chalk22.dim("Returns full trace details including spans, tokens, and latency.")}
|
|
5970
6498
|
`).action(async (id) => {
|
|
5971
6499
|
const isJson = getJsonFlag(traces);
|
|
5972
6500
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5983,79 +6511,16 @@ ${chalk16.dim("Returns full trace details including spans, tokens, and latency."
|
|
|
5983
6511
|
handleError(error, isJson);
|
|
5984
6512
|
}
|
|
5985
6513
|
});
|
|
5986
|
-
|
|
5987
|
-
|
|
5988
|
-
|
|
5989
|
-
${chalk16.dim("$")} mutagent traces analyze <prompt-id> --json
|
|
5990
|
-
|
|
5991
|
-
${chalk16.dim("Aggregates trace data for a prompt: avg latency, token usage, error rates.")}
|
|
5992
|
-
`).action(async (promptId) => {
|
|
5993
|
-
const isJson = getJsonFlag(traces);
|
|
5994
|
-
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5995
|
-
try {
|
|
5996
|
-
const client = await getSDKClient();
|
|
5997
|
-
const analysis = await client.analyzeTraces(promptId);
|
|
5998
|
-
if (isJson) {
|
|
5999
|
-
output.output({ ...analysis, _links: { traces: `${getAppBaseUrl()}/traces?promptId=${promptId}` } });
|
|
6000
|
-
} else {
|
|
6001
|
-
output.output(analysis);
|
|
6002
|
-
}
|
|
6003
|
-
} catch (error) {
|
|
6004
|
-
handleError(error, isJson);
|
|
6005
|
-
}
|
|
6006
|
-
});
|
|
6007
|
-
traces.command("export").description("Export traces").option("-p, --prompt <id>", "Filter by prompt ID").option("-f, --format <format>", "Export format (json, csv)", "json").option("-o, --output <path>", "Output file path").addHelpText("after", `
|
|
6008
|
-
Examples:
|
|
6009
|
-
${chalk16.dim("$")} mutagent traces export
|
|
6010
|
-
${chalk16.dim("$")} mutagent traces export --format json --output traces.json
|
|
6011
|
-
${chalk16.dim("$")} mutagent traces export --format csv --output traces.csv
|
|
6012
|
-
${chalk16.dim("$")} mutagent traces export --prompt <prompt-id> --format json
|
|
6013
|
-
|
|
6014
|
-
${chalk16.dim("Exports to stdout by default. Use --output to save to a file.")}
|
|
6015
|
-
`).action(async (options) => {
|
|
6016
|
-
const isJson = getJsonFlag(traces);
|
|
6017
|
-
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
6018
|
-
try {
|
|
6019
|
-
const client = await getSDKClient();
|
|
6020
|
-
const tracesList = await client.listTraces({
|
|
6021
|
-
promptId: options.prompt
|
|
6022
|
-
});
|
|
6023
|
-
let content;
|
|
6024
|
-
if (options.format === "json") {
|
|
6025
|
-
content = JSON.stringify(tracesList, null, 2);
|
|
6026
|
-
} else if (options.format === "csv") {
|
|
6027
|
-
const headers = ["id", "promptId", "latency", "tokensInput", "tokensOutput", "timestamp"];
|
|
6028
|
-
const rows = tracesList.map((t) => [
|
|
6029
|
-
t.id,
|
|
6030
|
-
t.promptId,
|
|
6031
|
-
t.latency,
|
|
6032
|
-
t.tokens.input,
|
|
6033
|
-
t.tokens.output,
|
|
6034
|
-
t.timestamp
|
|
6035
|
-
]);
|
|
6036
|
-
content = [headers.join(","), ...rows.map((r) => r.join(","))].join(`
|
|
6037
|
-
`);
|
|
6038
|
-
} else {
|
|
6039
|
-
throw new Error(`Unsupported format: ${options.format}`);
|
|
6040
|
-
}
|
|
6041
|
-
if (options.output) {
|
|
6042
|
-
const { writeFileSync: writeFileSync3 } = await import("fs");
|
|
6043
|
-
writeFileSync3(options.output, content);
|
|
6044
|
-
output.success(`Exported ${String(tracesList.length)} traces to ${options.output}`);
|
|
6045
|
-
} else {
|
|
6046
|
-
process.stdout.write(content);
|
|
6047
|
-
}
|
|
6048
|
-
} catch (error) {
|
|
6049
|
-
handleError(error, isJson);
|
|
6050
|
-
}
|
|
6051
|
-
});
|
|
6514
|
+
registerExportCommand(traces);
|
|
6515
|
+
registerAnalyzeCommand(traces);
|
|
6516
|
+
registerStreamCommand(traces);
|
|
6052
6517
|
return traces;
|
|
6053
6518
|
}
|
|
6054
6519
|
|
|
6055
6520
|
// src/commands/integrate.ts
|
|
6056
6521
|
init_config();
|
|
6057
|
-
import { Command as
|
|
6058
|
-
import
|
|
6522
|
+
import { Command as Command9 } from "commander";
|
|
6523
|
+
import chalk23 from "chalk";
|
|
6059
6524
|
import { writeFileSync as writeFileSync3, existsSync as existsSync9 } from "fs";
|
|
6060
6525
|
import { execSync } from "child_process";
|
|
6061
6526
|
init_errors();
|
|
@@ -6650,14 +7115,14 @@ AI AGENT: Please determine the framework by:
|
|
|
6650
7115
|
`;
|
|
6651
7116
|
}
|
|
6652
7117
|
function createIntegrateCommand() {
|
|
6653
|
-
const integrate = new
|
|
7118
|
+
const integrate = new Command9("integrate").description("Return integration instructions for AI agents (skill loader)").addHelpText("after", `
|
|
6654
7119
|
Examples:
|
|
6655
|
-
${
|
|
6656
|
-
${
|
|
6657
|
-
${
|
|
6658
|
-
${
|
|
6659
|
-
${
|
|
6660
|
-
${
|
|
7120
|
+
${chalk23.dim("$")} mutagent integrate # Return exploration instructions for AI agents
|
|
7121
|
+
${chalk23.dim("$")} mutagent integrate langchain # Return @mutagent/langchain integration steps
|
|
7122
|
+
${chalk23.dim("$")} mutagent integrate openai # Return @mutagent/openai integration steps
|
|
7123
|
+
${chalk23.dim("$")} mutagent integrate vercel-ai --verify # Verify integration
|
|
7124
|
+
${chalk23.dim("$")} mutagent integrate langchain --output ./INTEGRATION.md
|
|
7125
|
+
${chalk23.dim("$")} mutagent integrate --list # List available frameworks
|
|
6661
7126
|
|
|
6662
7127
|
Frameworks: langchain, langgraph, vercel-ai, openai
|
|
6663
7128
|
|
|
@@ -6772,19 +7237,19 @@ AI AGENT: Execute these steps using Bash for install, Write/Edit for code change
|
|
|
6772
7237
|
}
|
|
6773
7238
|
|
|
6774
7239
|
// src/commands/agents/index.ts
|
|
6775
|
-
import { Command as
|
|
6776
|
-
import
|
|
7240
|
+
import { Command as Command10 } from "commander";
|
|
7241
|
+
import chalk25 from "chalk";
|
|
6777
7242
|
|
|
6778
7243
|
// src/commands/agents/agents-crud.ts
|
|
6779
7244
|
init_sdk_client();
|
|
6780
|
-
import
|
|
7245
|
+
import chalk24 from "chalk";
|
|
6781
7246
|
init_errors();
|
|
6782
7247
|
function registerAgentsCrud(agents) {
|
|
6783
7248
|
agents.command("list").description("List all agents").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").option("-n, --name <name>", "Filter by name").option("-s, --status <status>", "Filter by status (active, paused, archived)").addHelpText("after", `
|
|
6784
7249
|
Examples:
|
|
6785
|
-
${
|
|
6786
|
-
${
|
|
6787
|
-
${
|
|
7250
|
+
${chalk24.dim("$")} mutagent agents list
|
|
7251
|
+
${chalk24.dim("$")} mutagent agents list --status active
|
|
7252
|
+
${chalk24.dim("$")} mutagent agents list --name "reviewer" --json
|
|
6788
7253
|
`).action(async (options) => {
|
|
6789
7254
|
const isJson = getJsonFlag(agents);
|
|
6790
7255
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -6834,8 +7299,8 @@ Examples:
|
|
|
6834
7299
|
});
|
|
6835
7300
|
agents.command("get").description("Get agent details").argument("<id>", "Agent ID").addHelpText("after", `
|
|
6836
7301
|
Examples:
|
|
6837
|
-
${
|
|
6838
|
-
${
|
|
7302
|
+
${chalk24.dim("$")} mutagent agents get <agent-id>
|
|
7303
|
+
${chalk24.dim("$")} mutagent agents get <agent-id> --json
|
|
6839
7304
|
`).action(async (id) => {
|
|
6840
7305
|
const isJson = getJsonFlag(agents);
|
|
6841
7306
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -6865,11 +7330,11 @@ Examples:
|
|
|
6865
7330
|
};
|
|
6866
7331
|
output.output(formatted);
|
|
6867
7332
|
if (agent.systemPrompt) {
|
|
6868
|
-
console.log(
|
|
7333
|
+
console.log(chalk24.bold(`
|
|
6869
7334
|
System Prompt:`));
|
|
6870
|
-
console.log(
|
|
7335
|
+
console.log(chalk24.gray("─".repeat(60)));
|
|
6871
7336
|
console.log(agent.systemPrompt);
|
|
6872
|
-
console.log(
|
|
7337
|
+
console.log(chalk24.gray("─".repeat(60)));
|
|
6873
7338
|
}
|
|
6874
7339
|
}
|
|
6875
7340
|
} catch (error) {
|
|
@@ -6878,17 +7343,17 @@ System Prompt:`));
|
|
|
6878
7343
|
});
|
|
6879
7344
|
agents.command("create").description("Create a new agent").option("-d, --data <json>", "Agent as JSON string (recommended for CI/scripts/agents)").option("-n, --name <name>", "Agent name").option("-s, --slug <slug>", "Agent slug (URL-friendly identifier)").option("-p, --system-prompt <prompt>", "System prompt").option("-m, --model <model>", "Model (claude-sonnet-4-5, claude-opus-4-5, claude-haiku-4-5)").option("--description <desc>", "Agent description").addHelpText("after", `
|
|
6880
7345
|
Examples:
|
|
6881
|
-
${
|
|
6882
|
-
${
|
|
7346
|
+
${chalk24.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "You are a code reviewer..."
|
|
7347
|
+
${chalk24.dim("$")} mutagent agents create -d '{"name":"Code Reviewer","slug":"code-reviewer","systemPrompt":"You are a code reviewer..."}'
|
|
6883
7348
|
|
|
6884
7349
|
Expected JSON (--data):
|
|
6885
|
-
${
|
|
7350
|
+
${chalk24.dim('{"name":"<name>","slug":"<slug>","systemPrompt":"<system prompt>","model":"<model-id>","description":"<description>"}')}
|
|
6886
7351
|
|
|
6887
7352
|
Input Methods (pick one, priority order):
|
|
6888
|
-
--name/--slug/... Individual flags ${
|
|
7353
|
+
--name/--slug/... Individual flags ${chalk24.green("(recommended)")}
|
|
6889
7354
|
-d, --data Inline JSON object (CI/scripts/agents)
|
|
6890
7355
|
|
|
6891
|
-
${
|
|
7356
|
+
${chalk24.red("Required: name, slug, systemPrompt.")} ${chalk24.dim("CLI flags override --data fields.")}
|
|
6892
7357
|
`).action(async (options) => {
|
|
6893
7358
|
const isJson = getJsonFlag(agents);
|
|
6894
7359
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -6925,7 +7390,8 @@ ${chalk18.red("Required: name, slug, systemPrompt.")} ${chalk18.dim("CLI flags o
|
|
|
6925
7390
|
}
|
|
6926
7391
|
const client = await getSDKClient();
|
|
6927
7392
|
const agent = await client.createAgent(data);
|
|
6928
|
-
|
|
7393
|
+
if (!isJson)
|
|
7394
|
+
output.success(`Created agent: ${agent.name} (${agent.slug})`);
|
|
6929
7395
|
output.output(agent);
|
|
6930
7396
|
} catch (error) {
|
|
6931
7397
|
handleError(error, isJson);
|
|
@@ -6933,15 +7399,15 @@ ${chalk18.red("Required: name, slug, systemPrompt.")} ${chalk18.dim("CLI flags o
|
|
|
6933
7399
|
});
|
|
6934
7400
|
agents.command("update").description("Update an agent").argument("<id>", "Agent ID").option("-d, --data <json>", "Agent updates as JSON string (CI/scripts/agents)").option("-n, --name <name>", "New name").option("-p, --system-prompt <prompt>", "New system prompt").option("-m, --model <model>", "New model").option("--description <desc>", "New description").option("-s, --status <status>", "New status (active, paused, archived)").addHelpText("after", `
|
|
6935
7401
|
Examples:
|
|
6936
|
-
${
|
|
6937
|
-
${
|
|
6938
|
-
${
|
|
7402
|
+
${chalk24.dim("$")} mutagent agents update <id> --name "New Name"
|
|
7403
|
+
${chalk24.dim("$")} mutagent agents update <id> --system-prompt "Updated prompt" --status active
|
|
7404
|
+
${chalk24.dim("$")} mutagent agents update <id> -d '{"name":"New Name","systemPrompt":"Updated prompt"}'
|
|
6939
7405
|
|
|
6940
7406
|
Input Methods (pick one, priority order):
|
|
6941
|
-
--name/--system-prompt/... Individual flags ${
|
|
7407
|
+
--name/--system-prompt/... Individual flags ${chalk24.green("(recommended)")}
|
|
6942
7408
|
-d, --data Inline JSON object (CI/scripts/agents)
|
|
6943
7409
|
|
|
6944
|
-
${
|
|
7410
|
+
${chalk24.dim("CLI flags override --data fields.")}
|
|
6945
7411
|
`).action(async (id, options) => {
|
|
6946
7412
|
const isJson = getJsonFlag(agents);
|
|
6947
7413
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -6980,7 +7446,8 @@ ${chalk18.dim("CLI flags override --data fields.")}
|
|
|
6980
7446
|
}
|
|
6981
7447
|
const client = await getSDKClient();
|
|
6982
7448
|
const agent = await client.updateAgent(id, data);
|
|
6983
|
-
|
|
7449
|
+
if (!isJson)
|
|
7450
|
+
output.success(`Updated agent: ${agent.name}`);
|
|
6984
7451
|
output.output(agent);
|
|
6985
7452
|
} catch (error) {
|
|
6986
7453
|
handleError(error, isJson);
|
|
@@ -6988,11 +7455,11 @@ ${chalk18.dim("CLI flags override --data fields.")}
|
|
|
6988
7455
|
});
|
|
6989
7456
|
agents.command("delete").description("Delete an agent").argument("<id>", "Agent ID").option("--force", "Skip confirmation").addHelpText("after", `
|
|
6990
7457
|
Examples:
|
|
6991
|
-
${
|
|
6992
|
-
${
|
|
6993
|
-
${
|
|
7458
|
+
${chalk24.dim("$")} mutagent agents delete <id>
|
|
7459
|
+
${chalk24.dim("$")} mutagent agents delete <id> --force
|
|
7460
|
+
${chalk24.dim("$")} mutagent agents delete <id> --force --json
|
|
6994
7461
|
|
|
6995
|
-
${
|
|
7462
|
+
${chalk24.dim("Tip: Use --force to skip confirmation (required for non-interactive/CI usage).")}
|
|
6996
7463
|
`).action(async (id, options) => {
|
|
6997
7464
|
const isJson = getJsonFlag(agents);
|
|
6998
7465
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7021,14 +7488,14 @@ ${chalk18.dim("Tip: Use --force to skip confirmation (required for non-interacti
|
|
|
7021
7488
|
|
|
7022
7489
|
// src/commands/agents/index.ts
|
|
7023
7490
|
function createAgentsCommand() {
|
|
7024
|
-
const agents = new
|
|
7491
|
+
const agents = new Command10("agents").description("Manage AI agents").addHelpText("after", `
|
|
7025
7492
|
Examples:
|
|
7026
|
-
${
|
|
7027
|
-
${
|
|
7028
|
-
${
|
|
7029
|
-
${
|
|
7030
|
-
${
|
|
7031
|
-
${
|
|
7493
|
+
${chalk25.dim("$")} mutagent agents list
|
|
7494
|
+
${chalk25.dim("$")} mutagent agents get <agent-id>
|
|
7495
|
+
${chalk25.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "You are a code reviewer..."
|
|
7496
|
+
${chalk25.dim("$")} mutagent agents create -d '{"name":"Code Reviewer","slug":"code-reviewer","systemPrompt":"You are..."}'
|
|
7497
|
+
${chalk25.dim("$")} mutagent agents update <agent-id> --name "Updated Name"
|
|
7498
|
+
${chalk25.dim("$")} mutagent agents delete <agent-id> --force
|
|
7032
7499
|
|
|
7033
7500
|
Subcommands:
|
|
7034
7501
|
list, get, create, update, delete
|
|
@@ -7039,23 +7506,23 @@ Subcommands:
|
|
|
7039
7506
|
|
|
7040
7507
|
// src/commands/config.ts
|
|
7041
7508
|
init_config();
|
|
7042
|
-
import { Command as
|
|
7043
|
-
import
|
|
7509
|
+
import { Command as Command11 } from "commander";
|
|
7510
|
+
import chalk26 from "chalk";
|
|
7044
7511
|
init_errors();
|
|
7045
7512
|
init_sdk_client();
|
|
7046
7513
|
var VALID_CONFIG_KEYS = ["apiKey", "endpoint", "format", "timeout", "defaultWorkspace", "defaultOrganization"];
|
|
7047
7514
|
function createConfigCommand() {
|
|
7048
|
-
const config = new
|
|
7515
|
+
const config = new Command11("config").description("Manage CLI configuration").addHelpText("after", `
|
|
7049
7516
|
Examples:
|
|
7050
|
-
${
|
|
7051
|
-
${
|
|
7052
|
-
${
|
|
7053
|
-
${
|
|
7517
|
+
${chalk26.dim("$")} mutagent config list
|
|
7518
|
+
${chalk26.dim("$")} mutagent config get endpoint
|
|
7519
|
+
${chalk26.dim("$")} mutagent config set workspace <workspace-id>
|
|
7520
|
+
${chalk26.dim("$")} mutagent config set org <org-id>
|
|
7054
7521
|
`);
|
|
7055
7522
|
config.command("list").description("List all configuration").addHelpText("after", `
|
|
7056
7523
|
Examples:
|
|
7057
|
-
${
|
|
7058
|
-
${
|
|
7524
|
+
${chalk26.dim("$")} mutagent config list
|
|
7525
|
+
${chalk26.dim("$")} mutagent config list --json
|
|
7059
7526
|
`).action(() => {
|
|
7060
7527
|
const isJson = getJsonFlag(config);
|
|
7061
7528
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7068,11 +7535,11 @@ Examples:
|
|
|
7068
7535
|
});
|
|
7069
7536
|
config.command("get").description("Get configuration value").argument("<key>", "Configuration key (apiKey, endpoint, format, timeout, defaultWorkspace, defaultOrganization)").addHelpText("after", `
|
|
7070
7537
|
Examples:
|
|
7071
|
-
${
|
|
7072
|
-
${
|
|
7073
|
-
${
|
|
7538
|
+
${chalk26.dim("$")} mutagent config get endpoint
|
|
7539
|
+
${chalk26.dim("$")} mutagent config get defaultWorkspace
|
|
7540
|
+
${chalk26.dim("$")} mutagent config get apiKey --json
|
|
7074
7541
|
|
|
7075
|
-
${
|
|
7542
|
+
${chalk26.dim("Keys: apiKey, endpoint, format, timeout, defaultWorkspace, defaultOrganization")}
|
|
7076
7543
|
`).action((key) => {
|
|
7077
7544
|
const isJson = getJsonFlag(config);
|
|
7078
7545
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7094,14 +7561,14 @@ ${chalk20.dim("Keys: apiKey, endpoint, format, timeout, defaultWorkspace, defaul
|
|
|
7094
7561
|
handleError(error, isJson);
|
|
7095
7562
|
}
|
|
7096
7563
|
});
|
|
7097
|
-
const set = new
|
|
7564
|
+
const set = new Command11("set").description("Set configuration value").action(() => {
|
|
7098
7565
|
set.help();
|
|
7099
7566
|
});
|
|
7100
7567
|
set.command("workspace").description("Set default workspace ID").argument("<id>", "Workspace ID to set as default").addHelpText("after", `
|
|
7101
7568
|
Examples:
|
|
7102
|
-
${
|
|
7569
|
+
${chalk26.dim("$")} mutagent config set workspace <workspace-id>
|
|
7103
7570
|
|
|
7104
|
-
${
|
|
7571
|
+
${chalk26.dim("Persists workspace ID so you don't need to pass headers on every request.")}
|
|
7105
7572
|
`).action((id) => {
|
|
7106
7573
|
const isJson = getJsonFlag(config);
|
|
7107
7574
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7115,9 +7582,9 @@ ${chalk20.dim("Persists workspace ID so you don't need to pass headers on every
|
|
|
7115
7582
|
});
|
|
7116
7583
|
set.command("org").description("Set default organization ID").argument("<id>", "Organization ID to set as default").addHelpText("after", `
|
|
7117
7584
|
Examples:
|
|
7118
|
-
${
|
|
7585
|
+
${chalk26.dim("$")} mutagent config set org <org-id>
|
|
7119
7586
|
|
|
7120
|
-
${
|
|
7587
|
+
${chalk26.dim("Persists organization ID for org-scoped API keys.")}
|
|
7121
7588
|
`).action((id) => {
|
|
7122
7589
|
const isJson = getJsonFlag(config);
|
|
7123
7590
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7135,8 +7602,8 @@ ${chalk20.dim("Persists organization ID for org-scoped API keys.")}
|
|
|
7135
7602
|
|
|
7136
7603
|
// src/commands/playground.ts
|
|
7137
7604
|
init_sdk_client();
|
|
7138
|
-
import { Command as
|
|
7139
|
-
import
|
|
7605
|
+
import { Command as Command12 } from "commander";
|
|
7606
|
+
import chalk27 from "chalk";
|
|
7140
7607
|
init_errors();
|
|
7141
7608
|
function parseSSELine(line3) {
|
|
7142
7609
|
if (!line3 || line3.startsWith(":")) {
|
|
@@ -7161,13 +7628,13 @@ function parsePromptStreamEvent(data) {
|
|
|
7161
7628
|
}
|
|
7162
7629
|
}
|
|
7163
7630
|
function createPlaygroundCommand() {
|
|
7164
|
-
const playground = new
|
|
7631
|
+
const playground = new Command12("playground").description("Execute and test prompts interactively").addHelpText("after", `
|
|
7165
7632
|
Examples:
|
|
7166
|
-
${
|
|
7167
|
-
${
|
|
7168
|
-
${
|
|
7169
|
-
${
|
|
7170
|
-
${
|
|
7633
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> --input '{"name": "John"}'
|
|
7634
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> --input '{}' --stream
|
|
7635
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> -i '{}' --model gpt-4-turbo
|
|
7636
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> --system "You are helpful" --human "Hello"
|
|
7637
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> --messages '[{"role":"user","content":"Hi"}]'
|
|
7171
7638
|
|
|
7172
7639
|
Input Format:
|
|
7173
7640
|
The input must be a valid JSON object matching the prompt's input schema.
|
|
@@ -7183,16 +7650,16 @@ Streaming:
|
|
|
7183
7650
|
`);
|
|
7184
7651
|
playground.command("run").description("Execute a prompt with input variables").argument("<prompt-id>", "Prompt ID to execute (from: mutagent prompts list)").option("-i, --input <json>", "Input variables as JSON").option("-s, --stream", "Stream the response").option("-m, --model <model>", "Override model").option("--system <text>", "Set system prompt text").option("--human <text>", "Set human/user message text").option("--messages <json>", "Pass full messages array as JSON string").addHelpText("after", `
|
|
7185
7652
|
Examples:
|
|
7186
|
-
${
|
|
7187
|
-
${
|
|
7188
|
-
${
|
|
7189
|
-
${
|
|
7653
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> --input '{"name": "John"}'
|
|
7654
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> --input '{}' --stream
|
|
7655
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> --system "You are helpful" --human "Hello"
|
|
7656
|
+
${chalk27.dim("$")} mutagent playground run <prompt-id> --input '{}' --model gpt-4-turbo --json
|
|
7190
7657
|
|
|
7191
7658
|
Input Methods (pick one, priority order):
|
|
7192
|
-
--system/--human Quick system + user message ${
|
|
7659
|
+
--system/--human Quick system + user message ${chalk27.green("(recommended)")}
|
|
7193
7660
|
--input '{"key":"value"}' Inline JSON variables
|
|
7194
7661
|
--messages '[...]' Full messages array
|
|
7195
|
-
${
|
|
7662
|
+
${chalk27.dim(`Hint: Test before evaluating: mutagent playground run <id> --input '{"key":"value"}'`)}
|
|
7196
7663
|
`).action(async (promptId, options) => {
|
|
7197
7664
|
const isJson = getJsonFlag(playground);
|
|
7198
7665
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7214,21 +7681,21 @@ ${chalk21.dim(`Hint: Test before evaluating: mutagent playground run <id> --inpu
|
|
|
7214
7681
|
}
|
|
7215
7682
|
});
|
|
7216
7683
|
} else {
|
|
7217
|
-
console.log(
|
|
7684
|
+
console.log(chalk27.bold(`
|
|
7218
7685
|
Execution Result:`));
|
|
7219
|
-
console.log(
|
|
7220
|
-
console.log(
|
|
7686
|
+
console.log(chalk27.gray("─".repeat(50)));
|
|
7687
|
+
console.log(chalk27.cyan("Output:"));
|
|
7221
7688
|
console.log(result.output);
|
|
7222
|
-
console.log(
|
|
7223
|
-
console.log(
|
|
7224
|
-
console.log(
|
|
7689
|
+
console.log(chalk27.gray("─".repeat(50)));
|
|
7690
|
+
console.log(chalk27.dim(`Model: ${result.model}`));
|
|
7691
|
+
console.log(chalk27.dim(`Execution Time: ${String(result.executionTimeMs)}ms`));
|
|
7225
7692
|
if (result.tokens) {
|
|
7226
|
-
console.log(
|
|
7693
|
+
console.log(chalk27.dim(`Tokens: ${String(result.tokens.prompt)} prompt + ${String(result.tokens.completion)} completion = ${String(result.tokens.total)} total`));
|
|
7227
7694
|
}
|
|
7228
7695
|
if (result.cost !== undefined) {
|
|
7229
|
-
console.log(
|
|
7696
|
+
console.log(chalk27.dim(`Cost: $${result.cost.toFixed(6)}`));
|
|
7230
7697
|
}
|
|
7231
|
-
console.log(
|
|
7698
|
+
console.log(chalk27.dim(`Playground: ${playgroundLink(promptId)}`));
|
|
7232
7699
|
console.log();
|
|
7233
7700
|
}
|
|
7234
7701
|
}
|
|
@@ -7317,9 +7784,9 @@ async function executeStreaming(client, promptId, input, model, isJson, output)
|
|
|
7317
7784
|
const decoder = new TextDecoder;
|
|
7318
7785
|
let buffer = "";
|
|
7319
7786
|
if (!isJson) {
|
|
7320
|
-
console.log(
|
|
7787
|
+
console.log(chalk27.bold(`
|
|
7321
7788
|
Streaming Output:`));
|
|
7322
|
-
console.log(
|
|
7789
|
+
console.log(chalk27.gray("─".repeat(50)));
|
|
7323
7790
|
}
|
|
7324
7791
|
try {
|
|
7325
7792
|
for (;; ) {
|
|
@@ -7358,15 +7825,15 @@ Streaming Output:`));
|
|
|
7358
7825
|
console.log(JSON.stringify({ type: "complete", result: event.result }));
|
|
7359
7826
|
} else {
|
|
7360
7827
|
console.log();
|
|
7361
|
-
console.log(
|
|
7828
|
+
console.log(chalk27.gray("─".repeat(50)));
|
|
7362
7829
|
if (event.result) {
|
|
7363
|
-
console.log(
|
|
7364
|
-
console.log(
|
|
7830
|
+
console.log(chalk27.dim(`Model: ${event.result.model}`));
|
|
7831
|
+
console.log(chalk27.dim(`Execution Time: ${String(event.result.executionTimeMs)}ms`));
|
|
7365
7832
|
if (event.result.tokens) {
|
|
7366
|
-
console.log(
|
|
7833
|
+
console.log(chalk27.dim(`Tokens: ${String(event.result.tokens.prompt)} prompt + ${String(event.result.tokens.completion)} completion = ${String(event.result.tokens.total)} total`));
|
|
7367
7834
|
}
|
|
7368
7835
|
if (event.result.cost !== undefined) {
|
|
7369
|
-
console.log(
|
|
7836
|
+
console.log(chalk27.dim(`Cost: $${event.result.cost.toFixed(6)}`));
|
|
7370
7837
|
}
|
|
7371
7838
|
}
|
|
7372
7839
|
console.log();
|
|
@@ -7390,14 +7857,14 @@ Streaming Output:`));
|
|
|
7390
7857
|
|
|
7391
7858
|
// src/commands/workspaces.ts
|
|
7392
7859
|
init_sdk_client();
|
|
7393
|
-
import { Command as
|
|
7394
|
-
import
|
|
7860
|
+
import { Command as Command13 } from "commander";
|
|
7861
|
+
import chalk28 from "chalk";
|
|
7395
7862
|
init_errors();
|
|
7396
7863
|
function createWorkspacesCommand() {
|
|
7397
|
-
const workspaces = new
|
|
7864
|
+
const workspaces = new Command13("workspaces").description("View workspaces (read-only)").addHelpText("after", `
|
|
7398
7865
|
Examples:
|
|
7399
|
-
${
|
|
7400
|
-
${
|
|
7866
|
+
${chalk28.dim("$")} mutagent workspaces list
|
|
7867
|
+
${chalk28.dim("$")} mutagent workspaces get <workspace-id>
|
|
7401
7868
|
|
|
7402
7869
|
Subcommands:
|
|
7403
7870
|
list, get
|
|
@@ -7406,8 +7873,8 @@ Note: Workspace management (create, update, delete) is available in the Admin Pa
|
|
|
7406
7873
|
`);
|
|
7407
7874
|
workspaces.command("list").description("List all workspaces").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").addHelpText("after", `
|
|
7408
7875
|
Examples:
|
|
7409
|
-
${
|
|
7410
|
-
${
|
|
7876
|
+
${chalk28.dim("$")} mutagent workspaces list
|
|
7877
|
+
${chalk28.dim("$")} mutagent workspaces list --limit 10 --json
|
|
7411
7878
|
`).action(async (options) => {
|
|
7412
7879
|
const isJson = getJsonFlag(workspaces);
|
|
7413
7880
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7447,8 +7914,8 @@ Examples:
|
|
|
7447
7914
|
});
|
|
7448
7915
|
workspaces.command("get").description("Get workspace details").argument("<id>", "Workspace ID").addHelpText("after", `
|
|
7449
7916
|
Examples:
|
|
7450
|
-
${
|
|
7451
|
-
${
|
|
7917
|
+
${chalk28.dim("$")} mutagent workspaces get <workspace-id>
|
|
7918
|
+
${chalk28.dim("$")} mutagent workspaces get <workspace-id> --json
|
|
7452
7919
|
`).action(async (id) => {
|
|
7453
7920
|
const isJson = getJsonFlag(workspaces);
|
|
7454
7921
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7480,13 +7947,13 @@ Examples:
|
|
|
7480
7947
|
|
|
7481
7948
|
// src/commands/providers/index.ts
|
|
7482
7949
|
init_sdk_client();
|
|
7483
|
-
import { Command as
|
|
7484
|
-
import
|
|
7950
|
+
import { Command as Command14 } from "commander";
|
|
7951
|
+
import chalk32 from "chalk";
|
|
7485
7952
|
init_errors();
|
|
7486
7953
|
|
|
7487
7954
|
// src/commands/providers/add.ts
|
|
7488
7955
|
init_sdk_client();
|
|
7489
|
-
import
|
|
7956
|
+
import chalk29 from "chalk";
|
|
7490
7957
|
init_errors();
|
|
7491
7958
|
function resolveScope(scopeFlag, client) {
|
|
7492
7959
|
const scope = scopeFlag ?? "workspace";
|
|
@@ -7509,10 +7976,10 @@ function resolveScope(scopeFlag, client) {
|
|
|
7509
7976
|
function registerAddCommand(parent) {
|
|
7510
7977
|
parent.command("add").description("Add a new provider configuration").requiredOption("-p, --provider <type>", "Provider type (openai, anthropic, google, ...)").requiredOption("-n, --name <name>", "Display name for this provider").requiredOption("-k, --api-key <key>", "API key for the provider").option("-s, --scope <scope>", "Scope: workspace (default), org, user", "workspace").option("--base-url <url>", "Custom base URL for the provider API").option("--set-default", "Set as default provider for this scope").addHelpText("after", `
|
|
7511
7978
|
Examples:
|
|
7512
|
-
${
|
|
7513
|
-
${
|
|
7514
|
-
${
|
|
7515
|
-
${
|
|
7979
|
+
${chalk29.dim("$")} mutagent providers add --provider openai --name "My OpenAI" --api-key $OPENAI_API_KEY
|
|
7980
|
+
${chalk29.dim("$")} mutagent providers add --provider anthropic --name "Team Claude" --api-key $KEY --scope org
|
|
7981
|
+
${chalk29.dim("$")} mutagent providers add --provider openai --name "Personal" --api-key $KEY --scope user --set-default
|
|
7982
|
+
${chalk29.dim("$")} mutagent providers add --provider custom --name "Ollama" --api-key none --base-url http://localhost:11434 --json
|
|
7516
7983
|
|
|
7517
7984
|
Scope Resolution:
|
|
7518
7985
|
workspace (default) Uses your configured workspace
|
|
@@ -7558,10 +8025,10 @@ The API key is encrypted server-side and never returned in plain text.
|
|
|
7558
8025
|
console.log(` Scope: ${options.scope ?? "workspace"}`);
|
|
7559
8026
|
console.log(` URL: ${providerLink(created.id)}`);
|
|
7560
8027
|
if (options.setDefault) {
|
|
7561
|
-
console.log(
|
|
8028
|
+
console.log(chalk29.green(" Set as default provider"));
|
|
7562
8029
|
}
|
|
7563
8030
|
console.log("");
|
|
7564
|
-
console.log(
|
|
8031
|
+
console.log(chalk29.dim("API key is encrypted server-side. Use `providers get` to see masked key."));
|
|
7565
8032
|
}
|
|
7566
8033
|
} catch (error) {
|
|
7567
8034
|
handleError(error, isJson);
|
|
@@ -7599,15 +8066,15 @@ function buildProviderCreatedDirective(provider, scope) {
|
|
|
7599
8066
|
|
|
7600
8067
|
// src/commands/providers/update.ts
|
|
7601
8068
|
init_sdk_client();
|
|
7602
|
-
import
|
|
8069
|
+
import chalk30 from "chalk";
|
|
7603
8070
|
init_errors();
|
|
7604
8071
|
function registerUpdateCommand(parent) {
|
|
7605
8072
|
parent.command("update").description("Update an existing provider configuration").argument("<id>", "Provider ID (from: mutagent providers list)").option("-n, --name <name>", "Updated display name").option("-k, --api-key <key>", "Updated API key (will be re-encrypted)").option("--active <bool>", "Activate or deactivate (true|false)").option("--set-default", "Set as default provider for its scope").option("--base-url <url>", 'Updated base URL (use "" to clear)').addHelpText("after", `
|
|
7606
8073
|
Examples:
|
|
7607
|
-
${
|
|
7608
|
-
${
|
|
7609
|
-
${
|
|
7610
|
-
${
|
|
8074
|
+
${chalk30.dim("$")} mutagent providers update <id> --name "New Name"
|
|
8075
|
+
${chalk30.dim("$")} mutagent providers update <id> --api-key $NEW_KEY --json
|
|
8076
|
+
${chalk30.dim("$")} mutagent providers update <id> --active false
|
|
8077
|
+
${chalk30.dim("$")} mutagent providers update <id> --set-default --json
|
|
7611
8078
|
|
|
7612
8079
|
PATCH semantics — only provided fields are updated.
|
|
7613
8080
|
`).action(async (id, options) => {
|
|
@@ -7658,7 +8125,7 @@ PATCH semantics — only provided fields are updated.
|
|
|
7658
8125
|
console.log(` ID: ${String(updated.id ?? id)}`);
|
|
7659
8126
|
console.log(` URL: ${providerLink(updated.id ?? id)}`);
|
|
7660
8127
|
if (options.apiKey) {
|
|
7661
|
-
console.log(
|
|
8128
|
+
console.log(chalk30.dim(" API key re-encrypted server-side."));
|
|
7662
8129
|
}
|
|
7663
8130
|
}
|
|
7664
8131
|
} catch (error) {
|
|
@@ -7697,17 +8164,17 @@ function buildProviderUpdatedDirective(provider, requestId) {
|
|
|
7697
8164
|
|
|
7698
8165
|
// src/commands/providers/delete.ts
|
|
7699
8166
|
init_sdk_client();
|
|
7700
|
-
import
|
|
8167
|
+
import chalk31 from "chalk";
|
|
7701
8168
|
init_errors();
|
|
7702
8169
|
function registerDeleteCommand(parent) {
|
|
7703
8170
|
parent.command("delete").description("Delete a provider configuration").argument("<id>", "Provider ID (from: mutagent providers list)").option("-f, --force", "Skip confirmation prompt").addHelpText("after", `
|
|
7704
8171
|
Examples:
|
|
7705
|
-
${
|
|
7706
|
-
${
|
|
7707
|
-
${
|
|
8172
|
+
${chalk31.dim("$")} mutagent providers delete <id>
|
|
8173
|
+
${chalk31.dim("$")} mutagent providers delete <id> --force
|
|
8174
|
+
${chalk31.dim("$")} mutagent providers delete <id> --force --json
|
|
7708
8175
|
|
|
7709
|
-
${
|
|
7710
|
-
${
|
|
8176
|
+
${chalk31.dim("Note: --force is required. The CLI is non-interactive — confirm with the user via your native flow, then pass --force.")}
|
|
8177
|
+
${chalk31.dim("Warning: API keys are AES-256-GCM encrypted and irrecoverable after deletion. Agents referencing this provider will lose their model config.")}
|
|
7711
8178
|
`).action(async (id, options) => {
|
|
7712
8179
|
const isJson = getJsonFlag(parent);
|
|
7713
8180
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7795,14 +8262,14 @@ function validateProviderType(type) {
|
|
|
7795
8262
|
return type;
|
|
7796
8263
|
}
|
|
7797
8264
|
function createProvidersCommand() {
|
|
7798
|
-
const providers = new
|
|
8265
|
+
const providers = new Command14("providers").description("Manage LLM provider configurations (BYOK)").addHelpText("after", `
|
|
7799
8266
|
Examples:
|
|
7800
|
-
${
|
|
7801
|
-
${
|
|
7802
|
-
${
|
|
7803
|
-
${
|
|
7804
|
-
${
|
|
7805
|
-
${
|
|
8267
|
+
${chalk32.dim("$")} mutagent providers list
|
|
8268
|
+
${chalk32.dim("$")} mutagent providers get <provider-id>
|
|
8269
|
+
${chalk32.dim("$")} mutagent providers add --provider openai --name "My OpenAI" --api-key $KEY
|
|
8270
|
+
${chalk32.dim("$")} mutagent providers update <id> --name "New Name"
|
|
8271
|
+
${chalk32.dim("$")} mutagent providers delete <id> --force
|
|
8272
|
+
${chalk32.dim("$")} mutagent providers test <provider-id>
|
|
7806
8273
|
|
|
7807
8274
|
Provider Types:
|
|
7808
8275
|
openai, anthropic, google, azure, bedrock, cohere, mistral, groq, together, replicate, custom
|
|
@@ -7812,9 +8279,9 @@ Subcommands:
|
|
|
7812
8279
|
`);
|
|
7813
8280
|
providers.command("list").description("List all providers").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").option("-t, --type <type>", "Filter by provider type").addHelpText("after", `
|
|
7814
8281
|
Examples:
|
|
7815
|
-
${
|
|
7816
|
-
${
|
|
7817
|
-
${
|
|
8282
|
+
${chalk32.dim("$")} mutagent providers list
|
|
8283
|
+
${chalk32.dim("$")} mutagent providers list --type openai
|
|
8284
|
+
${chalk32.dim("$")} mutagent providers list --json
|
|
7818
8285
|
`).action(async (options) => {
|
|
7819
8286
|
const isJson = getJsonFlag(providers);
|
|
7820
8287
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7863,8 +8330,8 @@ Examples:
|
|
|
7863
8330
|
});
|
|
7864
8331
|
providers.command("get").description("Get provider details").argument("<id>", "Provider ID (from: mutagent providers list)").addHelpText("after", `
|
|
7865
8332
|
Examples:
|
|
7866
|
-
${
|
|
7867
|
-
${
|
|
8333
|
+
${chalk32.dim("$")} mutagent providers get <provider-id>
|
|
8334
|
+
${chalk32.dim("$")} mutagent providers get <provider-id> --json
|
|
7868
8335
|
`).action(async (id) => {
|
|
7869
8336
|
const isJson = getJsonFlag(providers);
|
|
7870
8337
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7893,10 +8360,10 @@ Examples:
|
|
|
7893
8360
|
});
|
|
7894
8361
|
providers.command("test").description("Test provider connectivity").argument("<id>", "Provider ID (from: mutagent providers list)").addHelpText("after", `
|
|
7895
8362
|
Examples:
|
|
7896
|
-
${
|
|
7897
|
-
${
|
|
8363
|
+
${chalk32.dim("$")} mutagent providers test <provider-id>
|
|
8364
|
+
${chalk32.dim("$")} mutagent providers test <provider-id> --json
|
|
7898
8365
|
|
|
7899
|
-
${
|
|
8366
|
+
${chalk32.dim("Tests connectivity and lists available models for the provider.")}
|
|
7900
8367
|
`).action(async (id) => {
|
|
7901
8368
|
const isJson = getJsonFlag(providers);
|
|
7902
8369
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -7911,9 +8378,9 @@ ${chalk26.dim("Tests connectivity and lists available models for the provider.")
|
|
|
7911
8378
|
} else {
|
|
7912
8379
|
if (result.success) {
|
|
7913
8380
|
output.success(`Provider test passed (${String(result.responseTimeMs)}ms)`);
|
|
7914
|
-
console.log(
|
|
8381
|
+
console.log(chalk32.green(`Message: ${result.message}`));
|
|
7915
8382
|
if (result.availableModels && result.availableModels.length > 0) {
|
|
7916
|
-
console.log(
|
|
8383
|
+
console.log(chalk32.bold(`
|
|
7917
8384
|
Available Models:`));
|
|
7918
8385
|
result.availableModels.forEach((model) => {
|
|
7919
8386
|
console.log(` - ${model}`);
|
|
@@ -7922,7 +8389,7 @@ Available Models:`));
|
|
|
7922
8389
|
} else {
|
|
7923
8390
|
output.error(`Provider test failed: ${result.message}`);
|
|
7924
8391
|
if (result.error) {
|
|
7925
|
-
console.log(
|
|
8392
|
+
console.log(chalk32.red(`Error: ${result.error}`));
|
|
7926
8393
|
}
|
|
7927
8394
|
}
|
|
7928
8395
|
}
|
|
@@ -7938,9 +8405,9 @@ Available Models:`));
|
|
|
7938
8405
|
|
|
7939
8406
|
// src/commands/init.ts
|
|
7940
8407
|
init_config();
|
|
7941
|
-
import { Command as
|
|
8408
|
+
import { Command as Command15 } from "commander";
|
|
7942
8409
|
import inquirer2 from "inquirer";
|
|
7943
|
-
import
|
|
8410
|
+
import chalk33 from "chalk";
|
|
7944
8411
|
import { existsSync as existsSync11, mkdirSync as mkdirSync3, writeFileSync as writeFileSync4 } from "fs";
|
|
7945
8412
|
import { execSync as execSync3 } from "child_process";
|
|
7946
8413
|
import { join as join6 } from "path";
|
|
@@ -8061,15 +8528,15 @@ function writeRcConfig(config, cwd = process.cwd()) {
|
|
|
8061
8528
|
`);
|
|
8062
8529
|
}
|
|
8063
8530
|
function createInitCommand() {
|
|
8064
|
-
const init = new
|
|
8531
|
+
const init = new Command15("init").description("Initialize MutagenT in your project").option("--non-interactive", "Skip interactive prompts (defaults to CLI-only mode)").addHelpText("after", `
|
|
8065
8532
|
Examples:
|
|
8066
|
-
${
|
|
8067
|
-
${
|
|
8533
|
+
${chalk33.dim("$")} mutagent init # Interactive setup wizard
|
|
8534
|
+
${chalk33.dim("$")} mutagent init --non-interactive # CLI-only mode (no prompts)
|
|
8068
8535
|
|
|
8069
8536
|
Modes:
|
|
8070
|
-
${
|
|
8071
|
-
${
|
|
8072
|
-
${
|
|
8537
|
+
${chalk33.bold("Full scaffold")} Install SDK + integration package, create config, setup tracing
|
|
8538
|
+
${chalk33.bold("CLI-only")} Verify auth + create .mutagentrc.json with workspace/endpoint
|
|
8539
|
+
${chalk33.bold("Skip")} Exit without changes
|
|
8073
8540
|
`).action(async (options) => {
|
|
8074
8541
|
const isJson = getJsonFlag(init);
|
|
8075
8542
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -8340,24 +8807,24 @@ Modes:
|
|
|
8340
8807
|
}
|
|
8341
8808
|
|
|
8342
8809
|
// src/commands/explore.ts
|
|
8343
|
-
import { Command as
|
|
8344
|
-
import
|
|
8810
|
+
import { Command as Command16 } from "commander";
|
|
8811
|
+
import chalk34 from "chalk";
|
|
8345
8812
|
import { resolve as resolve3 } from "path";
|
|
8346
8813
|
init_errors();
|
|
8347
8814
|
function createExploreCommand() {
|
|
8348
|
-
const explore = new
|
|
8815
|
+
const explore = new Command16("explore").description("Scan codebase for prompts, datasets, and MutagenT markers").option("-p, --path <dir>", "Directory to scan", ".").option("--depth <n>", "Max directory depth", "10").option("--include <glob>", "Include file pattern", "**/*.{ts,js,py,tsx,jsx}").option("--exclude <dirs>", "Comma-separated directories to exclude", "node_modules,dist,.git,build,.next,__pycache__,venv,.venv").option("--markers-only", "Only find existing MutagenT markers").addHelpText("after", `
|
|
8349
8816
|
Examples:
|
|
8350
|
-
${
|
|
8351
|
-
${
|
|
8352
|
-
${
|
|
8353
|
-
${
|
|
8354
|
-
${
|
|
8817
|
+
${chalk34.dim("$")} mutagent explore
|
|
8818
|
+
${chalk34.dim("$")} mutagent explore --path ./src
|
|
8819
|
+
${chalk34.dim("$")} mutagent explore --include "**/*.{ts,py}" --depth 5
|
|
8820
|
+
${chalk34.dim("$")} mutagent explore --markers-only
|
|
8821
|
+
${chalk34.dim("$")} mutagent explore --json
|
|
8355
8822
|
|
|
8356
8823
|
Detection modes:
|
|
8357
|
-
${
|
|
8358
|
-
${
|
|
8824
|
+
${chalk34.dim("Heuristic")} Template variables ({{var}}), prompt constants, schema definitions
|
|
8825
|
+
${chalk34.dim("Marker")} MutagenT:START/END comment markers from previous uploads
|
|
8359
8826
|
|
|
8360
|
-
${
|
|
8827
|
+
${chalk34.dim("Results are saved to .mutagent/mutation-context.md for use by other commands.")}
|
|
8361
8828
|
`).action((options) => {
|
|
8362
8829
|
const isJson = getJsonFlag(explore);
|
|
8363
8830
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -8375,7 +8842,7 @@ ${chalk28.dim("Results are saved to .mutagent/mutation-context.md for use by oth
|
|
|
8375
8842
|
markersOnly
|
|
8376
8843
|
};
|
|
8377
8844
|
if (!isJson) {
|
|
8378
|
-
console.log(
|
|
8845
|
+
console.log(chalk34.cyan(`
|
|
8379
8846
|
Scanning ${scanPath}...
|
|
8380
8847
|
`));
|
|
8381
8848
|
}
|
|
@@ -8404,41 +8871,41 @@ Scanning ${scanPath}...
|
|
|
8404
8871
|
const totalFindings = result.prompts.length + result.datasets.length + result.markers.length;
|
|
8405
8872
|
if (totalFindings === 0) {
|
|
8406
8873
|
output.info("No prompts, datasets, or markers found.");
|
|
8407
|
-
console.log(
|
|
8874
|
+
console.log(chalk34.dim(`
|
|
8408
8875
|
Tip: Create a prompt with template variables like {{input}} to get started.`));
|
|
8409
8876
|
return;
|
|
8410
8877
|
}
|
|
8411
8878
|
if (result.prompts.length > 0) {
|
|
8412
|
-
console.log(
|
|
8879
|
+
console.log(chalk34.bold(` Prompts Found (${String(result.prompts.length)}):`));
|
|
8413
8880
|
console.log();
|
|
8414
8881
|
for (const p of result.prompts) {
|
|
8415
|
-
const confidenceTag = p.confidence === "high" ?
|
|
8416
|
-
const reasonTag =
|
|
8417
|
-
console.log(` ${confidenceTag} ${
|
|
8418
|
-
console.log(` ${
|
|
8882
|
+
const confidenceTag = p.confidence === "high" ? chalk34.green("[high]") : p.confidence === "medium" ? chalk34.yellow("[medium]") : chalk34.dim("[low]");
|
|
8883
|
+
const reasonTag = chalk34.dim(`[${p.reason}]`);
|
|
8884
|
+
console.log(` ${confidenceTag} ${chalk34.green(p.file)}:${chalk34.yellow(String(p.line))} ${reasonTag}`);
|
|
8885
|
+
console.log(` ${chalk34.dim(p.preview)}`);
|
|
8419
8886
|
}
|
|
8420
8887
|
console.log();
|
|
8421
8888
|
}
|
|
8422
8889
|
if (result.datasets.length > 0) {
|
|
8423
|
-
console.log(
|
|
8890
|
+
console.log(chalk34.bold(` Datasets Found (${String(result.datasets.length)}):`));
|
|
8424
8891
|
console.log();
|
|
8425
8892
|
for (const d of result.datasets) {
|
|
8426
|
-
console.log(` ${
|
|
8893
|
+
console.log(` ${chalk34.green(d.file)} ${chalk34.dim(`(${String(d.items)} items)`)}`);
|
|
8427
8894
|
}
|
|
8428
8895
|
console.log();
|
|
8429
8896
|
}
|
|
8430
8897
|
if (result.markers.length > 0) {
|
|
8431
|
-
console.log(
|
|
8898
|
+
console.log(chalk34.bold(` MutagenT Markers (${String(result.markers.length)}):`));
|
|
8432
8899
|
console.log();
|
|
8433
8900
|
for (const m of result.markers) {
|
|
8434
|
-
const idPart = m.platformId ?
|
|
8435
|
-
console.log(` ${
|
|
8901
|
+
const idPart = m.platformId ? chalk34.cyan(` id=${m.platformId}`) : "";
|
|
8902
|
+
console.log(` ${chalk34.green(m.file)}:${chalk34.yellow(String(m.line))} ${chalk34.magenta(m.type)}${idPart}`);
|
|
8436
8903
|
}
|
|
8437
8904
|
console.log();
|
|
8438
8905
|
}
|
|
8439
|
-
console.log(
|
|
8440
|
-
console.log(` ${
|
|
8441
|
-
console.log(
|
|
8906
|
+
console.log(chalk34.dim(" ─────────────────────────────────"));
|
|
8907
|
+
console.log(` ${chalk34.bold("Summary:")} ${String(result.prompts.length)} prompts, ${String(result.datasets.length)} datasets, ${String(result.markers.length)} markers`);
|
|
8908
|
+
console.log(chalk34.dim(` Saved to .mutagent/mutation-context.md`));
|
|
8442
8909
|
console.log();
|
|
8443
8910
|
}
|
|
8444
8911
|
} catch (error) {
|
|
@@ -8449,8 +8916,8 @@ Scanning ${scanPath}...
|
|
|
8449
8916
|
}
|
|
8450
8917
|
|
|
8451
8918
|
// src/commands/skills.ts
|
|
8452
|
-
import { Command as
|
|
8453
|
-
import
|
|
8919
|
+
import { Command as Command17 } from "commander";
|
|
8920
|
+
import chalk35 from "chalk";
|
|
8454
8921
|
import { existsSync as existsSync12, mkdirSync as mkdirSync4, writeFileSync as writeFileSync5 } from "fs";
|
|
8455
8922
|
import { join as join7 } from "path";
|
|
8456
8923
|
import { execSync as execSync4 } from "child_process";
|
|
@@ -8571,10 +9038,10 @@ Or run: \`mutagent init\` (option 1 installs skill + hooks together)`;
|
|
|
8571
9038
|
var SKILL_DIR = ".claude/skills/mutagent-cli";
|
|
8572
9039
|
var SKILL_FILE = "SKILL.md";
|
|
8573
9040
|
function createSkillsCommand() {
|
|
8574
|
-
const skills = new
|
|
9041
|
+
const skills = new Command17("skills").description("Manage MutagenT CLI skills for coding agents");
|
|
8575
9042
|
skills.command("install").description("Install MutagenT CLI skill for Claude Code").addHelpText("after", `
|
|
8576
9043
|
Examples:
|
|
8577
|
-
${
|
|
9044
|
+
${chalk35.dim("$")} mutagent skills install
|
|
8578
9045
|
|
|
8579
9046
|
This creates a Claude Code skill at .claude/skills/mutagent-cli/SKILL.md
|
|
8580
9047
|
that teaches coding agents how to use the MutagenT CLI effectively.
|
|
@@ -8601,10 +9068,10 @@ ${SKILL_BODY}
|
|
|
8601
9068
|
});
|
|
8602
9069
|
} else {
|
|
8603
9070
|
output.success(`Installed MutagenT CLI skill`);
|
|
8604
|
-
console.log(` ${
|
|
9071
|
+
console.log(` ${chalk35.dim("Path:")} ${skillPath}`);
|
|
8605
9072
|
console.log("");
|
|
8606
|
-
console.log(` ${
|
|
8607
|
-
console.log(` ${
|
|
9073
|
+
console.log(` ${chalk35.dim("This skill teaches coding agents how to use the MutagenT CLI.")}`);
|
|
9074
|
+
console.log(` ${chalk35.dim("It will be automatically loaded by Claude Code when relevant triggers match.")}`);
|
|
8608
9075
|
}
|
|
8609
9076
|
});
|
|
8610
9077
|
return skills;
|
|
@@ -8612,16 +9079,16 @@ ${SKILL_BODY}
|
|
|
8612
9079
|
|
|
8613
9080
|
// src/commands/usage.ts
|
|
8614
9081
|
init_config();
|
|
8615
|
-
import { Command as
|
|
8616
|
-
import
|
|
9082
|
+
import { Command as Command18 } from "commander";
|
|
9083
|
+
import chalk36 from "chalk";
|
|
8617
9084
|
init_errors();
|
|
8618
9085
|
init_sdk_client();
|
|
8619
9086
|
var PROVIDERS_URL = "https://app.mutagent.io/settings/providers";
|
|
8620
9087
|
function createUsageCommand() {
|
|
8621
|
-
const usage = new
|
|
9088
|
+
const usage = new Command18("usage").description("Show resource counts (prompts, datasets, evaluations, optimizations, experiments)").addHelpText("after", `
|
|
8622
9089
|
Examples:
|
|
8623
|
-
${
|
|
8624
|
-
${
|
|
9090
|
+
${chalk36.dim("$")} mutagent usage
|
|
9091
|
+
${chalk36.dim("$")} mutagent usage --json
|
|
8625
9092
|
`);
|
|
8626
9093
|
usage.action(async () => {
|
|
8627
9094
|
const isJson = getJsonFlag(usage);
|
|
@@ -8676,17 +9143,17 @@ Examples:
|
|
|
8676
9143
|
});
|
|
8677
9144
|
} else {
|
|
8678
9145
|
console.log("");
|
|
8679
|
-
console.log(
|
|
8680
|
-
console.log(
|
|
9146
|
+
console.log(chalk36.bold("\uD83D\uDCCA MutagenT Usage"));
|
|
9147
|
+
console.log(chalk36.dim("─".repeat(45)));
|
|
8681
9148
|
console.log("");
|
|
8682
|
-
console.log(
|
|
8683
|
-
console.log(` Prompts: ${
|
|
8684
|
-
console.log(` Datasets: ${
|
|
8685
|
-
console.log(` Evaluations: ${
|
|
8686
|
-
console.log(` Optimizations: ${
|
|
8687
|
-
console.log(` Experiments: ${
|
|
9149
|
+
console.log(chalk36.bold("Resources:"));
|
|
9150
|
+
console.log(` Prompts: ${chalk36.cyan(String(promptCount))}`);
|
|
9151
|
+
console.log(` Datasets: ${chalk36.cyan(String(datasetCount))}`);
|
|
9152
|
+
console.log(` Evaluations: ${chalk36.cyan(String(evaluationCount))}`);
|
|
9153
|
+
console.log(` Optimizations: ${chalk36.cyan(String(optimizationCount))}`);
|
|
9154
|
+
console.log(` Experiments: ${chalk36.cyan(String(experimentCount))}`);
|
|
8688
9155
|
console.log("");
|
|
8689
|
-
console.log(` Providers: ${
|
|
9156
|
+
console.log(` Providers: ${chalk36.underline(PROVIDERS_URL)}`);
|
|
8690
9157
|
console.log("");
|
|
8691
9158
|
}
|
|
8692
9159
|
} catch (error) {
|
|
@@ -8698,7 +9165,7 @@ Examples:
|
|
|
8698
9165
|
|
|
8699
9166
|
// src/commands/hooks.ts
|
|
8700
9167
|
init_config();
|
|
8701
|
-
import { Command as
|
|
9168
|
+
import { Command as Command19 } from "commander";
|
|
8702
9169
|
import { randomUUID } from "crypto";
|
|
8703
9170
|
import { join as join8 } from "path";
|
|
8704
9171
|
import { tmpdir } from "os";
|
|
@@ -8939,7 +9406,7 @@ async function handlePostToolUse() {
|
|
|
8939
9406
|
]);
|
|
8940
9407
|
}
|
|
8941
9408
|
function createHooksCommand() {
|
|
8942
|
-
const hooks = new
|
|
9409
|
+
const hooks = new Command19("hooks").description("Hook handlers for AI coding assistants").addHelpText("after", `
|
|
8943
9410
|
Claude Code Session Telemetry:
|
|
8944
9411
|
Sends lightweight session activity to the MutagenT traces API for observability.
|
|
8945
9412
|
|
|
@@ -8973,8 +9440,8 @@ Claude Code Session Telemetry:
|
|
|
8973
9440
|
}
|
|
8974
9441
|
|
|
8975
9442
|
// src/commands/feedback.ts
|
|
8976
|
-
import { Command as
|
|
8977
|
-
import
|
|
9443
|
+
import { Command as Command20 } from "commander";
|
|
9444
|
+
import chalk37 from "chalk";
|
|
8978
9445
|
init_errors();
|
|
8979
9446
|
init_config();
|
|
8980
9447
|
import { readFileSync as readFileSync11 } from "fs";
|
|
@@ -9027,13 +9494,13 @@ async function postToServer(payload, endpoint, apiKey, workspaceId, organization
|
|
|
9027
9494
|
throw new MutagentError("API_ERROR", errorMessage, `Server responded with status ${String(response.status)}. Check your configuration with: mutagent config show`);
|
|
9028
9495
|
}
|
|
9029
9496
|
function createFeedbackCommand() {
|
|
9030
|
-
const feedback = new
|
|
9031
|
-
${
|
|
9032
|
-
${
|
|
9033
|
-
${
|
|
9034
|
-
${
|
|
9497
|
+
const feedback = new Command20("feedback").description("Send product feedback to MutagenT").addHelpText("after", `
|
|
9498
|
+
${chalk37.bold("Examples:")}
|
|
9499
|
+
${chalk37.cyan('mutagent feedback send -m "Great optimization results!"')}
|
|
9500
|
+
${chalk37.cyan('mutagent feedback send -m "CLI crashed on export" --category bug')}
|
|
9501
|
+
${chalk37.cyan('mutagent feedback send -m "Need batch operations" --category feature --json')}
|
|
9035
9502
|
|
|
9036
|
-
${
|
|
9503
|
+
${chalk37.yellow("AI Agent (MANDATORY):")}
|
|
9037
9504
|
ALWAYS use --json: mutagent feedback send -m "..." --category improvement --json
|
|
9038
9505
|
Use this command to report bugs, request features, or share UX feedback.
|
|
9039
9506
|
`).action(() => {
|
|
@@ -9044,18 +9511,18 @@ ${chalk31.yellow("AI Agent (MANDATORY):")}
|
|
|
9044
9511
|
}
|
|
9045
9512
|
function registerFeedbackSend(feedback) {
|
|
9046
9513
|
feedback.command("send").description("Send feedback about the MutagenT platform").requiredOption("-m, --message <text>", "Feedback message").option("--category <type>", `Feedback category: ${VALID_CATEGORIES.join(", ")}`, "improvement").option("--session <id>", "Link feedback to a specific session").addHelpText("after", `
|
|
9047
|
-
${
|
|
9048
|
-
${
|
|
9049
|
-
${
|
|
9050
|
-
${
|
|
9051
|
-
|
|
9052
|
-
${
|
|
9053
|
-
${
|
|
9054
|
-
${
|
|
9055
|
-
${
|
|
9056
|
-
${
|
|
9057
|
-
|
|
9058
|
-
${
|
|
9514
|
+
${chalk37.bold("Examples:")}
|
|
9515
|
+
${chalk37.dim("$")} mutagent feedback send -m "The optimization UX could show progress better"
|
|
9516
|
+
${chalk37.dim("$")} mutagent feedback send -m "CLI errored on traces export" --category bug
|
|
9517
|
+
${chalk37.dim("$")} mutagent feedback send -m "Love the guided eval!" --category praise --json
|
|
9518
|
+
|
|
9519
|
+
${chalk37.bold("Categories:")}
|
|
9520
|
+
${chalk37.bold("bug")} Something is broken or not working as expected
|
|
9521
|
+
${chalk37.bold("feature")} Request a new capability
|
|
9522
|
+
${chalk37.bold("improvement")} Suggest a UX or workflow enhancement (default)
|
|
9523
|
+
${chalk37.bold("praise")} Share what you love about the platform
|
|
9524
|
+
|
|
9525
|
+
${chalk37.yellow("AI Agent (MANDATORY):")}
|
|
9059
9526
|
ALWAYS use --json: mutagent feedback send -m "..." --json
|
|
9060
9527
|
Auto-captured context (CLI version, platform, node version) is included automatically.
|
|
9061
9528
|
`).action(async (options) => {
|
|
@@ -9118,7 +9585,7 @@ if (process.env.CLI_VERSION) {
|
|
|
9118
9585
|
cliVersion = pkg.version ?? cliVersion;
|
|
9119
9586
|
} catch {}
|
|
9120
9587
|
}
|
|
9121
|
-
var program = new
|
|
9588
|
+
var program = new Command21;
|
|
9122
9589
|
program.name("mutagent").description(`MutagenT CLI - AI-native prompt optimization platform
|
|
9123
9590
|
|
|
9124
9591
|
Documentation: https://docs.mutagent.io/cli
|
|
@@ -9127,90 +9594,90 @@ program.name("mutagent").description(`MutagenT CLI - AI-native prompt optimizati
|
|
|
9127
9594
|
showGlobalOptions: true
|
|
9128
9595
|
});
|
|
9129
9596
|
program.addHelpText("after", `
|
|
9130
|
-
${
|
|
9131
|
-
export MUTAGENT_API_KEY=mt_... ${
|
|
9132
|
-
--json ${
|
|
9133
|
-
|
|
9134
|
-
${
|
|
9135
|
-
mutagent login ${
|
|
9136
|
-
mutagent auth status ${
|
|
9137
|
-
mutagent init ${
|
|
9138
|
-
mutagent explore ${
|
|
9139
|
-
mutagent workspaces list --json ${
|
|
9140
|
-
mutagent config set workspace <id> ${
|
|
9141
|
-
mutagent usage --json ${
|
|
9142
|
-
|
|
9143
|
-
mutagent prompts create --help ${
|
|
9144
|
-
mutagent prompts list --json ${
|
|
9145
|
-
mutagent prompts get <id> --json ${
|
|
9146
|
-
|
|
9147
|
-
mutagent prompts dataset add --help ${
|
|
9148
|
-
mutagent prompts dataset list <id> ${
|
|
9149
|
-
|
|
9150
|
-
mutagent prompts evaluation create --help ${
|
|
9151
|
-
mutagent prompts evaluation create <id> --guided --json ${
|
|
9152
|
-
mutagent prompts evaluation list <id> --json ${
|
|
9153
|
-
|
|
9154
|
-
mutagent prompts optimize start --help ${
|
|
9155
|
-
mutagent prompts optimize status <job-id> ${
|
|
9156
|
-
mutagent prompts optimize results <job-id> ${
|
|
9157
|
-
|
|
9158
|
-
mutagent feedback send -m "..." ${
|
|
9159
|
-
mutagent feedback send -m "..." --category bug ${
|
|
9160
|
-
|
|
9161
|
-
mutagent integrate <framework> ${
|
|
9162
|
-
mutagent hooks --help ${
|
|
9163
|
-
mutagent playground run <id> --input '{...}' ${
|
|
9164
|
-
|
|
9165
|
-
${
|
|
9166
|
-
1. mutagent explore ${
|
|
9167
|
-
2. mutagent integrate <framework> ${
|
|
9168
|
-
3. Apply tracing code to your codebase ${
|
|
9169
|
-
4. mutagent traces list --json ${
|
|
9170
|
-
|
|
9171
|
-
${
|
|
9172
|
-
1. mutagent prompts create --help ${
|
|
9173
|
-
2. mutagent prompts create ... --json ${
|
|
9174
|
-
3. mutagent prompts dataset add --help ${
|
|
9175
|
-
4. mutagent prompts dataset add <id> ... --json ${
|
|
9176
|
-
5. mutagent prompts evaluation create <id> --guided --json ${
|
|
9597
|
+
${chalk38.yellow("Non-Interactive Mode (CI/CD & Coding Agents):")}
|
|
9598
|
+
export MUTAGENT_API_KEY=mt_... ${chalk38.dim("or")} --api-key mt_...
|
|
9599
|
+
--json ${chalk38.dim("for structured output")} --non-interactive ${chalk38.dim("to disable prompts")}
|
|
9600
|
+
|
|
9601
|
+
${chalk38.yellow("Command Navigation:")}
|
|
9602
|
+
mutagent login ${chalk38.dim("Login (browser OAuth — recommended)")}
|
|
9603
|
+
mutagent auth status ${chalk38.dim("Check auth + workspace")}
|
|
9604
|
+
mutagent init ${chalk38.dim("Initialize project (.mutagentrc.json)")}
|
|
9605
|
+
mutagent explore ${chalk38.dim("Discover prompts in codebase")}
|
|
9606
|
+
mutagent workspaces list --json ${chalk38.dim("List workspaces (verify ID)")}
|
|
9607
|
+
mutagent config set workspace <id> ${chalk38.dim("Set active workspace")}
|
|
9608
|
+
mutagent usage --json ${chalk38.dim("Show account usage + provider status")}
|
|
9609
|
+
|
|
9610
|
+
mutagent prompts create --help ${chalk38.dim("Upload prompt (read help first!)")}
|
|
9611
|
+
mutagent prompts list --json ${chalk38.dim("List prompts")}
|
|
9612
|
+
mutagent prompts get <id> --json ${chalk38.dim("Full prompt details + schemas")}
|
|
9613
|
+
|
|
9614
|
+
mutagent prompts dataset add --help ${chalk38.dim("Upload dataset (read help first!)")}
|
|
9615
|
+
mutagent prompts dataset list <id> ${chalk38.dim("List datasets")}
|
|
9616
|
+
|
|
9617
|
+
mutagent prompts evaluation create --help ${chalk38.dim("Create eval (read help first!)")}
|
|
9618
|
+
mutagent prompts evaluation create <id> --guided --json ${chalk38.dim("Guided eval workflow")}
|
|
9619
|
+
mutagent prompts evaluation list <id> --json ${chalk38.dim("List evaluations")}
|
|
9620
|
+
|
|
9621
|
+
mutagent prompts optimize start --help ${chalk38.dim("Run optimization (read help first!)")}
|
|
9622
|
+
mutagent prompts optimize status <job-id> ${chalk38.dim("Poll progress")}
|
|
9623
|
+
mutagent prompts optimize results <job-id> ${chalk38.dim("View scorecard")}
|
|
9624
|
+
|
|
9625
|
+
mutagent feedback send -m "..." ${chalk38.dim("Send product feedback")}
|
|
9626
|
+
mutagent feedback send -m "..." --category bug ${chalk38.dim("Report a bug")}
|
|
9627
|
+
|
|
9628
|
+
mutagent integrate <framework> ${chalk38.dim("Framework integration guide")}
|
|
9629
|
+
mutagent hooks --help ${chalk38.dim("Hook setup for Claude Code telemetry")}
|
|
9630
|
+
mutagent playground run <id> --input '{...}' ${chalk38.dim("Quick test")}
|
|
9631
|
+
|
|
9632
|
+
${chalk38.yellow("★ Workflow: Framework Integration (Tracing):")}
|
|
9633
|
+
1. mutagent explore ${chalk38.dim("← discover prompts/agents in codebase")}
|
|
9634
|
+
2. mutagent integrate <framework> ${chalk38.dim("← get integration instructions")}
|
|
9635
|
+
3. Apply tracing code to your codebase ${chalk38.dim("← follow the guide output")}
|
|
9636
|
+
4. mutagent traces list --json ${chalk38.dim("← verify traces are arriving")}
|
|
9637
|
+
|
|
9638
|
+
${chalk38.yellow("★ Workflow: Evaluate → Optimize:")}
|
|
9639
|
+
1. mutagent prompts create --help ${chalk38.dim("← read help")}
|
|
9640
|
+
2. mutagent prompts create ... --json ${chalk38.dim("← upload prompt with {variables} + inputSchema")}
|
|
9641
|
+
3. mutagent prompts dataset add --help ${chalk38.dim("← read help")}
|
|
9642
|
+
4. mutagent prompts dataset add <id> ... --json ${chalk38.dim("← upload dataset")}
|
|
9643
|
+
5. mutagent prompts evaluation create <id> --guided --json ${chalk38.dim("← guided eval")}
|
|
9177
9644
|
6. mutagent prompts optimize start <id> --dataset <d> --evaluation <e> --json
|
|
9178
9645
|
|
|
9179
|
-
${
|
|
9180
|
-
After ${
|
|
9181
|
-
${
|
|
9182
|
-
${
|
|
9183
|
-
${
|
|
9646
|
+
${chalk38.yellow("Post-Onboarding Decision Tree:")}
|
|
9647
|
+
After ${chalk38.bold("mutagent auth login")}, users land in one of 3 paths:
|
|
9648
|
+
${chalk38.bold("Path A")} (Tracing): explore → integrate <framework> → apply tracing → verify
|
|
9649
|
+
${chalk38.bold("Path B")} (Optimization): explore → prompts create → dataset add → eval create → optimize
|
|
9650
|
+
${chalk38.bold("Path C")} (Manual): Use CLI commands directly — run mutagent <command> --help
|
|
9184
9651
|
|
|
9185
|
-
${
|
|
9652
|
+
${chalk38.yellow("Directive System:")}
|
|
9186
9653
|
Every --json response may include:
|
|
9187
|
-
${
|
|
9188
|
-
${
|
|
9189
|
-
${
|
|
9190
|
-
${
|
|
9654
|
+
${chalk38.bold("_directive.renderedCard")} Pre-formatted card for the user ${chalk38.red("(MUST be shown in chat)")}
|
|
9655
|
+
${chalk38.bold("_directive.instruction")} Next step for the agent
|
|
9656
|
+
${chalk38.bold("_directive.next")} Array of suggested follow-up commands
|
|
9657
|
+
${chalk38.bold("_links")} Dashboard/API URLs (format as markdown links)
|
|
9191
9658
|
|
|
9192
|
-
${
|
|
9193
|
-
Each criterion MUST have: ${
|
|
9659
|
+
${chalk38.yellow("Evaluation Criteria Format:")}
|
|
9660
|
+
Each criterion MUST have: ${chalk38.bold("name")}, ${chalk38.bold("description")} (scoring rubric), ${chalk38.bold("evaluationParameter")}
|
|
9194
9661
|
evaluationParameter MUST match an inputSchema or outputSchema field name
|
|
9195
9662
|
No duplicate evaluationParameter values — each criterion targets a unique field
|
|
9196
9663
|
ALL schema fields must be covered (missing fields = error)
|
|
9197
|
-
Use ${
|
|
9664
|
+
Use ${chalk38.bold("--guided --json")} to generate criteria templates from prompt schemas
|
|
9198
9665
|
|
|
9199
|
-
${
|
|
9200
|
-
Default max-iterations is 1. ${
|
|
9666
|
+
${chalk38.yellow("Optimization Cost Control:")}
|
|
9667
|
+
Default max-iterations is 1. ${chalk38.red("NEVER increase without explicit user request.")}
|
|
9201
9668
|
Each iteration incurs LLM costs — confirm with user before starting >1.
|
|
9202
9669
|
|
|
9203
|
-
${
|
|
9204
|
-
After ${
|
|
9205
|
-
Then offer choices: ${
|
|
9670
|
+
${chalk38.yellow("Post-Optimization:")}
|
|
9671
|
+
After ${chalk38.bold("optimize results")}: ALWAYS show the before/after diff to the user first.
|
|
9672
|
+
Then offer choices: ${chalk38.bold("Apply")} / ${chalk38.bold("Reject")}.
|
|
9206
9673
|
|
|
9207
|
-
${
|
|
9674
|
+
${chalk38.yellow("State Tracking:")}
|
|
9208
9675
|
.mutagent/mutation-context.md — Codebase index of discovered/uploaded prompts
|
|
9209
9676
|
Update after explore, create, and dataset operations
|
|
9210
9677
|
mutagent auth status — Auth + workspace state
|
|
9211
9678
|
Comment markers (// MutagenT:START ... // MutagenT:END) in source files
|
|
9212
9679
|
|
|
9213
|
-
${
|
|
9680
|
+
${chalk38.yellow("AI Agent Rules (MANDATORY for coding agents):")}
|
|
9214
9681
|
1. Login (two paths):
|
|
9215
9682
|
- CI / fully automated: export MUTAGENT_API_KEY=mt_... then mutagent login --json
|
|
9216
9683
|
- Helping a user onboard: mutagent login --browser --json
|
|
@@ -9221,15 +9688,15 @@ ${chalk32.yellow("AI Agent Rules (MANDATORY for coding agents):")}
|
|
|
9221
9688
|
3. Run <command> --help BEFORE first use of any command
|
|
9222
9689
|
4. Use --guided --json for evaluation creation (NEVER --guided alone)
|
|
9223
9690
|
5. Parse _directive.renderedCard and copy it into your CHAT RESPONSE verbatim
|
|
9224
|
-
${
|
|
9691
|
+
${chalk38.red("HARD STOP")}: do NOT run further commands until the card is rendered in chat
|
|
9225
9692
|
6. After mutagent init, verify workspace: mutagent workspaces list --json
|
|
9226
9693
|
7. Use {single_braces} for template variables in prompts
|
|
9227
9694
|
8. Collect evaluation criteria from the user — NEVER auto-generate
|
|
9228
9695
|
9. ALL user interaction via AskUserQuestion — CLI is non-interactive
|
|
9229
9696
|
${!hasCredentials() ? `
|
|
9230
|
-
` +
|
|
9697
|
+
` + chalk38.yellow(" Warning: Not authenticated. Run: mutagent login") + `
|
|
9231
9698
|
` : ""}${!hasRcConfig() ? `
|
|
9232
|
-
` +
|
|
9699
|
+
` + chalk38.green(" Get started: mutagent init") + `
|
|
9233
9700
|
` : ""}`);
|
|
9234
9701
|
var rawArgs = process.argv.slice(2);
|
|
9235
9702
|
if (rawArgs.includes("-v") || rawArgs.includes("--version")) {
|
|
@@ -9270,5 +9737,5 @@ program.addCommand(createHooksCommand());
|
|
|
9270
9737
|
program.addCommand(createFeedbackCommand());
|
|
9271
9738
|
program.parse();
|
|
9272
9739
|
|
|
9273
|
-
//# debugId=
|
|
9740
|
+
//# debugId=C3949CD3A9488F4264756E2164756E21
|
|
9274
9741
|
//# sourceMappingURL=cli.js.map
|