@mutagent/cli 0.1.16 → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cli.js +455 -106
- package/dist/bin/cli.js.map +15 -14
- package/dist/index.js +27 -2
- package/dist/index.js.map +3 -3
- package/package.json +1 -1
package/dist/bin/cli.js
CHANGED
|
@@ -577,6 +577,13 @@ class SDKClientWrapper {
|
|
|
577
577
|
this.handleError(error);
|
|
578
578
|
}
|
|
579
579
|
}
|
|
580
|
+
async deleteEvaluation(evaluationId) {
|
|
581
|
+
try {
|
|
582
|
+
await this.request(`/api/prompts/evaluations/${evaluationId}`, { method: "DELETE" });
|
|
583
|
+
} catch (error) {
|
|
584
|
+
this.handleError(error);
|
|
585
|
+
}
|
|
586
|
+
}
|
|
580
587
|
async getEvaluationResults(runId) {
|
|
581
588
|
try {
|
|
582
589
|
return await this.request(`/api/prompts/evaluations/${runId}/result`);
|
|
@@ -592,7 +599,7 @@ class SDKClientWrapper {
|
|
|
592
599
|
datasetId: parseInt(datasetId, 10),
|
|
593
600
|
config: {
|
|
594
601
|
maxIterations: config?.maxIterations ?? 1,
|
|
595
|
-
targetScore: config?.targetScore,
|
|
602
|
+
targetScore: config?.targetScore ?? 0.8,
|
|
596
603
|
patience: config?.patience,
|
|
597
604
|
model: config?.model,
|
|
598
605
|
...config?.evalModel ? { tuningParams: { evaluationModel: config.evalModel } } : {}
|
|
@@ -931,8 +938,8 @@ var init_sdk_client = __esm(() => {
|
|
|
931
938
|
});
|
|
932
939
|
|
|
933
940
|
// src/bin/cli.ts
|
|
934
|
-
import { Command as
|
|
935
|
-
import
|
|
941
|
+
import { Command as Command15 } from "commander";
|
|
942
|
+
import chalk19 from "chalk";
|
|
936
943
|
import { readFileSync as readFileSync12 } from "fs";
|
|
937
944
|
import { join as join7, dirname } from "path";
|
|
938
945
|
import { fileURLToPath } from "url";
|
|
@@ -1067,6 +1074,24 @@ ${String(data.length)} result(s)`));
|
|
|
1067
1074
|
}
|
|
1068
1075
|
}
|
|
1069
1076
|
}
|
|
1077
|
+
function createSpinner(text, isJson) {
|
|
1078
|
+
if (isJson) {
|
|
1079
|
+
const noop = () => {
|
|
1080
|
+
return;
|
|
1081
|
+
};
|
|
1082
|
+
return {
|
|
1083
|
+
start: noop,
|
|
1084
|
+
stop: noop,
|
|
1085
|
+
succeed: noop,
|
|
1086
|
+
fail: noop
|
|
1087
|
+
};
|
|
1088
|
+
}
|
|
1089
|
+
const start = async () => {
|
|
1090
|
+
const { default: ora } = await import("ora");
|
|
1091
|
+
return ora(text).start();
|
|
1092
|
+
};
|
|
1093
|
+
return { start };
|
|
1094
|
+
}
|
|
1070
1095
|
|
|
1071
1096
|
// src/commands/auth.ts
|
|
1072
1097
|
init_errors();
|
|
@@ -2563,9 +2588,17 @@ async function runGuidedEvalCreator(promptId) {
|
|
|
2563
2588
|
}
|
|
2564
2589
|
}]);
|
|
2565
2590
|
let targetField;
|
|
2566
|
-
|
|
2591
|
+
const usedParams = new Set(criteria.map((c) => c.evaluationParameter));
|
|
2592
|
+
const availableFields = allFields.filter((f) => {
|
|
2593
|
+
const param = f.startsWith("output.") ? f.slice("output.".length) : f.startsWith("input.") ? f.slice("input.".length) : f;
|
|
2594
|
+
return !usedParams.has(param);
|
|
2595
|
+
});
|
|
2596
|
+
if (availableFields.length === 0 && allFields.length > 0) {
|
|
2597
|
+
console.log(chalk5.yellow(" All detected schema fields are already used by existing criteria."));
|
|
2598
|
+
}
|
|
2599
|
+
if (availableFields.length > 0) {
|
|
2567
2600
|
const fieldChoices = [
|
|
2568
|
-
...
|
|
2601
|
+
...availableFields.map((f) => ({ name: f, value: f })),
|
|
2569
2602
|
{ name: "(custom field name)", value: "__custom__" }
|
|
2570
2603
|
];
|
|
2571
2604
|
const { field } = await inquirer3.prompt([{
|
|
@@ -2623,6 +2656,10 @@ async function runGuidedEvalCreator(promptId) {
|
|
|
2623
2656
|
scoringRubric = rubric;
|
|
2624
2657
|
}
|
|
2625
2658
|
const evaluationParameter = targetField.startsWith("output.") ? targetField.slice("output.".length) : targetField.startsWith("input.") ? targetField.slice("input.".length) : targetField;
|
|
2659
|
+
if (criteria.some((c) => c.evaluationParameter === evaluationParameter)) {
|
|
2660
|
+
console.log(chalk5.red(` Error: "${evaluationParameter}" is already used by another criterion. Each criterion must target a unique output field.`));
|
|
2661
|
+
continue;
|
|
2662
|
+
}
|
|
2626
2663
|
criteria.push({
|
|
2627
2664
|
name: criterionName.trim(),
|
|
2628
2665
|
description: scoringRubric,
|
|
@@ -2806,7 +2843,7 @@ function updateMutationContext(updater) {
|
|
|
2806
2843
|
} catch {}
|
|
2807
2844
|
}
|
|
2808
2845
|
var PREREQUISITES_TEXT = `
|
|
2809
|
-
${chalk7.
|
|
2846
|
+
${chalk7.red("Prerequisites (required):")}
|
|
2810
2847
|
1. Evaluation criteria defined ${chalk7.dim("(via dashboard or evaluation create)")}
|
|
2811
2848
|
2. Dataset uploaded ${chalk7.dim("mutagent prompts dataset list <prompt-id>")}
|
|
2812
2849
|
${chalk7.dim("Note: LLM provider config is only required when the server uses external providers (USE_EXT_PROVIDERS=true)")}`;
|
|
@@ -2990,7 +3027,7 @@ Examples:
|
|
|
2990
3027
|
Subcommands:
|
|
2991
3028
|
list, get, create, update, delete
|
|
2992
3029
|
dataset list|add|remove
|
|
2993
|
-
evaluation list|create|results
|
|
3030
|
+
evaluation list|create|delete|results
|
|
2994
3031
|
optimize start|status|results
|
|
2995
3032
|
`);
|
|
2996
3033
|
prompts.command("list").description("List all prompts").option("-l, --limit <n>", "Limit results", "50").addHelpText("after", `
|
|
@@ -3068,20 +3105,24 @@ ${chalk7.dim("Tip: Combine --with-datasets and --with-evals to fetch all nested
|
|
|
3068
3105
|
});
|
|
3069
3106
|
prompts.command("create").description("Create a new prompt").option("-d, --data <json>", "Prompt as JSON string (recommended — curl-style inline)").option("-f, --file <path>", "Create from JSON file").option("--raw-file <path>", "Create from plain text file (used as rawPrompt)").option("-n, --name <name>", "Prompt name").option("-c, --content <content>", "Prompt content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).option("--output-schema <json>", "Output schema as JSON string (required for optimization)").addHelpText("after", `
|
|
3070
3107
|
Examples:
|
|
3071
|
-
${chalk7.dim("$")} mutagent prompts create
|
|
3072
|
-
${chalk7.dim("$")} mutagent prompts create --name "
|
|
3073
|
-
${chalk7.dim("$")} mutagent prompts create
|
|
3074
|
-
${chalk7.dim("$")} mutagent prompts create --
|
|
3108
|
+
${chalk7.dim("$")} mutagent prompts create --name "my-prompt" --system "You are helpful" --human "{{input}}" --output-schema '{"type":"object","properties":{"result":{"type":"string","description":"The result"}}}'
|
|
3109
|
+
${chalk7.dim("$")} mutagent prompts create --name "raw-prompt" --raw "Summarize: {{text}}" --output-schema '{"type":"object","properties":{"summary":{"type":"string","description":"Summary"}}}'
|
|
3110
|
+
${chalk7.dim("$")} mutagent prompts create -d '{"name":"summarizer","systemPrompt":"Summarize","humanPrompt":"{{text}}","outputSchema":{"type":"object","properties":{"summary":{"type":"string","description":"Summary"}}}}'
|
|
3111
|
+
${chalk7.dim("$")} mutagent prompts create --file prompt.json ${chalk7.dim("# full prompt object as JSON file")}
|
|
3075
3112
|
|
|
3076
3113
|
Prompt Input Methods (pick one, priority order):
|
|
3077
|
-
|
|
3078
|
-
--file Load from JSON file (full prompt object)
|
|
3079
|
-
--raw-file Load plain text file as raw prompt
|
|
3080
|
-
--system/--human Structured system + user message pair
|
|
3114
|
+
--system/--human Structured system + user message pair ${chalk7.green("(recommended)")}
|
|
3081
3115
|
--raw Single raw prompt text with {{variables}}
|
|
3116
|
+
-d, --data Inline JSON object (CI/scripts/agents)
|
|
3082
3117
|
--messages Full messages array as JSON
|
|
3118
|
+
--raw-file Load plain text file as raw prompt
|
|
3119
|
+
--file Load from JSON file (full prompt object)
|
|
3120
|
+
|
|
3121
|
+
Expected JSON (--data):
|
|
3122
|
+
${chalk7.dim(`'{"name":"my-prompt","systemPrompt":"You are...","humanPrompt":"{{input}}","outputSchema":{"type":"object","properties":{"result":{"type":"string","description":"The result"}}},"inputSchema":{"type":"object","properties":{"input":{"type":"string","description":"User input"}}}}'`)}
|
|
3083
3123
|
|
|
3084
|
-
${chalk7.
|
|
3124
|
+
${chalk7.yellow("Note: Prefer --system/--human or --data over --file to avoid stale files living in your repo.")}
|
|
3125
|
+
${chalk7.red("outputSchema is required.")} ${chalk7.dim("--data and --file are mutually exclusive.")}
|
|
3085
3126
|
`).action(async (options) => {
|
|
3086
3127
|
const isJson = getJsonFlag(prompts);
|
|
3087
3128
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3229,15 +3270,15 @@ Or use interactive mode (TTY) to define variables step by step.`);
|
|
|
3229
3270
|
});
|
|
3230
3271
|
prompts.command("update").description("Update a prompt").argument("<id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Update fields as JSON string (recommended — curl-style inline)").option("-f, --file <path>", "Update from JSON file").option("--raw-file <path>", "Update from plain text file (used as rawPrompt)").option("-n, --name <name>", "New name").option("-c, --content <content>", "New content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).option("--input-schema <json>", "Input schema as JSON string").option("--input-schema-file <path>", "Input schema from JSON file").option("--output-schema <json>", "Output schema as JSON string").option("--output-schema-file <path>", "Output schema from JSON file").addHelpText("after", `
|
|
3231
3272
|
Examples:
|
|
3232
|
-
${chalk7.dim("$")} mutagent prompts update <id>
|
|
3273
|
+
${chalk7.dim("$")} mutagent prompts update <id> --system "Updated system prompt" --human "{{input}}"
|
|
3233
3274
|
${chalk7.dim("$")} mutagent prompts update <id> --name "new-name"
|
|
3234
|
-
${chalk7.dim("$")} mutagent prompts update <id> --
|
|
3235
|
-
${chalk7.dim("$")} mutagent prompts update <id>
|
|
3236
|
-
${chalk7.dim("$")} mutagent prompts update <id> --file updated-prompt.json
|
|
3275
|
+
${chalk7.dim("$")} mutagent prompts update <id> --raw "Summarize: {{text}}"
|
|
3276
|
+
${chalk7.dim("$")} mutagent prompts update <id> -d '{"name":"new-name","systemPrompt":"Updated prompt"}'
|
|
3237
3277
|
${chalk7.dim("$")} mutagent prompts update <id> --input-schema '{"type":"object","properties":{"text":{"type":"string","description":"Input text"}}}'
|
|
3238
|
-
${chalk7.dim("$")} mutagent prompts update <id> --
|
|
3278
|
+
${chalk7.dim("$")} mutagent prompts update <id> --file updated-prompt.json ${chalk7.dim("# full prompt object")}
|
|
3239
3279
|
|
|
3240
|
-
${chalk7.
|
|
3280
|
+
${chalk7.yellow("Note: Prefer --system/--human or --data over --file to avoid stale files living in your repo.")}
|
|
3281
|
+
${chalk7.dim("--data and --file are mutually exclusive. CLI flags (--name) override --data fields.")}
|
|
3241
3282
|
`).action(async (id, options) => {
|
|
3242
3283
|
const isJson = getJsonFlag(prompts);
|
|
3243
3284
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3398,12 +3439,15 @@ ${chalk7.dim("Tip: Use --force to skip confirmation (required for non-interactiv
|
|
|
3398
3439
|
handleError(error, isJson);
|
|
3399
3440
|
}
|
|
3400
3441
|
});
|
|
3401
|
-
const dataset =
|
|
3442
|
+
const dataset = new Command3("dataset").description("Manage datasets for prompts").addHelpText("after", `
|
|
3402
3443
|
Examples:
|
|
3403
3444
|
${chalk7.dim("$")} mutagent prompts dataset list <prompt-id>
|
|
3404
3445
|
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.jsonl
|
|
3405
3446
|
${chalk7.dim("$")} mutagent prompts dataset remove <prompt-id> <dataset-id>
|
|
3406
|
-
`)
|
|
3447
|
+
`).action(() => {
|
|
3448
|
+
dataset.help();
|
|
3449
|
+
});
|
|
3450
|
+
prompts.addCommand(dataset);
|
|
3407
3451
|
dataset.command("list").description("List datasets for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").addHelpText("after", `
|
|
3408
3452
|
Examples:
|
|
3409
3453
|
${chalk7.dim("$")} mutagent prompts dataset list <prompt-id>
|
|
@@ -3433,10 +3477,9 @@ Examples:
|
|
|
3433
3477
|
});
|
|
3434
3478
|
dataset.command("add").description("Add dataset to a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").option("-f, --file <path>", "Dataset file (JSON array, JSONL, or CSV)").option("-d, --data <json>", "Inline JSON array of dataset items").option("-n, --name <name>", "Dataset name (default: timestamped)").addHelpText("after", `
|
|
3435
3479
|
Examples:
|
|
3436
|
-
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.json
|
|
3437
|
-
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.jsonl --name "My Dataset"
|
|
3438
|
-
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.csv
|
|
3439
3480
|
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> -d '[{"input":{"text":"hello"},"expectedOutput":{"result":"world"}}]'
|
|
3481
|
+
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> -d '[{"input":{"text":"hello"},"expectedOutput":{"result":"world"}}]' --name "My Dataset"
|
|
3482
|
+
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.jsonl ${chalk7.dim("# also supports .json, .csv")}
|
|
3440
3483
|
|
|
3441
3484
|
Supported file formats:
|
|
3442
3485
|
${chalk7.dim(".json")} JSON array of objects: [{"input": {...}, "expectedOutput": {...}}, ...]
|
|
@@ -3447,7 +3490,12 @@ Inline data format (-d):
|
|
|
3447
3490
|
JSON array of objects, e.g.:
|
|
3448
3491
|
${chalk7.dim('[{"input": {"text": "hello"}, "expectedOutput": {"result": "world"}}]')}
|
|
3449
3492
|
|
|
3450
|
-
|
|
3493
|
+
Expected item format:
|
|
3494
|
+
${chalk7.dim('{"input": {"<field>": "<value>"}, "expectedOutput": {"<field>": "<value>"}}')}
|
|
3495
|
+
|
|
3496
|
+
${chalk7.red("Required: --data or --file must be provided.")}
|
|
3497
|
+
${chalk7.yellow("Note: Prefer -d/--data for inline JSON over --file to avoid stale files living in your repo.")}
|
|
3498
|
+
${chalk7.dim("--file and -d are mutually exclusive.")}
|
|
3451
3499
|
`).action(async (promptId, options) => {
|
|
3452
3500
|
const isJson = getJsonFlag(prompts);
|
|
3453
3501
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3543,12 +3591,16 @@ Examples:
|
|
|
3543
3591
|
handleError(error, isJson);
|
|
3544
3592
|
}
|
|
3545
3593
|
});
|
|
3546
|
-
const evaluation =
|
|
3594
|
+
const evaluation = new Command3("evaluation").description("Manage evaluations for prompts").addHelpText("after", `
|
|
3547
3595
|
Examples:
|
|
3548
3596
|
${chalk7.dim("$")} mutagent prompts evaluation list <prompt-id>
|
|
3549
3597
|
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "My Eval"
|
|
3598
|
+
${chalk7.dim("$")} mutagent prompts evaluation delete <evaluation-id>
|
|
3550
3599
|
${chalk7.dim("$")} mutagent prompts evaluation results <run-id>
|
|
3551
|
-
`)
|
|
3600
|
+
`).action(() => {
|
|
3601
|
+
evaluation.help();
|
|
3602
|
+
});
|
|
3603
|
+
prompts.addCommand(evaluation);
|
|
3552
3604
|
evaluation.command("list").description("List evaluations for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").addHelpText("after", `
|
|
3553
3605
|
Examples:
|
|
3554
3606
|
${chalk7.dim("$")} mutagent prompts evaluation list <prompt-id>
|
|
@@ -3578,17 +3630,19 @@ Examples:
|
|
|
3578
3630
|
});
|
|
3579
3631
|
evaluation.command("create").description("Create an evaluation configuration for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Evaluation as JSON string (recommended — curl-style inline)").option("-n, --name <name>", "Evaluation name (required unless --guided)").option("-f, --file <path>", "Load evaluation criteria from JSON file").option("--description <text>", "Evaluation description").option("--dataset <id>", "Dataset ID to associate (from: mutagent prompts dataset list)").option("--guided", "Interactive guided mode — build criteria step by step").addHelpText("after", `
|
|
3580
3632
|
Examples:
|
|
3581
|
-
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --guided
|
|
3582
|
-
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --
|
|
3583
|
-
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Accuracy Check"
|
|
3633
|
+
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --guided ${chalk7.dim("# recommended: interactive walkthrough")}
|
|
3634
|
+
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Accuracy" -d '{"evalConfig":{"criteria":[{"name":"Accuracy","description":"Score 1.0 if output matches expected","evaluationParameter":"result"}]}}'
|
|
3584
3635
|
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Full Eval" --file criteria.json --dataset <dataset-id>
|
|
3585
|
-
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "QA" --description "Quality assurance eval" --json
|
|
3586
3636
|
|
|
3587
|
-
|
|
3588
|
-
${chalk7.dim('{
|
|
3637
|
+
Expected JSON (--data):
|
|
3638
|
+
${chalk7.dim('{"evalConfig":{"criteria":[{"name":"<name>","description":"<scoring rubric>","evaluationParameter":"<output field to score>"}]}}')}
|
|
3639
|
+
|
|
3640
|
+
${chalk7.red("Each criterion MUST have: name, description, evaluationParameter")}
|
|
3641
|
+
evaluationParameter maps to output schema fields (e.g., "result", "classification")
|
|
3589
3642
|
|
|
3590
|
-
${chalk7.
|
|
3591
|
-
${chalk7.
|
|
3643
|
+
${chalk7.yellow("Note: Prefer --guided or --data over --file to avoid stale files living in your repo.")}
|
|
3644
|
+
${chalk7.red("Required: --name (unless --guided). Criteria must include evaluationParameter.")}
|
|
3645
|
+
${chalk7.dim("--data and --file are mutually exclusive. CLI flags (--name, --description) override --data fields.")}
|
|
3592
3646
|
${chalk7.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent prompts dataset list <prompt-id>")}
|
|
3593
3647
|
`).action(async (promptId, options) => {
|
|
3594
3648
|
const isJson = getJsonFlag(prompts);
|
|
@@ -3675,8 +3729,24 @@ ${chalk7.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent
|
|
|
3675
3729
|
evalData.description = options.description;
|
|
3676
3730
|
const criteria = evalData.evalConfig?.criteria;
|
|
3677
3731
|
if (!criteria || !Array.isArray(criteria) || criteria.length === 0) {
|
|
3678
|
-
|
|
3679
|
-
|
|
3732
|
+
let fieldsHint = "";
|
|
3733
|
+
try {
|
|
3734
|
+
const client2 = getSDKClient();
|
|
3735
|
+
const prompt = await client2.getPrompt(promptId);
|
|
3736
|
+
if (prompt.outputSchema && typeof prompt.outputSchema === "object") {
|
|
3737
|
+
const props = prompt.outputSchema.properties;
|
|
3738
|
+
if (props && typeof props === "object") {
|
|
3739
|
+
const fields = Object.keys(props);
|
|
3740
|
+
if (fields.length > 0) {
|
|
3741
|
+
fieldsHint = `
|
|
3742
|
+
Detected output fields from prompt schema: ${fields.join(", ")}
|
|
3743
|
+
`;
|
|
3744
|
+
}
|
|
3745
|
+
}
|
|
3746
|
+
}
|
|
3747
|
+
} catch {}
|
|
3748
|
+
throw new MutagentError("VALIDATION_ERROR", "Evaluation criteria are required. Provide criteria via --data, --file, or use --guided mode.", `Each criterion needs: name, description, evaluationParameter (the output field to score).
|
|
3749
|
+
` + fieldsHint + `
|
|
3680
3750
|
Example JSON (--data flag):
|
|
3681
3751
|
--data '{"evalConfig":{"criteria":[{"name":"Accuracy","description":"Score 1.0 if output matches expected, 0.0 otherwise","evaluationParameter":"classification"}]}}'
|
|
3682
3752
|
|
|
@@ -3699,6 +3769,37 @@ Or use --guided for interactive creation: mutagent prompts evaluation create <id
|
|
|
3699
3769
|
delete c.targetField;
|
|
3700
3770
|
}
|
|
3701
3771
|
}
|
|
3772
|
+
const seenParams = new Set;
|
|
3773
|
+
const duplicateParams = [];
|
|
3774
|
+
for (const c of criteria) {
|
|
3775
|
+
const param = c.evaluationParameter;
|
|
3776
|
+
if (param) {
|
|
3777
|
+
if (seenParams.has(param)) {
|
|
3778
|
+
duplicateParams.push(param);
|
|
3779
|
+
}
|
|
3780
|
+
seenParams.add(param);
|
|
3781
|
+
}
|
|
3782
|
+
}
|
|
3783
|
+
if (duplicateParams.length > 0) {
|
|
3784
|
+
let availableFields = [];
|
|
3785
|
+
try {
|
|
3786
|
+
const client2 = getSDKClient();
|
|
3787
|
+
const prompt = await client2.getPrompt(promptId);
|
|
3788
|
+
if (prompt.outputSchema && typeof prompt.outputSchema === "object") {
|
|
3789
|
+
const props = prompt.outputSchema.properties;
|
|
3790
|
+
if (props && typeof props === "object") {
|
|
3791
|
+
availableFields = Object.keys(props);
|
|
3792
|
+
}
|
|
3793
|
+
}
|
|
3794
|
+
} catch {}
|
|
3795
|
+
const uniqueDupes = [...new Set(duplicateParams)];
|
|
3796
|
+
const fieldsHint = availableFields.length > 0 ? `
|
|
3797
|
+
Available output fields: ${availableFields.join(", ")}` : "";
|
|
3798
|
+
throw new MutagentError("VALIDATION_ERROR", `Duplicate evaluationParameter: "${uniqueDupes.join('", "')}". Each criterion must target a unique output field.`, "Each criterion scores a different output field. Fix by changing the evaluationParameter to a unique field." + fieldsHint + `
|
|
3799
|
+
|
|
3800
|
+
Example:
|
|
3801
|
+
--data '{"evalConfig":{"criteria":[{"name":"Accuracy","description":"...","evaluationParameter":"classification"},{"name":"Confidence","description":"...","evaluationParameter":"confidence"}]}}'`);
|
|
3802
|
+
}
|
|
3702
3803
|
if (options.dataset) {
|
|
3703
3804
|
evalData.datasetId = parseInt(options.dataset, 10);
|
|
3704
3805
|
if (isNaN(evalData.datasetId)) {
|
|
@@ -3746,15 +3847,37 @@ Examples:
|
|
|
3746
3847
|
handleError(error, isJson);
|
|
3747
3848
|
}
|
|
3748
3849
|
});
|
|
3749
|
-
|
|
3850
|
+
evaluation.command("delete").description("Delete an evaluation").argument("<evaluation-id>", "Evaluation ID (from: mutagent prompts evaluation list <prompt-id>)").addHelpText("after", `
|
|
3851
|
+
Examples:
|
|
3852
|
+
${chalk7.dim("$")} mutagent prompts evaluation delete <evaluation-id>
|
|
3853
|
+
${chalk7.dim("$")} mutagent prompts evaluation delete <evaluation-id> --json
|
|
3854
|
+
`).action(async (evaluationId) => {
|
|
3855
|
+
const isJson = getJsonFlag(prompts);
|
|
3856
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
3857
|
+
try {
|
|
3858
|
+
const client = getSDKClient();
|
|
3859
|
+
await client.deleteEvaluation(evaluationId);
|
|
3860
|
+
if (isJson) {
|
|
3861
|
+
output.output({ success: true, message: "Evaluation deleted successfully", evaluationId });
|
|
3862
|
+
} else {
|
|
3863
|
+
output.success(`Evaluation ${evaluationId} deleted successfully`);
|
|
3864
|
+
}
|
|
3865
|
+
} catch (error) {
|
|
3866
|
+
handleError(error, isJson);
|
|
3867
|
+
}
|
|
3868
|
+
});
|
|
3869
|
+
const optimize = new Command3("optimize").description("Manage prompt optimization jobs").addHelpText("after", `
|
|
3750
3870
|
Examples:
|
|
3751
3871
|
${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id>
|
|
3752
3872
|
${chalk7.dim("$")} mutagent prompts optimize status <job-id>
|
|
3753
3873
|
${chalk7.dim("$")} mutagent prompts optimize results <job-id>
|
|
3754
3874
|
|
|
3755
3875
|
Workflow: start -> status (poll) -> results
|
|
3756
|
-
`)
|
|
3757
|
-
|
|
3876
|
+
`).action(() => {
|
|
3877
|
+
optimize.help();
|
|
3878
|
+
});
|
|
3879
|
+
prompts.addCommand(optimize);
|
|
3880
|
+
optimize.command("start").description("Start prompt optimization").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").requiredOption("-d, --dataset <id>", "Dataset ID for optimization (from: mutagent prompts dataset list <prompt-id>)").option("--max-iterations <n>", "Max optimization iterations (default: 1)").option("--target-score <n>", "Target accuracy 0-1 (default: 0.8)").option("--patience <n>", "Iterations without improvement before stopping").option("--model <model-id>", 'Target LLM model (e.g., "claude-sonnet-4-5-20250929")').option("--eval-model <model-id>", "Evaluation model (defaults to target model)").addHelpText("after", `
|
|
3758
3881
|
Examples:
|
|
3759
3882
|
${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id>
|
|
3760
3883
|
${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --max-iterations 5
|
|
@@ -3763,6 +3886,8 @@ Examples:
|
|
|
3763
3886
|
${PREREQUISITES_TEXT}
|
|
3764
3887
|
|
|
3765
3888
|
${chalk7.dim("Monitor progress with: mutagent prompts optimize status <job-id>")}
|
|
3889
|
+
|
|
3890
|
+
${chalk7.dim("AI Agent Note: After running commands, present CLI output to the user as a status report. Use --json for structured parsing.")}
|
|
3766
3891
|
`).action(async (promptId, options) => {
|
|
3767
3892
|
const isJson = getJsonFlag(prompts);
|
|
3768
3893
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4961,19 +5086,47 @@ System Prompt:`));
|
|
|
4961
5086
|
handleError(error, isJson);
|
|
4962
5087
|
}
|
|
4963
5088
|
});
|
|
4964
|
-
agents.command("create").description("Create a new agent").option("-f, --file <path>", "Create from JSON file").option("-n, --name <name>", "Agent name").option("-s, --slug <slug>", "Agent slug (URL-friendly identifier)").option("-p, --system-prompt <prompt>", "System prompt").option("-m, --model <model>", "Model (claude-sonnet-4-5, claude-opus-4-5, claude-haiku-4-5)").option("
|
|
5089
|
+
agents.command("create").description("Create a new agent").option("-d, --data <json>", "Agent as JSON string (recommended for CI/scripts/agents)").option("-f, --file <path>", "Create from JSON file").option("-n, --name <name>", "Agent name").option("-s, --slug <slug>", "Agent slug (URL-friendly identifier)").option("-p, --system-prompt <prompt>", "System prompt").option("-m, --model <model>", "Model (claude-sonnet-4-5, claude-opus-4-5, claude-haiku-4-5)").option("--description <desc>", "Agent description").addHelpText("after", `
|
|
4965
5090
|
Examples:
|
|
4966
|
-
${chalk10.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "
|
|
4967
|
-
${chalk10.dim("$")} mutagent agents create
|
|
4968
|
-
${chalk10.dim("$")} mutagent agents create --
|
|
5091
|
+
${chalk10.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "You are a code reviewer..."
|
|
5092
|
+
${chalk10.dim("$")} mutagent agents create -d '{"name":"Code Reviewer","slug":"code-reviewer","systemPrompt":"You are a code reviewer..."}'
|
|
5093
|
+
${chalk10.dim("$")} mutagent agents create --file agent.json ${chalk10.dim("# full agent object as JSON file")}
|
|
5094
|
+
|
|
5095
|
+
Expected JSON (--data):
|
|
5096
|
+
${chalk10.dim('{"name":"<name>","slug":"<slug>","systemPrompt":"<system prompt>","model":"<model-id>","description":"<description>"}')}
|
|
5097
|
+
|
|
5098
|
+
Input Methods (pick one, priority order):
|
|
5099
|
+
--name/--slug/... Individual flags ${chalk10.green("(recommended)")}
|
|
5100
|
+
-d, --data Inline JSON object (CI/scripts/agents)
|
|
5101
|
+
--file Load from JSON file
|
|
4969
5102
|
|
|
4970
|
-
${chalk10.
|
|
5103
|
+
${chalk10.yellow("Note: Prefer individual flags or --data over --file to avoid stale files living in your repo.")}
|
|
5104
|
+
${chalk10.red("Required: name, slug, systemPrompt.")} ${chalk10.dim("--data and --file are mutually exclusive. CLI flags override --data fields.")}
|
|
4971
5105
|
`).action(async (options) => {
|
|
4972
5106
|
const isJson = getJsonFlag(agents);
|
|
4973
5107
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
4974
5108
|
try {
|
|
4975
5109
|
let data;
|
|
4976
|
-
if (options.file) {
|
|
5110
|
+
if (options.data && options.file) {
|
|
5111
|
+
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --data and --file together", "Use --data for inline JSON or --file for file-based input, not both");
|
|
5112
|
+
}
|
|
5113
|
+
if (options.data) {
|
|
5114
|
+
try {
|
|
5115
|
+
data = JSON.parse(options.data);
|
|
5116
|
+
} catch {
|
|
5117
|
+
throw new MutagentError("INVALID_JSON", "Invalid JSON in --data flag", `Provide a valid JSON object, e.g., '{"name":"my-agent","slug":"my-agent","systemPrompt":"You are..."}'`);
|
|
5118
|
+
}
|
|
5119
|
+
if (options.name)
|
|
5120
|
+
data.name = options.name;
|
|
5121
|
+
if (options.slug)
|
|
5122
|
+
data.slug = options.slug;
|
|
5123
|
+
if (options.systemPrompt)
|
|
5124
|
+
data.systemPrompt = options.systemPrompt;
|
|
5125
|
+
if (options.model)
|
|
5126
|
+
data.model = options.model;
|
|
5127
|
+
if (options.description)
|
|
5128
|
+
data.description = options.description;
|
|
5129
|
+
} else if (options.file) {
|
|
4977
5130
|
const content = readFileSync9(options.file, "utf-8");
|
|
4978
5131
|
data = JSON.parse(content);
|
|
4979
5132
|
} else if (options.name && options.slug && options.systemPrompt) {
|
|
@@ -4987,7 +5140,7 @@ ${chalk10.dim("Required: --name, --slug, and --system-prompt (or --file).")}
|
|
|
4987
5140
|
if (options.description)
|
|
4988
5141
|
data.description = options.description;
|
|
4989
5142
|
} else {
|
|
4990
|
-
throw new MutagentError("MISSING_ARGUMENTS", "Either --file or (--name, --slug, and --system-prompt) are required", "Use --file to load from JSON or provide --name, --slug, and --system-prompt");
|
|
5143
|
+
throw new MutagentError("MISSING_ARGUMENTS", "Either --data, --file, or (--name, --slug, and --system-prompt) are required", "Use --data for inline JSON, --file to load from JSON, or provide --name, --slug, and --system-prompt");
|
|
4991
5144
|
}
|
|
4992
5145
|
const client = getSDKClient();
|
|
4993
5146
|
const agent = await client.createAgent(data);
|
|
@@ -4997,17 +5150,45 @@ ${chalk10.dim("Required: --name, --slug, and --system-prompt (or --file).")}
|
|
|
4997
5150
|
handleError(error, isJson);
|
|
4998
5151
|
}
|
|
4999
5152
|
});
|
|
5000
|
-
agents.command("update").description("Update an agent").argument("<id>", "Agent ID").option("-f, --file <path>", "Update from JSON file").option("-n, --name <name>", "New name").option("-p, --system-prompt <prompt>", "New system prompt").option("-m, --model <model>", "New model").option("
|
|
5153
|
+
agents.command("update").description("Update an agent").argument("<id>", "Agent ID").option("-d, --data <json>", "Agent updates as JSON string (CI/scripts/agents)").option("-f, --file <path>", "Update from JSON file").option("-n, --name <name>", "New name").option("-p, --system-prompt <prompt>", "New system prompt").option("-m, --model <model>", "New model").option("--description <desc>", "New description").option("-s, --status <status>", "New status (active, paused, archived)").addHelpText("after", `
|
|
5001
5154
|
Examples:
|
|
5002
5155
|
${chalk10.dim("$")} mutagent agents update <id> --name "New Name"
|
|
5003
|
-
${chalk10.dim("$")} mutagent agents update <id> --status
|
|
5004
|
-
${chalk10.dim("$")} mutagent agents update <id>
|
|
5156
|
+
${chalk10.dim("$")} mutagent agents update <id> --system-prompt "Updated prompt" --status active
|
|
5157
|
+
${chalk10.dim("$")} mutagent agents update <id> -d '{"name":"New Name","systemPrompt":"Updated prompt"}'
|
|
5158
|
+
${chalk10.dim("$")} mutagent agents update <id> --file updated-agent.json ${chalk10.dim("# full agent object")}
|
|
5159
|
+
|
|
5160
|
+
Input Methods (pick one, priority order):
|
|
5161
|
+
--name/--system-prompt/... Individual flags ${chalk10.green("(recommended)")}
|
|
5162
|
+
-d, --data Inline JSON object (CI/scripts/agents)
|
|
5163
|
+
--file Load from JSON file
|
|
5164
|
+
|
|
5165
|
+
${chalk10.yellow("Note: Prefer individual flags or --data over --file to avoid stale files living in your repo.")}
|
|
5166
|
+
${chalk10.dim("--data and --file are mutually exclusive. CLI flags override --data fields.")}
|
|
5005
5167
|
`).action(async (id, options) => {
|
|
5006
5168
|
const isJson = getJsonFlag(agents);
|
|
5007
5169
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5008
5170
|
try {
|
|
5009
5171
|
let data = {};
|
|
5010
|
-
if (options.file) {
|
|
5172
|
+
if (options.data && options.file) {
|
|
5173
|
+
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --data and --file together", "Use --data for inline JSON or --file for file-based input, not both");
|
|
5174
|
+
}
|
|
5175
|
+
if (options.data) {
|
|
5176
|
+
try {
|
|
5177
|
+
data = JSON.parse(options.data);
|
|
5178
|
+
} catch {
|
|
5179
|
+
throw new MutagentError("INVALID_JSON", "Invalid JSON in --data flag", `Provide a valid JSON object, e.g., '{"name":"new-name","systemPrompt":"Updated prompt"}'`);
|
|
5180
|
+
}
|
|
5181
|
+
if (options.name)
|
|
5182
|
+
data.name = options.name;
|
|
5183
|
+
if (options.systemPrompt)
|
|
5184
|
+
data.systemPrompt = options.systemPrompt;
|
|
5185
|
+
if (options.model)
|
|
5186
|
+
data.model = options.model;
|
|
5187
|
+
if (options.description)
|
|
5188
|
+
data.description = options.description;
|
|
5189
|
+
if (options.status)
|
|
5190
|
+
data.status = options.status;
|
|
5191
|
+
} else if (options.file) {
|
|
5011
5192
|
const content = readFileSync9(options.file, "utf-8");
|
|
5012
5193
|
data = JSON.parse(content);
|
|
5013
5194
|
} else {
|
|
@@ -5023,7 +5204,7 @@ Examples:
|
|
|
5023
5204
|
data.status = options.status;
|
|
5024
5205
|
}
|
|
5025
5206
|
if (Object.keys(data).length === 0) {
|
|
5026
|
-
throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --file, --name, --system-prompt, --model, --description, or --status");
|
|
5207
|
+
throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --data, --file, --name, --system-prompt, --model, --description, or --status");
|
|
5027
5208
|
}
|
|
5028
5209
|
const client = getSDKClient();
|
|
5029
5210
|
const agent = await client.updateAgent(id, data);
|
|
@@ -5064,12 +5245,15 @@ ${chalk10.dim("Tip: Use --force to skip confirmation (required for non-interacti
|
|
|
5064
5245
|
handleError(error, isJson);
|
|
5065
5246
|
}
|
|
5066
5247
|
});
|
|
5067
|
-
const conversations =
|
|
5248
|
+
const conversations = new Command6("conversations").description("Manage conversations for agents").addHelpText("after", `
|
|
5068
5249
|
Examples:
|
|
5069
5250
|
${chalk10.dim("$")} mutagent agents conversations list <agent-id>
|
|
5070
5251
|
${chalk10.dim("$")} mutagent agents conversations create <agent-id>
|
|
5071
5252
|
${chalk10.dim("$")} mutagent agents conversations messages <agent-id> <conversation-id>
|
|
5072
|
-
`)
|
|
5253
|
+
`).action(() => {
|
|
5254
|
+
conversations.help();
|
|
5255
|
+
});
|
|
5256
|
+
agents.addCommand(conversations);
|
|
5073
5257
|
conversations.command("list").description("List conversations for an agent").argument("<agent-id>", "Agent ID").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").addHelpText("after", `
|
|
5074
5258
|
Examples:
|
|
5075
5259
|
${chalk10.dim("$")} mutagent agents conversations list <agent-id>
|
|
@@ -5218,6 +5402,8 @@ Examples:
|
|
|
5218
5402
|
init_config();
|
|
5219
5403
|
import { Command as Command7 } from "commander";
|
|
5220
5404
|
import chalk11 from "chalk";
|
|
5405
|
+
init_errors();
|
|
5406
|
+
var VALID_CONFIG_KEYS = ["apiKey", "endpoint", "format", "timeout", "defaultWorkspace", "defaultOrganization"];
|
|
5221
5407
|
function createConfigCommand() {
|
|
5222
5408
|
const config = new Command7("config").description("Manage CLI configuration").addHelpText("after", `
|
|
5223
5409
|
Examples:
|
|
@@ -5250,21 +5436,27 @@ ${chalk11.dim("Keys: apiKey, endpoint, format, timeout, defaultWorkspace, defaul
|
|
|
5250
5436
|
`).action((key) => {
|
|
5251
5437
|
const isJson = getJsonFlag(config);
|
|
5252
5438
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5253
|
-
|
|
5254
|
-
|
|
5255
|
-
|
|
5256
|
-
|
|
5257
|
-
|
|
5258
|
-
|
|
5259
|
-
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
|
|
5439
|
+
try {
|
|
5440
|
+
const cfg = loadConfig();
|
|
5441
|
+
if (!(key in cfg)) {
|
|
5442
|
+
throw new MutagentError("INVALID_ARGUMENTS", `Unknown config key: "${key}"`, `Valid keys: ${VALID_CONFIG_KEYS.join(", ")}
|
|
5443
|
+
` + "Example: mutagent config get endpoint");
|
|
5444
|
+
}
|
|
5445
|
+
const value = cfg[key];
|
|
5446
|
+
const stringValue = typeof value === "object" ? JSON.stringify(value) : String(value ?? "");
|
|
5447
|
+
const displayValue = key === "apiKey" && value ? `${stringValue.slice(0, 8)}...` : stringValue;
|
|
5448
|
+
if (isJson) {
|
|
5449
|
+
output.output({ [key]: displayValue });
|
|
5450
|
+
} else {
|
|
5451
|
+
output.info(displayValue);
|
|
5452
|
+
}
|
|
5453
|
+
} catch (error) {
|
|
5454
|
+
handleError(error, isJson);
|
|
5265
5455
|
}
|
|
5266
5456
|
});
|
|
5267
|
-
const set = new Command7("set").description("Set configuration value")
|
|
5457
|
+
const set = new Command7("set").description("Set configuration value").action(() => {
|
|
5458
|
+
set.help();
|
|
5459
|
+
});
|
|
5268
5460
|
set.command("workspace").description("Set default workspace ID").argument("<id>", "Workspace ID to set as default").addHelpText("after", `
|
|
5269
5461
|
Examples:
|
|
5270
5462
|
${chalk11.dim("$")} mutagent config set workspace <workspace-id>
|
|
@@ -5356,12 +5548,13 @@ Examples:
|
|
|
5356
5548
|
${chalk12.dim("$")} mutagent playground run <prompt-id> --system "You are helpful" --human "Hello"
|
|
5357
5549
|
${chalk12.dim("$")} mutagent playground run <prompt-id> --input '{}' --model gpt-4-turbo --json
|
|
5358
5550
|
|
|
5359
|
-
Input Methods (pick one):
|
|
5551
|
+
Input Methods (pick one, priority order):
|
|
5552
|
+
--system/--human Quick system + user message ${chalk12.green("(recommended)")}
|
|
5360
5553
|
--input '{"key":"value"}' Inline JSON variables
|
|
5361
|
-
--file input.json Load from JSON file
|
|
5362
|
-
--system/--human Quick system + user message
|
|
5363
5554
|
--messages '[...]' Full messages array
|
|
5555
|
+
--file input.json Load from JSON file
|
|
5364
5556
|
|
|
5557
|
+
${chalk12.yellow("Note: Prefer --system/--human or --input over --file to avoid stale files living in your repo.")}
|
|
5365
5558
|
${chalk12.dim(`Hint: Test before evaluating: mutagent playground run <id> --input '{"key":"value"}'`)}
|
|
5366
5559
|
`).action(async (promptId, options) => {
|
|
5367
5560
|
const isJson = getJsonFlag(playground);
|
|
@@ -5693,6 +5886,10 @@ Subcommands:
|
|
|
5693
5886
|
list, get, test
|
|
5694
5887
|
|
|
5695
5888
|
Note: Provider management (create, update, delete) is available in the Admin Panel only.
|
|
5889
|
+
|
|
5890
|
+
${chalk14.yellow("Note:")} The providers module is not yet active. This is a placeholder
|
|
5891
|
+
for future external provider configuration. The server currently uses
|
|
5892
|
+
built-in provider settings.
|
|
5696
5893
|
`);
|
|
5697
5894
|
providers.command("list").description("List all providers").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").option("-t, --type <type>", "Filter by provider type").addHelpText("after", `
|
|
5698
5895
|
Examples:
|
|
@@ -5721,6 +5918,8 @@ Examples:
|
|
|
5721
5918
|
}));
|
|
5722
5919
|
output.output({ ...result, data: withLinks });
|
|
5723
5920
|
} else {
|
|
5921
|
+
console.log(chalk14.yellow("Note: The providers module is not yet active. This is a placeholder for future external provider configuration."));
|
|
5922
|
+
console.log("");
|
|
5724
5923
|
if (result.data.length === 0) {
|
|
5725
5924
|
output.info("No providers configured.");
|
|
5726
5925
|
output.info(`Configure at: ${providerSettingsLink()}`);
|
|
@@ -5754,6 +5953,8 @@ Examples:
|
|
|
5754
5953
|
if (isJson) {
|
|
5755
5954
|
output.output({ ...provider, _links: providerLinks(provider.id) });
|
|
5756
5955
|
} else {
|
|
5956
|
+
console.log(chalk14.yellow("Note: The providers module is not yet active. This is a placeholder for future external provider configuration."));
|
|
5957
|
+
console.log("");
|
|
5757
5958
|
const formatted = {
|
|
5758
5959
|
id: provider.id,
|
|
5759
5960
|
name: provider.name,
|
|
@@ -5783,6 +5984,8 @@ ${chalk14.dim("Tests connectivity and lists available models for the provider.")
|
|
|
5783
5984
|
try {
|
|
5784
5985
|
const client = getSDKClient();
|
|
5785
5986
|
if (!isJson) {
|
|
5987
|
+
console.log(chalk14.yellow("Note: The providers module is not yet active. This is a placeholder for future external provider configuration."));
|
|
5988
|
+
console.log("");
|
|
5786
5989
|
output.info(`Testing provider ${id}...`);
|
|
5787
5990
|
}
|
|
5788
5991
|
const result = await client.testProvider(id);
|
|
@@ -5818,7 +6021,7 @@ init_config();
|
|
|
5818
6021
|
import { Command as Command11 } from "commander";
|
|
5819
6022
|
import inquirer3 from "inquirer";
|
|
5820
6023
|
import chalk15 from "chalk";
|
|
5821
|
-
import { existsSync as existsSync10, readFileSync as readFileSync11, writeFileSync as writeFileSync4 } from "fs";
|
|
6024
|
+
import { existsSync as existsSync10, mkdirSync as mkdirSync3, readFileSync as readFileSync11, writeFileSync as writeFileSync4 } from "fs";
|
|
5822
6025
|
import { execSync as execSync2 } from "child_process";
|
|
5823
6026
|
import { join as join5 } from "path";
|
|
5824
6027
|
init_errors();
|
|
@@ -6091,6 +6294,32 @@ Modes:
|
|
|
6091
6294
|
output.info('Ready! Run "mutagent --help" to see available commands.');
|
|
6092
6295
|
}
|
|
6093
6296
|
}
|
|
6297
|
+
if (!isNonInteractive) {
|
|
6298
|
+
const skillPath = join5(cwd, ".claude/skills/mutagent-cli/SKILL.md");
|
|
6299
|
+
if (!existsSync10(skillPath)) {
|
|
6300
|
+
const { installSkill } = await inquirer3.prompt([{
|
|
6301
|
+
type: "confirm",
|
|
6302
|
+
name: "installSkill",
|
|
6303
|
+
message: "Install MutagenT skill for Claude Code? (Teaches AI agents how to use the CLI)",
|
|
6304
|
+
default: true
|
|
6305
|
+
}]);
|
|
6306
|
+
if (installSkill) {
|
|
6307
|
+
try {
|
|
6308
|
+
const skillDir = join5(cwd, ".claude/skills/mutagent-cli");
|
|
6309
|
+
if (!existsSync10(skillDir)) {
|
|
6310
|
+
mkdirSync3(skillDir, { recursive: true });
|
|
6311
|
+
}
|
|
6312
|
+
execSync2("node " + join5(cwd, "node_modules/.bin/mutagent") + " skills install", {
|
|
6313
|
+
cwd,
|
|
6314
|
+
stdio: "ignore"
|
|
6315
|
+
});
|
|
6316
|
+
output.success("Installed MutagenT CLI skill for Claude Code");
|
|
6317
|
+
} catch {
|
|
6318
|
+
output.info("Install skill manually: mutagent skills install");
|
|
6319
|
+
}
|
|
6320
|
+
}
|
|
6321
|
+
}
|
|
6322
|
+
}
|
|
6094
6323
|
if (isJson) {
|
|
6095
6324
|
output.output({
|
|
6096
6325
|
success: true,
|
|
@@ -6219,7 +6448,7 @@ Scanning ${scanPath}...
|
|
|
6219
6448
|
// src/commands/skills.ts
|
|
6220
6449
|
import { Command as Command13 } from "commander";
|
|
6221
6450
|
import chalk17 from "chalk";
|
|
6222
|
-
import { existsSync as existsSync11, mkdirSync as
|
|
6451
|
+
import { existsSync as existsSync11, mkdirSync as mkdirSync4, writeFileSync as writeFileSync5 } from "fs";
|
|
6223
6452
|
import { join as join6 } from "path";
|
|
6224
6453
|
var SKILL_FRONTMATTER = `---
|
|
6225
6454
|
name: mutagent-cli
|
|
@@ -6234,6 +6463,22 @@ var SKILL_BODY = `# MutagenT CLI Skill
|
|
|
6234
6463
|
|
|
6235
6464
|
## Quick Reference
|
|
6236
6465
|
Run \`mutagent --help\` for full command list and examples.
|
|
6466
|
+
Run \`mutagent <command> --help\` for on-the-fly instructions before using any command.
|
|
6467
|
+
|
|
6468
|
+
Key commands:
|
|
6469
|
+
- \`mutagent auth login\` — Authenticate
|
|
6470
|
+
- \`mutagent prompts list\` — List prompts
|
|
6471
|
+
- \`mutagent usage\` — Check plan limits and remaining optimization runs
|
|
6472
|
+
- \`mutagent explore\` — Discover prompts in codebase
|
|
6473
|
+
- \`mutagent integrate <framework>\` — Framework integration instructions
|
|
6474
|
+
|
|
6475
|
+
## Agent Usage Guidelines
|
|
6476
|
+
|
|
6477
|
+
- **ALWAYS** run \`mutagent prompts list --json\` and \`mutagent usage --json\` BEFORE creating or modifying resources
|
|
6478
|
+
- **ALWAYS** use the \`--json\` flag for machine-readable output
|
|
6479
|
+
- **ALWAYS** confirm with the user before write operations (create, update, delete, optimize)
|
|
6480
|
+
- **CHECK** \`mutagent usage\` before running \`optimize start\` to verify remaining optimization runs
|
|
6481
|
+
- Format dashboard links as markdown: \`[View on Dashboard](url)\`
|
|
6237
6482
|
|
|
6238
6483
|
## Post-Onboarding Decision Tree
|
|
6239
6484
|
|
|
@@ -6252,8 +6497,9 @@ After \`mutagent auth login\`, the user lands in one of 3 paths:
|
|
|
6252
6497
|
3. \`mutagent prompts create --data '{...}'\` — Upload with REQUIRED outputSchema
|
|
6253
6498
|
4. Guided eval creator OR \`mutagent prompts evaluation create\`
|
|
6254
6499
|
5. \`mutagent prompts dataset add\` — Upload/curate datasets (named)
|
|
6255
|
-
6. \`mutagent
|
|
6256
|
-
7.
|
|
6500
|
+
6. \`mutagent usage --json\` — Verify remaining optimization runs
|
|
6501
|
+
7. \`mutagent prompts optimize start\` — Run optimization
|
|
6502
|
+
8. Review scorecard → Apply/Reject optimized prompt
|
|
6257
6503
|
|
|
6258
6504
|
### Path C: Manual
|
|
6259
6505
|
User uses CLI commands directly. Run \`mutagent --help\`.
|
|
@@ -6327,7 +6573,7 @@ that teaches coding agents how to use the MutagenT CLI effectively.
|
|
|
6327
6573
|
const skillDir = join6(process.cwd(), SKILL_DIR);
|
|
6328
6574
|
const skillPath = join6(skillDir, SKILL_FILE);
|
|
6329
6575
|
if (!existsSync11(skillDir)) {
|
|
6330
|
-
|
|
6576
|
+
mkdirSync4(skillDir, { recursive: true });
|
|
6331
6577
|
}
|
|
6332
6578
|
const content = `${SKILL_FRONTMATTER}
|
|
6333
6579
|
|
|
@@ -6351,8 +6597,108 @@ ${SKILL_BODY}
|
|
|
6351
6597
|
return skills;
|
|
6352
6598
|
}
|
|
6353
6599
|
|
|
6600
|
+
// src/commands/usage.ts
|
|
6601
|
+
init_config();
|
|
6602
|
+
import { Command as Command14 } from "commander";
|
|
6603
|
+
import chalk18 from "chalk";
|
|
6604
|
+
init_errors();
|
|
6605
|
+
init_sdk_client();
|
|
6606
|
+
var TRIAL_OPTIMIZATION_LIMIT = 5;
|
|
6607
|
+
var BILLING_URL = "https://app.mutagent.io/settings/billing";
|
|
6608
|
+
function renderProgressBar(used, limit, width = 30) {
|
|
6609
|
+
const ratio = Math.min(used / limit, 1);
|
|
6610
|
+
const filled = Math.round(ratio * width);
|
|
6611
|
+
const empty = width - filled;
|
|
6612
|
+
const bar = "█".repeat(filled) + "░".repeat(empty);
|
|
6613
|
+
const percent = Math.round(ratio * 100);
|
|
6614
|
+
return `${bar} ${String(percent)}%`;
|
|
6615
|
+
}
|
|
6616
|
+
function createUsageCommand() {
|
|
6617
|
+
const usage = new Command14("usage").description("Show resource counts and optimization run limits").addHelpText("after", `
|
|
6618
|
+
Examples:
|
|
6619
|
+
${chalk18.dim("$")} mutagent usage
|
|
6620
|
+
${chalk18.dim("$")} mutagent usage --json
|
|
6621
|
+
`);
|
|
6622
|
+
usage.action(async () => {
|
|
6623
|
+
const isJson = getJsonFlag(usage);
|
|
6624
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
6625
|
+
try {
|
|
6626
|
+
const apiKey = getApiKey();
|
|
6627
|
+
if (!apiKey) {
|
|
6628
|
+
throw new AuthenticationError;
|
|
6629
|
+
}
|
|
6630
|
+
const spinner = await createSpinner("Fetching usage data...", isJson).start();
|
|
6631
|
+
const client = getSDKClient();
|
|
6632
|
+
const prompts = await client.listPrompts();
|
|
6633
|
+
const promptCount = prompts.length;
|
|
6634
|
+
let datasetCount = 0;
|
|
6635
|
+
let evaluationCount = 0;
|
|
6636
|
+
if (promptCount > 0) {
|
|
6637
|
+
const results = await Promise.all(prompts.map(async (prompt) => {
|
|
6638
|
+
const id = String(prompt.id);
|
|
6639
|
+
const [datasets, evaluations] = await Promise.all([
|
|
6640
|
+
client.listDatasets(id).catch(() => []),
|
|
6641
|
+
client.listEvaluations(id).catch(() => [])
|
|
6642
|
+
]);
|
|
6643
|
+
return { datasets: datasets.length, evaluations: evaluations.length };
|
|
6644
|
+
}));
|
|
6645
|
+
for (const r of results) {
|
|
6646
|
+
datasetCount += r.datasets;
|
|
6647
|
+
evaluationCount += r.evaluations;
|
|
6648
|
+
}
|
|
6649
|
+
}
|
|
6650
|
+
const optimizationLimit = TRIAL_OPTIMIZATION_LIMIT;
|
|
6651
|
+
const optimizationUsed = 0;
|
|
6652
|
+
const optimizationRemaining = optimizationLimit - optimizationUsed;
|
|
6653
|
+
if (spinner && typeof spinner.stop === "function") {
|
|
6654
|
+
spinner.stop();
|
|
6655
|
+
}
|
|
6656
|
+
if (isJson) {
|
|
6657
|
+
output.output({
|
|
6658
|
+
resources: {
|
|
6659
|
+
prompts: promptCount,
|
|
6660
|
+
datasets: datasetCount,
|
|
6661
|
+
evaluations: evaluationCount
|
|
6662
|
+
},
|
|
6663
|
+
optimizationRuns: {
|
|
6664
|
+
used: optimizationUsed,
|
|
6665
|
+
limit: optimizationLimit,
|
|
6666
|
+
remaining: optimizationRemaining,
|
|
6667
|
+
plan: "trial"
|
|
6668
|
+
},
|
|
6669
|
+
_links: {
|
|
6670
|
+
billing: BILLING_URL
|
|
6671
|
+
}
|
|
6672
|
+
});
|
|
6673
|
+
} else {
|
|
6674
|
+
console.log("");
|
|
6675
|
+
console.log(chalk18.bold("\uD83D\uDCCA MutagenT Usage"));
|
|
6676
|
+
console.log(chalk18.dim("─".repeat(45)));
|
|
6677
|
+
console.log("");
|
|
6678
|
+
console.log(chalk18.bold("Resources:"));
|
|
6679
|
+
console.log(` Prompts: ${chalk18.cyan(String(promptCount))}`);
|
|
6680
|
+
console.log(` Datasets: ${chalk18.cyan(String(datasetCount))}`);
|
|
6681
|
+
console.log(` Evaluations: ${chalk18.cyan(String(evaluationCount))}`);
|
|
6682
|
+
console.log("");
|
|
6683
|
+
console.log(chalk18.bold(`Optimization Runs (${chalk18.yellow("trial")} plan):`));
|
|
6684
|
+
console.log(` Remaining: ${chalk18.cyan(String(optimizationRemaining))} / ${String(optimizationLimit)}`);
|
|
6685
|
+
console.log(` ${renderProgressBar(optimizationUsed, optimizationLimit)}`);
|
|
6686
|
+
console.log("");
|
|
6687
|
+
console.log(chalk18.yellow(` ⚠ ${String(optimizationRemaining)} optimization runs remaining`));
|
|
6688
|
+
console.log(chalk18.dim(` ℹ Optimization run counts are approximate`));
|
|
6689
|
+
console.log(` Upgrade: ${chalk18.underline(BILLING_URL)}`);
|
|
6690
|
+
console.log("");
|
|
6691
|
+
}
|
|
6692
|
+
} catch (error) {
|
|
6693
|
+
handleError(error, isJson);
|
|
6694
|
+
}
|
|
6695
|
+
});
|
|
6696
|
+
return usage;
|
|
6697
|
+
}
|
|
6698
|
+
|
|
6354
6699
|
// src/bin/cli.ts
|
|
6355
6700
|
init_config();
|
|
6701
|
+
import { existsSync as existsSync12 } from "fs";
|
|
6356
6702
|
var cliVersion = "0.1.1";
|
|
6357
6703
|
if (process.env.CLI_VERSION) {
|
|
6358
6704
|
cliVersion = process.env.CLI_VERSION;
|
|
@@ -6364,7 +6710,7 @@ if (process.env.CLI_VERSION) {
|
|
|
6364
6710
|
cliVersion = pkg.version ?? cliVersion;
|
|
6365
6711
|
} catch {}
|
|
6366
6712
|
}
|
|
6367
|
-
var program = new
|
|
6713
|
+
var program = new Command15;
|
|
6368
6714
|
program.name("mutagent").description(`MutagenT CLI - AI-native prompt optimization platform
|
|
6369
6715
|
|
|
6370
6716
|
Documentation: https://docs.mutagent.io/cli
|
|
@@ -6373,45 +6719,47 @@ program.name("mutagent").description(`MutagenT CLI - AI-native prompt optimizati
|
|
|
6373
6719
|
showGlobalOptions: true
|
|
6374
6720
|
});
|
|
6375
6721
|
program.addHelpText("after", `
|
|
6376
|
-
${
|
|
6377
|
-
Export your API key: ${
|
|
6378
|
-
Or pass inline: ${
|
|
6379
|
-
Machine-readable output: ${
|
|
6380
|
-
Disable prompts: ${
|
|
6381
|
-
Set default workspace: ${
|
|
6382
|
-
Set default org: ${
|
|
6722
|
+
${chalk19.yellow("Non-Interactive Mode (CI/CD & Coding Agents):")}
|
|
6723
|
+
Export your API key: ${chalk19.green("export MUTAGENT_API_KEY=mt_your_key_here")}
|
|
6724
|
+
Or pass inline: ${chalk19.green("mutagent prompts list --api-key mt_your_key")}
|
|
6725
|
+
Machine-readable output: ${chalk19.green("mutagent prompts list --json")}
|
|
6726
|
+
Disable prompts: ${chalk19.green("mutagent prompts list --non-interactive")}
|
|
6727
|
+
Set default workspace: ${chalk19.green("mutagent config set workspace <workspace-id>")}
|
|
6728
|
+
Set default org: ${chalk19.green("mutagent config set org <org-id>")}
|
|
6383
6729
|
`);
|
|
6384
6730
|
program.addHelpText("after", `
|
|
6385
|
-
${
|
|
6386
|
-
${
|
|
6731
|
+
${chalk19.yellow("Workflows:")}
|
|
6732
|
+
${chalk19.bold("Evaluate → Optimize Loop:")}
|
|
6387
6733
|
1. mutagent prompts create --name "..." --raw-file prompt.txt
|
|
6388
6734
|
2. mutagent prompts dataset add <prompt-id> --name "..." --file data.json
|
|
6389
6735
|
3. mutagent prompts evaluation create <prompt-id> --name "..." --file criteria.json
|
|
6390
6736
|
4. mutagent prompts optimize start <prompt-id> --dataset <id> --max-iterations 3
|
|
6391
6737
|
|
|
6392
|
-
${
|
|
6738
|
+
${chalk19.bold("Quick Test:")}
|
|
6393
6739
|
mutagent playground run <prompt-id> --input '{"key":"value"}'
|
|
6394
6740
|
|
|
6395
|
-
${
|
|
6396
|
-
${
|
|
6397
|
-
${
|
|
6398
|
-
${
|
|
6399
|
-
${
|
|
6741
|
+
${chalk19.bold("Prerequisites for Optimization:")}
|
|
6742
|
+
${chalk19.green("✓")} Prompt with input/output parameters
|
|
6743
|
+
${chalk19.green("✓")} Dataset with items (input + expectedOutput pairs)
|
|
6744
|
+
${chalk19.green("✓")} Evaluation with criteria (field-level, input/output focused)
|
|
6745
|
+
${chalk19.dim("•")} LLM provider ${chalk19.dim("(only when server uses external providers)")}
|
|
6400
6746
|
`);
|
|
6401
6747
|
program.addHelpText("after", `
|
|
6402
|
-
${
|
|
6403
|
-
${
|
|
6404
|
-
${
|
|
6405
|
-
${
|
|
6406
|
-
${
|
|
6407
|
-
${
|
|
6408
|
-
${
|
|
6409
|
-
${
|
|
6410
|
-
${
|
|
6411
|
-
${!hasCredentials() ?
|
|
6748
|
+
${chalk19.cyan("┌─ AI AGENT INTEGRATION HINT ────────────────────────────────────────────────┐")}
|
|
6749
|
+
${chalk19.cyan("│")} ${chalk19.cyan("│")}
|
|
6750
|
+
${chalk19.cyan("│")} Frameworks: langchain, langgraph, vercel-ai, openai ${chalk19.cyan("│")}
|
|
6751
|
+
${chalk19.cyan("│")} ${chalk19.cyan("│")}
|
|
6752
|
+
${chalk19.cyan("│")} Get integration guide: mutagent integrate <framework> ${chalk19.cyan("│")}
|
|
6753
|
+
${chalk19.cyan("│")} Verify setup: mutagent integrate <framework> --verify ${chalk19.cyan("│")}
|
|
6754
|
+
${chalk19.cyan("│")} Use --json for AI parsing: mutagent <command> --json ${chalk19.cyan("│")}
|
|
6755
|
+
${chalk19.cyan("│")} ${chalk19.cyan("│")}
|
|
6756
|
+
${chalk19.cyan("└────────────────────────────────────────────────────────────────────────────┘")}
|
|
6757
|
+
${!hasCredentials() ? chalk19.yellow(`
|
|
6412
6758
|
Warning: Not authenticated. Run: mutagent auth login --browser
|
|
6413
|
-
`) : ""}${!hasRcConfig() ?
|
|
6759
|
+
`) : ""}${!hasRcConfig() ? chalk19.green(`
|
|
6414
6760
|
Get started: mutagent init
|
|
6761
|
+
`) : ""}${!existsSync12(join7(process.cwd(), ".claude/skills/mutagent-cli/SKILL.md")) ? chalk19.magenta(`
|
|
6762
|
+
Using Claude Code? Install the MutagenT skill: mutagent skills install
|
|
6415
6763
|
`) : ""}`);
|
|
6416
6764
|
program.hook("preAction", (thisCommand) => {
|
|
6417
6765
|
const globalOpts = thisCommand.optsWithGlobals();
|
|
@@ -6438,7 +6786,8 @@ program.addCommand(createWorkspacesCommand());
|
|
|
6438
6786
|
program.addCommand(createProvidersCommand());
|
|
6439
6787
|
program.addCommand(createExploreCommand());
|
|
6440
6788
|
program.addCommand(createSkillsCommand());
|
|
6789
|
+
program.addCommand(createUsageCommand());
|
|
6441
6790
|
program.parse();
|
|
6442
6791
|
|
|
6443
|
-
//# debugId=
|
|
6792
|
+
//# debugId=88B4F3961CA6D43E64756E2164756E21
|
|
6444
6793
|
//# sourceMappingURL=cli.js.map
|