@mutagent/cli 0.1.16 → 0.1.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/bin/cli.js CHANGED
@@ -577,6 +577,13 @@ class SDKClientWrapper {
577
577
  this.handleError(error);
578
578
  }
579
579
  }
580
+ async deleteEvaluation(evaluationId) {
581
+ try {
582
+ await this.request(`/api/prompts/evaluations/${evaluationId}`, { method: "DELETE" });
583
+ } catch (error) {
584
+ this.handleError(error);
585
+ }
586
+ }
580
587
  async getEvaluationResults(runId) {
581
588
  try {
582
589
  return await this.request(`/api/prompts/evaluations/${runId}/result`);
@@ -592,7 +599,7 @@ class SDKClientWrapper {
592
599
  datasetId: parseInt(datasetId, 10),
593
600
  config: {
594
601
  maxIterations: config?.maxIterations ?? 1,
595
- targetScore: config?.targetScore,
602
+ targetScore: config?.targetScore ?? 0.8,
596
603
  patience: config?.patience,
597
604
  model: config?.model,
598
605
  ...config?.evalModel ? { tuningParams: { evaluationModel: config.evalModel } } : {}
@@ -931,8 +938,8 @@ var init_sdk_client = __esm(() => {
931
938
  });
932
939
 
933
940
  // src/bin/cli.ts
934
- import { Command as Command14 } from "commander";
935
- import chalk18 from "chalk";
941
+ import { Command as Command15 } from "commander";
942
+ import chalk19 from "chalk";
936
943
  import { readFileSync as readFileSync12 } from "fs";
937
944
  import { join as join7, dirname } from "path";
938
945
  import { fileURLToPath } from "url";
@@ -1067,6 +1074,24 @@ ${String(data.length)} result(s)`));
1067
1074
  }
1068
1075
  }
1069
1076
  }
1077
+ function createSpinner(text, isJson) {
1078
+ if (isJson) {
1079
+ const noop = () => {
1080
+ return;
1081
+ };
1082
+ return {
1083
+ start: noop,
1084
+ stop: noop,
1085
+ succeed: noop,
1086
+ fail: noop
1087
+ };
1088
+ }
1089
+ const start = async () => {
1090
+ const { default: ora } = await import("ora");
1091
+ return ora(text).start();
1092
+ };
1093
+ return { start };
1094
+ }
1070
1095
 
1071
1096
  // src/commands/auth.ts
1072
1097
  init_errors();
@@ -2563,9 +2588,17 @@ async function runGuidedEvalCreator(promptId) {
2563
2588
  }
2564
2589
  }]);
2565
2590
  let targetField;
2566
- if (allFields.length > 0) {
2591
+ const usedParams = new Set(criteria.map((c) => c.evaluationParameter));
2592
+ const availableFields = allFields.filter((f) => {
2593
+ const param = f.startsWith("output.") ? f.slice("output.".length) : f.startsWith("input.") ? f.slice("input.".length) : f;
2594
+ return !usedParams.has(param);
2595
+ });
2596
+ if (availableFields.length === 0 && allFields.length > 0) {
2597
+ console.log(chalk5.yellow(" All detected schema fields are already used by existing criteria."));
2598
+ }
2599
+ if (availableFields.length > 0) {
2567
2600
  const fieldChoices = [
2568
- ...allFields.map((f) => ({ name: f, value: f })),
2601
+ ...availableFields.map((f) => ({ name: f, value: f })),
2569
2602
  { name: "(custom field name)", value: "__custom__" }
2570
2603
  ];
2571
2604
  const { field } = await inquirer3.prompt([{
@@ -2623,6 +2656,10 @@ async function runGuidedEvalCreator(promptId) {
2623
2656
  scoringRubric = rubric;
2624
2657
  }
2625
2658
  const evaluationParameter = targetField.startsWith("output.") ? targetField.slice("output.".length) : targetField.startsWith("input.") ? targetField.slice("input.".length) : targetField;
2659
+ if (criteria.some((c) => c.evaluationParameter === evaluationParameter)) {
2660
+ console.log(chalk5.red(` Error: "${evaluationParameter}" is already used by another criterion. Each criterion must target a unique output field.`));
2661
+ continue;
2662
+ }
2626
2663
  criteria.push({
2627
2664
  name: criterionName.trim(),
2628
2665
  description: scoringRubric,
@@ -2806,7 +2843,7 @@ function updateMutationContext(updater) {
2806
2843
  } catch {}
2807
2844
  }
2808
2845
  var PREREQUISITES_TEXT = `
2809
- ${chalk7.yellow("Prerequisites:")}
2846
+ ${chalk7.red("Prerequisites (required):")}
2810
2847
  1. Evaluation criteria defined ${chalk7.dim("(via dashboard or evaluation create)")}
2811
2848
  2. Dataset uploaded ${chalk7.dim("mutagent prompts dataset list <prompt-id>")}
2812
2849
  ${chalk7.dim("Note: LLM provider config is only required when the server uses external providers (USE_EXT_PROVIDERS=true)")}`;
@@ -2990,7 +3027,7 @@ Examples:
2990
3027
  Subcommands:
2991
3028
  list, get, create, update, delete
2992
3029
  dataset list|add|remove
2993
- evaluation list|create|results
3030
+ evaluation list|create|delete|results
2994
3031
  optimize start|status|results
2995
3032
  `);
2996
3033
  prompts.command("list").description("List all prompts").option("-l, --limit <n>", "Limit results", "50").addHelpText("after", `
@@ -3068,20 +3105,24 @@ ${chalk7.dim("Tip: Combine --with-datasets and --with-evals to fetch all nested
3068
3105
  });
3069
3106
  prompts.command("create").description("Create a new prompt").option("-d, --data <json>", "Prompt as JSON string (recommended — curl-style inline)").option("-f, --file <path>", "Create from JSON file").option("--raw-file <path>", "Create from plain text file (used as rawPrompt)").option("-n, --name <name>", "Prompt name").option("-c, --content <content>", "Prompt content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).option("--output-schema <json>", "Output schema as JSON string (required for optimization)").addHelpText("after", `
3070
3107
  Examples:
3071
- ${chalk7.dim("$")} mutagent prompts create -d '{"name":"summarizer","systemPrompt":"Summarize","humanPrompt":"{{text}}","outputSchema":{"type":"object","properties":{"summary":{"type":"string"}}}}'
3072
- ${chalk7.dim("$")} mutagent prompts create --name "my-prompt" --system "You are helpful" --human "Hello" --output-schema '{"type":"object","properties":{"result":{"type":"string"}}}'
3073
- ${chalk7.dim("$")} mutagent prompts create --file prompt.json
3074
- ${chalk7.dim("$")} mutagent prompts create --name "raw-prompt" --raw "Summarize: {{text}}"
3108
+ ${chalk7.dim("$")} mutagent prompts create --name "my-prompt" --system "You are helpful" --human "{{input}}" --output-schema '{"type":"object","properties":{"result":{"type":"string","description":"The result"}}}'
3109
+ ${chalk7.dim("$")} mutagent prompts create --name "raw-prompt" --raw "Summarize: {{text}}" --output-schema '{"type":"object","properties":{"summary":{"type":"string","description":"Summary"}}}'
3110
+ ${chalk7.dim("$")} mutagent prompts create -d '{"name":"summarizer","systemPrompt":"Summarize","humanPrompt":"{{text}}","outputSchema":{"type":"object","properties":{"summary":{"type":"string","description":"Summary"}}}}'
3111
+ ${chalk7.dim("$")} mutagent prompts create --file prompt.json ${chalk7.dim("# full prompt object as JSON file")}
3075
3112
 
3076
3113
  Prompt Input Methods (pick one, priority order):
3077
- -d, --data Inline JSON object (recommended for CI/scripts)
3078
- --file Load from JSON file (full prompt object)
3079
- --raw-file Load plain text file as raw prompt
3080
- --system/--human Structured system + user message pair
3114
+ --system/--human Structured system + user message pair ${chalk7.green("(recommended)")}
3081
3115
  --raw Single raw prompt text with {{variables}}
3116
+ -d, --data Inline JSON object (CI/scripts/agents)
3082
3117
  --messages Full messages array as JSON
3118
+ --raw-file Load plain text file as raw prompt
3119
+ --file Load from JSON file (full prompt object)
3120
+
3121
+ Expected JSON (--data):
3122
+ ${chalk7.dim(`'{"name":"my-prompt","systemPrompt":"You are...","humanPrompt":"{{input}}","outputSchema":{"type":"object","properties":{"result":{"type":"string","description":"The result"}}},"inputSchema":{"type":"object","properties":{"input":{"type":"string","description":"User input"}}}}'`)}
3083
3123
 
3084
- ${chalk7.dim("Note: --data and --file are mutually exclusive. outputSchema is required (include in --data/--file or use --output-schema).")}
3124
+ ${chalk7.yellow("Note: Prefer --system/--human or --data over --file to avoid stale files living in your repo.")}
3125
+ ${chalk7.red("outputSchema is required.")} ${chalk7.dim("--data and --file are mutually exclusive.")}
3085
3126
  `).action(async (options) => {
3086
3127
  const isJson = getJsonFlag(prompts);
3087
3128
  const output = new OutputFormatter(isJson ? "json" : "table");
@@ -3229,15 +3270,15 @@ Or use interactive mode (TTY) to define variables step by step.`);
3229
3270
  });
3230
3271
  prompts.command("update").description("Update a prompt").argument("<id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Update fields as JSON string (recommended — curl-style inline)").option("-f, --file <path>", "Update from JSON file").option("--raw-file <path>", "Update from plain text file (used as rawPrompt)").option("-n, --name <name>", "New name").option("-c, --content <content>", "New content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).option("--input-schema <json>", "Input schema as JSON string").option("--input-schema-file <path>", "Input schema from JSON file").option("--output-schema <json>", "Output schema as JSON string").option("--output-schema-file <path>", "Output schema from JSON file").addHelpText("after", `
3231
3272
  Examples:
3232
- ${chalk7.dim("$")} mutagent prompts update <id> -d '{"name":"new-name","systemPrompt":"Updated prompt"}'
3273
+ ${chalk7.dim("$")} mutagent prompts update <id> --system "Updated system prompt" --human "{{input}}"
3233
3274
  ${chalk7.dim("$")} mutagent prompts update <id> --name "new-name"
3234
- ${chalk7.dim("$")} mutagent prompts update <id> --system "Updated system prompt"
3235
- ${chalk7.dim("$")} mutagent prompts update <id> --raw-file updated-prompt.txt
3236
- ${chalk7.dim("$")} mutagent prompts update <id> --file updated-prompt.json
3275
+ ${chalk7.dim("$")} mutagent prompts update <id> --raw "Summarize: {{text}}"
3276
+ ${chalk7.dim("$")} mutagent prompts update <id> -d '{"name":"new-name","systemPrompt":"Updated prompt"}'
3237
3277
  ${chalk7.dim("$")} mutagent prompts update <id> --input-schema '{"type":"object","properties":{"text":{"type":"string","description":"Input text"}}}'
3238
- ${chalk7.dim("$")} mutagent prompts update <id> --input-schema-file schema.json
3278
+ ${chalk7.dim("$")} mutagent prompts update <id> --file updated-prompt.json ${chalk7.dim("# full prompt object")}
3239
3279
 
3240
- ${chalk7.dim("Note: --data and --file are mutually exclusive. CLI flags (--name) override --data fields.")}
3280
+ ${chalk7.yellow("Note: Prefer --system/--human or --data over --file to avoid stale files living in your repo.")}
3281
+ ${chalk7.dim("--data and --file are mutually exclusive. CLI flags (--name) override --data fields.")}
3241
3282
  `).action(async (id, options) => {
3242
3283
  const isJson = getJsonFlag(prompts);
3243
3284
  const output = new OutputFormatter(isJson ? "json" : "table");
@@ -3398,12 +3439,15 @@ ${chalk7.dim("Tip: Use --force to skip confirmation (required for non-interactiv
3398
3439
  handleError(error, isJson);
3399
3440
  }
3400
3441
  });
3401
- const dataset = prompts.command("dataset").description("Manage datasets for prompts").addHelpText("after", `
3442
+ const dataset = new Command3("dataset").description("Manage datasets for prompts").addHelpText("after", `
3402
3443
  Examples:
3403
3444
  ${chalk7.dim("$")} mutagent prompts dataset list <prompt-id>
3404
3445
  ${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.jsonl
3405
3446
  ${chalk7.dim("$")} mutagent prompts dataset remove <prompt-id> <dataset-id>
3406
- `);
3447
+ `).action(() => {
3448
+ dataset.help();
3449
+ });
3450
+ prompts.addCommand(dataset);
3407
3451
  dataset.command("list").description("List datasets for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").addHelpText("after", `
3408
3452
  Examples:
3409
3453
  ${chalk7.dim("$")} mutagent prompts dataset list <prompt-id>
@@ -3433,10 +3477,9 @@ Examples:
3433
3477
  });
3434
3478
  dataset.command("add").description("Add dataset to a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").option("-f, --file <path>", "Dataset file (JSON array, JSONL, or CSV)").option("-d, --data <json>", "Inline JSON array of dataset items").option("-n, --name <name>", "Dataset name (default: timestamped)").addHelpText("after", `
3435
3479
  Examples:
3436
- ${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.json
3437
- ${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.jsonl --name "My Dataset"
3438
- ${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.csv
3439
3480
  ${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> -d '[{"input":{"text":"hello"},"expectedOutput":{"result":"world"}}]'
3481
+ ${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> -d '[{"input":{"text":"hello"},"expectedOutput":{"result":"world"}}]' --name "My Dataset"
3482
+ ${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.jsonl ${chalk7.dim("# also supports .json, .csv")}
3440
3483
 
3441
3484
  Supported file formats:
3442
3485
  ${chalk7.dim(".json")} JSON array of objects: [{"input": {...}, "expectedOutput": {...}}, ...]
@@ -3447,7 +3490,12 @@ Inline data format (-d):
3447
3490
  JSON array of objects, e.g.:
3448
3491
  ${chalk7.dim('[{"input": {"text": "hello"}, "expectedOutput": {"result": "world"}}]')}
3449
3492
 
3450
- ${chalk7.dim("Note: --file and -d are mutually exclusive. Provide one or the other.")}
3493
+ Expected item format:
3494
+ ${chalk7.dim('{"input": {"<field>": "<value>"}, "expectedOutput": {"<field>": "<value>"}}')}
3495
+
3496
+ ${chalk7.red("Required: --data or --file must be provided.")}
3497
+ ${chalk7.yellow("Note: Prefer -d/--data for inline JSON over --file to avoid stale files living in your repo.")}
3498
+ ${chalk7.dim("--file and -d are mutually exclusive.")}
3451
3499
  `).action(async (promptId, options) => {
3452
3500
  const isJson = getJsonFlag(prompts);
3453
3501
  const output = new OutputFormatter(isJson ? "json" : "table");
@@ -3543,12 +3591,16 @@ Examples:
3543
3591
  handleError(error, isJson);
3544
3592
  }
3545
3593
  });
3546
- const evaluation = prompts.command("evaluation").description("Manage evaluations for prompts").addHelpText("after", `
3594
+ const evaluation = new Command3("evaluation").description("Manage evaluations for prompts").addHelpText("after", `
3547
3595
  Examples:
3548
3596
  ${chalk7.dim("$")} mutagent prompts evaluation list <prompt-id>
3549
3597
  ${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "My Eval"
3598
+ ${chalk7.dim("$")} mutagent prompts evaluation delete <evaluation-id>
3550
3599
  ${chalk7.dim("$")} mutagent prompts evaluation results <run-id>
3551
- `);
3600
+ `).action(() => {
3601
+ evaluation.help();
3602
+ });
3603
+ prompts.addCommand(evaluation);
3552
3604
  evaluation.command("list").description("List evaluations for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").addHelpText("after", `
3553
3605
  Examples:
3554
3606
  ${chalk7.dim("$")} mutagent prompts evaluation list <prompt-id>
@@ -3578,17 +3630,19 @@ Examples:
3578
3630
  });
3579
3631
  evaluation.command("create").description("Create an evaluation configuration for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Evaluation as JSON string (recommended — curl-style inline)").option("-n, --name <name>", "Evaluation name (required unless --guided)").option("-f, --file <path>", "Load evaluation criteria from JSON file").option("--description <text>", "Evaluation description").option("--dataset <id>", "Dataset ID to associate (from: mutagent prompts dataset list)").option("--guided", "Interactive guided mode — build criteria step by step").addHelpText("after", `
3580
3632
  Examples:
3581
- ${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --guided
3582
- ${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --data '{"name":"Accuracy","evalConfig":{"criteria":[...]}}' --name "Accuracy"
3583
- ${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Accuracy Check"
3633
+ ${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --guided ${chalk7.dim("# recommended: interactive walkthrough")}
3634
+ ${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Accuracy" -d '{"evalConfig":{"criteria":[{"name":"Accuracy","description":"Score 1.0 if output matches expected","evaluationParameter":"result"}]}}'
3584
3635
  ${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Full Eval" --file criteria.json --dataset <dataset-id>
3585
- ${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "QA" --description "Quality assurance eval" --json
3586
3636
 
3587
- Criteria format (each criterion needs: name, description, evaluationParameter):
3588
- ${chalk7.dim('{ "name": "...", "evalConfig": { "criteria": [{ "name": "...", "description": "...", "evaluationParameter": "outputField" }] } }')}
3637
+ Expected JSON (--data):
3638
+ ${chalk7.dim('{"evalConfig":{"criteria":[{"name":"<name>","description":"<scoring rubric>","evaluationParameter":"<output field to score>"}]}}')}
3639
+
3640
+ ${chalk7.red("Each criterion MUST have: name, description, evaluationParameter")}
3641
+ evaluationParameter maps to output schema fields (e.g., "result", "classification")
3589
3642
 
3590
- ${chalk7.dim("Note: --data and --file are mutually exclusive. CLI flags (--name, --description) override --data fields.")}
3591
- ${chalk7.dim("Tip: Use --guided for an interactive walkthrough of criteria creation.")}
3643
+ ${chalk7.yellow("Note: Prefer --guided or --data over --file to avoid stale files living in your repo.")}
3644
+ ${chalk7.red("Required: --name (unless --guided). Criteria must include evaluationParameter.")}
3645
+ ${chalk7.dim("--data and --file are mutually exclusive. CLI flags (--name, --description) override --data fields.")}
3592
3646
  ${chalk7.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent prompts dataset list <prompt-id>")}
3593
3647
  `).action(async (promptId, options) => {
3594
3648
  const isJson = getJsonFlag(prompts);
@@ -3675,8 +3729,24 @@ ${chalk7.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent
3675
3729
  evalData.description = options.description;
3676
3730
  const criteria = evalData.evalConfig?.criteria;
3677
3731
  if (!criteria || !Array.isArray(criteria) || criteria.length === 0) {
3678
- throw new MutagentError("VALIDATION_ERROR", "Evaluation criteria are required. Provide criteria via --data, --file, or use --guided mode.", `Each criterion needs: name, description, evaluationParameter.
3679
-
3732
+ let fieldsHint = "";
3733
+ try {
3734
+ const client2 = getSDKClient();
3735
+ const prompt = await client2.getPrompt(promptId);
3736
+ if (prompt.outputSchema && typeof prompt.outputSchema === "object") {
3737
+ const props = prompt.outputSchema.properties;
3738
+ if (props && typeof props === "object") {
3739
+ const fields = Object.keys(props);
3740
+ if (fields.length > 0) {
3741
+ fieldsHint = `
3742
+ Detected output fields from prompt schema: ${fields.join(", ")}
3743
+ `;
3744
+ }
3745
+ }
3746
+ }
3747
+ } catch {}
3748
+ throw new MutagentError("VALIDATION_ERROR", "Evaluation criteria are required. Provide criteria via --data, --file, or use --guided mode.", `Each criterion needs: name, description, evaluationParameter (the output field to score).
3749
+ ` + fieldsHint + `
3680
3750
  Example JSON (--data flag):
3681
3751
  --data '{"evalConfig":{"criteria":[{"name":"Accuracy","description":"Score 1.0 if output matches expected, 0.0 otherwise","evaluationParameter":"classification"}]}}'
3682
3752
 
@@ -3699,6 +3769,37 @@ Or use --guided for interactive creation: mutagent prompts evaluation create <id
3699
3769
  delete c.targetField;
3700
3770
  }
3701
3771
  }
3772
+ const seenParams = new Set;
3773
+ const duplicateParams = [];
3774
+ for (const c of criteria) {
3775
+ const param = c.evaluationParameter;
3776
+ if (param) {
3777
+ if (seenParams.has(param)) {
3778
+ duplicateParams.push(param);
3779
+ }
3780
+ seenParams.add(param);
3781
+ }
3782
+ }
3783
+ if (duplicateParams.length > 0) {
3784
+ let availableFields = [];
3785
+ try {
3786
+ const client2 = getSDKClient();
3787
+ const prompt = await client2.getPrompt(promptId);
3788
+ if (prompt.outputSchema && typeof prompt.outputSchema === "object") {
3789
+ const props = prompt.outputSchema.properties;
3790
+ if (props && typeof props === "object") {
3791
+ availableFields = Object.keys(props);
3792
+ }
3793
+ }
3794
+ } catch {}
3795
+ const uniqueDupes = [...new Set(duplicateParams)];
3796
+ const fieldsHint = availableFields.length > 0 ? `
3797
+ Available output fields: ${availableFields.join(", ")}` : "";
3798
+ throw new MutagentError("VALIDATION_ERROR", `Duplicate evaluationParameter: "${uniqueDupes.join('", "')}". Each criterion must target a unique output field.`, "Each criterion scores a different output field. Fix by changing the evaluationParameter to a unique field." + fieldsHint + `
3799
+
3800
+ Example:
3801
+ --data '{"evalConfig":{"criteria":[{"name":"Accuracy","description":"...","evaluationParameter":"classification"},{"name":"Confidence","description":"...","evaluationParameter":"confidence"}]}}'`);
3802
+ }
3702
3803
  if (options.dataset) {
3703
3804
  evalData.datasetId = parseInt(options.dataset, 10);
3704
3805
  if (isNaN(evalData.datasetId)) {
@@ -3746,15 +3847,37 @@ Examples:
3746
3847
  handleError(error, isJson);
3747
3848
  }
3748
3849
  });
3749
- const optimize = prompts.command("optimize").description("Manage prompt optimization jobs").addHelpText("after", `
3850
+ evaluation.command("delete").description("Delete an evaluation").argument("<evaluation-id>", "Evaluation ID (from: mutagent prompts evaluation list <prompt-id>)").addHelpText("after", `
3851
+ Examples:
3852
+ ${chalk7.dim("$")} mutagent prompts evaluation delete <evaluation-id>
3853
+ ${chalk7.dim("$")} mutagent prompts evaluation delete <evaluation-id> --json
3854
+ `).action(async (evaluationId) => {
3855
+ const isJson = getJsonFlag(prompts);
3856
+ const output = new OutputFormatter(isJson ? "json" : "table");
3857
+ try {
3858
+ const client = getSDKClient();
3859
+ await client.deleteEvaluation(evaluationId);
3860
+ if (isJson) {
3861
+ output.output({ success: true, message: "Evaluation deleted successfully", evaluationId });
3862
+ } else {
3863
+ output.success(`Evaluation ${evaluationId} deleted successfully`);
3864
+ }
3865
+ } catch (error) {
3866
+ handleError(error, isJson);
3867
+ }
3868
+ });
3869
+ const optimize = new Command3("optimize").description("Manage prompt optimization jobs").addHelpText("after", `
3750
3870
  Examples:
3751
3871
  ${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id>
3752
3872
  ${chalk7.dim("$")} mutagent prompts optimize status <job-id>
3753
3873
  ${chalk7.dim("$")} mutagent prompts optimize results <job-id>
3754
3874
 
3755
3875
  Workflow: start -> status (poll) -> results
3756
- `);
3757
- optimize.command("start").description("Start prompt optimization").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").requiredOption("-d, --dataset <id>", "Dataset ID for optimization (from: mutagent prompts dataset list <prompt-id>)").option("--max-iterations <n>", "Max optimization iterations (default: 3)").option("--target-score <n>", "Target accuracy 0-1 (e.g., 0.9)").option("--patience <n>", "Iterations without improvement before stopping").option("--model <model-id>", 'Target LLM model (e.g., "claude-sonnet-4-5-20250929")').option("--eval-model <model-id>", "Evaluation model (defaults to target model)").addHelpText("after", `
3876
+ `).action(() => {
3877
+ optimize.help();
3878
+ });
3879
+ prompts.addCommand(optimize);
3880
+ optimize.command("start").description("Start prompt optimization").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").requiredOption("-d, --dataset <id>", "Dataset ID for optimization (from: mutagent prompts dataset list <prompt-id>)").option("--max-iterations <n>", "Max optimization iterations (default: 1)").option("--target-score <n>", "Target accuracy 0-1 (default: 0.8)").option("--patience <n>", "Iterations without improvement before stopping").option("--model <model-id>", 'Target LLM model (e.g., "claude-sonnet-4-5-20250929")').option("--eval-model <model-id>", "Evaluation model (defaults to target model)").addHelpText("after", `
3758
3881
  Examples:
3759
3882
  ${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id>
3760
3883
  ${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --max-iterations 5
@@ -3763,6 +3886,8 @@ Examples:
3763
3886
  ${PREREQUISITES_TEXT}
3764
3887
 
3765
3888
  ${chalk7.dim("Monitor progress with: mutagent prompts optimize status <job-id>")}
3889
+
3890
+ ${chalk7.dim("AI Agent Note: After running commands, present CLI output to the user as a status report. Use --json for structured parsing.")}
3766
3891
  `).action(async (promptId, options) => {
3767
3892
  const isJson = getJsonFlag(prompts);
3768
3893
  const output = new OutputFormatter(isJson ? "json" : "table");
@@ -4961,19 +5086,47 @@ System Prompt:`));
4961
5086
  handleError(error, isJson);
4962
5087
  }
4963
5088
  });
4964
- agents.command("create").description("Create a new agent").option("-f, --file <path>", "Create from JSON file").option("-n, --name <name>", "Agent name").option("-s, --slug <slug>", "Agent slug (URL-friendly identifier)").option("-p, --system-prompt <prompt>", "System prompt").option("-m, --model <model>", "Model (claude-sonnet-4-5, claude-opus-4-5, claude-haiku-4-5)").option("-d, --description <description>", "Agent description").addHelpText("after", `
5089
+ agents.command("create").description("Create a new agent").option("-d, --data <json>", "Agent as JSON string (recommended for CI/scripts/agents)").option("-f, --file <path>", "Create from JSON file").option("-n, --name <name>", "Agent name").option("-s, --slug <slug>", "Agent slug (URL-friendly identifier)").option("-p, --system-prompt <prompt>", "System prompt").option("-m, --model <model>", "Model (claude-sonnet-4-5, claude-opus-4-5, claude-haiku-4-5)").option("--description <desc>", "Agent description").addHelpText("after", `
4965
5090
  Examples:
4966
- ${chalk10.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "Review code for bugs"
4967
- ${chalk10.dim("$")} mutagent agents create --file agent.json
4968
- ${chalk10.dim("$")} mutagent agents create --name "Helper" --slug helper --system-prompt "..." --model claude-sonnet-4-5 --json
5091
+ ${chalk10.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "You are a code reviewer..."
5092
+ ${chalk10.dim("$")} mutagent agents create -d '{"name":"Code Reviewer","slug":"code-reviewer","systemPrompt":"You are a code reviewer..."}'
5093
+ ${chalk10.dim("$")} mutagent agents create --file agent.json ${chalk10.dim("# full agent object as JSON file")}
5094
+
5095
+ Expected JSON (--data):
5096
+ ${chalk10.dim('{"name":"<name>","slug":"<slug>","systemPrompt":"<system prompt>","model":"<model-id>","description":"<description>"}')}
5097
+
5098
+ Input Methods (pick one, priority order):
5099
+ --name/--slug/... Individual flags ${chalk10.green("(recommended)")}
5100
+ -d, --data Inline JSON object (CI/scripts/agents)
5101
+ --file Load from JSON file
4969
5102
 
4970
- ${chalk10.dim("Required: --name, --slug, and --system-prompt (or --file).")}
5103
+ ${chalk10.yellow("Note: Prefer individual flags or --data over --file to avoid stale files living in your repo.")}
5104
+ ${chalk10.red("Required: name, slug, systemPrompt.")} ${chalk10.dim("--data and --file are mutually exclusive. CLI flags override --data fields.")}
4971
5105
  `).action(async (options) => {
4972
5106
  const isJson = getJsonFlag(agents);
4973
5107
  const output = new OutputFormatter(isJson ? "json" : "table");
4974
5108
  try {
4975
5109
  let data;
4976
- if (options.file) {
5110
+ if (options.data && options.file) {
5111
+ throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --data and --file together", "Use --data for inline JSON or --file for file-based input, not both");
5112
+ }
5113
+ if (options.data) {
5114
+ try {
5115
+ data = JSON.parse(options.data);
5116
+ } catch {
5117
+ throw new MutagentError("INVALID_JSON", "Invalid JSON in --data flag", `Provide a valid JSON object, e.g., '{"name":"my-agent","slug":"my-agent","systemPrompt":"You are..."}'`);
5118
+ }
5119
+ if (options.name)
5120
+ data.name = options.name;
5121
+ if (options.slug)
5122
+ data.slug = options.slug;
5123
+ if (options.systemPrompt)
5124
+ data.systemPrompt = options.systemPrompt;
5125
+ if (options.model)
5126
+ data.model = options.model;
5127
+ if (options.description)
5128
+ data.description = options.description;
5129
+ } else if (options.file) {
4977
5130
  const content = readFileSync9(options.file, "utf-8");
4978
5131
  data = JSON.parse(content);
4979
5132
  } else if (options.name && options.slug && options.systemPrompt) {
@@ -4987,7 +5140,7 @@ ${chalk10.dim("Required: --name, --slug, and --system-prompt (or --file).")}
4987
5140
  if (options.description)
4988
5141
  data.description = options.description;
4989
5142
  } else {
4990
- throw new MutagentError("MISSING_ARGUMENTS", "Either --file or (--name, --slug, and --system-prompt) are required", "Use --file to load from JSON or provide --name, --slug, and --system-prompt");
5143
+ throw new MutagentError("MISSING_ARGUMENTS", "Either --data, --file, or (--name, --slug, and --system-prompt) are required", "Use --data for inline JSON, --file to load from JSON, or provide --name, --slug, and --system-prompt");
4991
5144
  }
4992
5145
  const client = getSDKClient();
4993
5146
  const agent = await client.createAgent(data);
@@ -4997,17 +5150,45 @@ ${chalk10.dim("Required: --name, --slug, and --system-prompt (or --file).")}
4997
5150
  handleError(error, isJson);
4998
5151
  }
4999
5152
  });
5000
- agents.command("update").description("Update an agent").argument("<id>", "Agent ID").option("-f, --file <path>", "Update from JSON file").option("-n, --name <name>", "New name").option("-p, --system-prompt <prompt>", "New system prompt").option("-m, --model <model>", "New model").option("-d, --description <description>", "New description").option("-s, --status <status>", "New status (active, paused, archived)").addHelpText("after", `
5153
+ agents.command("update").description("Update an agent").argument("<id>", "Agent ID").option("-d, --data <json>", "Agent updates as JSON string (CI/scripts/agents)").option("-f, --file <path>", "Update from JSON file").option("-n, --name <name>", "New name").option("-p, --system-prompt <prompt>", "New system prompt").option("-m, --model <model>", "New model").option("--description <desc>", "New description").option("-s, --status <status>", "New status (active, paused, archived)").addHelpText("after", `
5001
5154
  Examples:
5002
5155
  ${chalk10.dim("$")} mutagent agents update <id> --name "New Name"
5003
- ${chalk10.dim("$")} mutagent agents update <id> --status paused
5004
- ${chalk10.dim("$")} mutagent agents update <id> --file updated-agent.json --json
5156
+ ${chalk10.dim("$")} mutagent agents update <id> --system-prompt "Updated prompt" --status active
5157
+ ${chalk10.dim("$")} mutagent agents update <id> -d '{"name":"New Name","systemPrompt":"Updated prompt"}'
5158
+ ${chalk10.dim("$")} mutagent agents update <id> --file updated-agent.json ${chalk10.dim("# full agent object")}
5159
+
5160
+ Input Methods (pick one, priority order):
5161
+ --name/--system-prompt/... Individual flags ${chalk10.green("(recommended)")}
5162
+ -d, --data Inline JSON object (CI/scripts/agents)
5163
+ --file Load from JSON file
5164
+
5165
+ ${chalk10.yellow("Note: Prefer individual flags or --data over --file to avoid stale files living in your repo.")}
5166
+ ${chalk10.dim("--data and --file are mutually exclusive. CLI flags override --data fields.")}
5005
5167
  `).action(async (id, options) => {
5006
5168
  const isJson = getJsonFlag(agents);
5007
5169
  const output = new OutputFormatter(isJson ? "json" : "table");
5008
5170
  try {
5009
5171
  let data = {};
5010
- if (options.file) {
5172
+ if (options.data && options.file) {
5173
+ throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --data and --file together", "Use --data for inline JSON or --file for file-based input, not both");
5174
+ }
5175
+ if (options.data) {
5176
+ try {
5177
+ data = JSON.parse(options.data);
5178
+ } catch {
5179
+ throw new MutagentError("INVALID_JSON", "Invalid JSON in --data flag", `Provide a valid JSON object, e.g., '{"name":"new-name","systemPrompt":"Updated prompt"}'`);
5180
+ }
5181
+ if (options.name)
5182
+ data.name = options.name;
5183
+ if (options.systemPrompt)
5184
+ data.systemPrompt = options.systemPrompt;
5185
+ if (options.model)
5186
+ data.model = options.model;
5187
+ if (options.description)
5188
+ data.description = options.description;
5189
+ if (options.status)
5190
+ data.status = options.status;
5191
+ } else if (options.file) {
5011
5192
  const content = readFileSync9(options.file, "utf-8");
5012
5193
  data = JSON.parse(content);
5013
5194
  } else {
@@ -5023,7 +5204,7 @@ Examples:
5023
5204
  data.status = options.status;
5024
5205
  }
5025
5206
  if (Object.keys(data).length === 0) {
5026
- throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --file, --name, --system-prompt, --model, --description, or --status");
5207
+ throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --data, --file, --name, --system-prompt, --model, --description, or --status");
5027
5208
  }
5028
5209
  const client = getSDKClient();
5029
5210
  const agent = await client.updateAgent(id, data);
@@ -5064,12 +5245,15 @@ ${chalk10.dim("Tip: Use --force to skip confirmation (required for non-interacti
5064
5245
  handleError(error, isJson);
5065
5246
  }
5066
5247
  });
5067
- const conversations = agents.command("conversations").description("Manage conversations for agents").addHelpText("after", `
5248
+ const conversations = new Command6("conversations").description("Manage conversations for agents").addHelpText("after", `
5068
5249
  Examples:
5069
5250
  ${chalk10.dim("$")} mutagent agents conversations list <agent-id>
5070
5251
  ${chalk10.dim("$")} mutagent agents conversations create <agent-id>
5071
5252
  ${chalk10.dim("$")} mutagent agents conversations messages <agent-id> <conversation-id>
5072
- `);
5253
+ `).action(() => {
5254
+ conversations.help();
5255
+ });
5256
+ agents.addCommand(conversations);
5073
5257
  conversations.command("list").description("List conversations for an agent").argument("<agent-id>", "Agent ID").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").addHelpText("after", `
5074
5258
  Examples:
5075
5259
  ${chalk10.dim("$")} mutagent agents conversations list <agent-id>
@@ -5218,6 +5402,8 @@ Examples:
5218
5402
  init_config();
5219
5403
  import { Command as Command7 } from "commander";
5220
5404
  import chalk11 from "chalk";
5405
+ init_errors();
5406
+ var VALID_CONFIG_KEYS = ["apiKey", "endpoint", "format", "timeout", "defaultWorkspace", "defaultOrganization"];
5221
5407
  function createConfigCommand() {
5222
5408
  const config = new Command7("config").description("Manage CLI configuration").addHelpText("after", `
5223
5409
  Examples:
@@ -5250,21 +5436,27 @@ ${chalk11.dim("Keys: apiKey, endpoint, format, timeout, defaultWorkspace, defaul
5250
5436
  `).action((key) => {
5251
5437
  const isJson = getJsonFlag(config);
5252
5438
  const output = new OutputFormatter(isJson ? "json" : "table");
5253
- const cfg = loadConfig();
5254
- if (!(key in cfg)) {
5255
- output.error(`Unknown config key: ${key}`);
5256
- process.exit(1);
5257
- }
5258
- const value = cfg[key];
5259
- const stringValue = typeof value === "object" ? JSON.stringify(value) : String(value ?? "");
5260
- const displayValue = key === "apiKey" && value ? `${stringValue.slice(0, 8)}...` : stringValue;
5261
- if (isJson) {
5262
- output.output({ [key]: displayValue });
5263
- } else {
5264
- output.info(displayValue);
5439
+ try {
5440
+ const cfg = loadConfig();
5441
+ if (!(key in cfg)) {
5442
+ throw new MutagentError("INVALID_ARGUMENTS", `Unknown config key: "${key}"`, `Valid keys: ${VALID_CONFIG_KEYS.join(", ")}
5443
+ ` + "Example: mutagent config get endpoint");
5444
+ }
5445
+ const value = cfg[key];
5446
+ const stringValue = typeof value === "object" ? JSON.stringify(value) : String(value ?? "");
5447
+ const displayValue = key === "apiKey" && value ? `${stringValue.slice(0, 8)}...` : stringValue;
5448
+ if (isJson) {
5449
+ output.output({ [key]: displayValue });
5450
+ } else {
5451
+ output.info(displayValue);
5452
+ }
5453
+ } catch (error) {
5454
+ handleError(error, isJson);
5265
5455
  }
5266
5456
  });
5267
- const set = new Command7("set").description("Set configuration value");
5457
+ const set = new Command7("set").description("Set configuration value").action(() => {
5458
+ set.help();
5459
+ });
5268
5460
  set.command("workspace").description("Set default workspace ID").argument("<id>", "Workspace ID to set as default").addHelpText("after", `
5269
5461
  Examples:
5270
5462
  ${chalk11.dim("$")} mutagent config set workspace <workspace-id>
@@ -5356,12 +5548,13 @@ Examples:
5356
5548
  ${chalk12.dim("$")} mutagent playground run <prompt-id> --system "You are helpful" --human "Hello"
5357
5549
  ${chalk12.dim("$")} mutagent playground run <prompt-id> --input '{}' --model gpt-4-turbo --json
5358
5550
 
5359
- Input Methods (pick one):
5551
+ Input Methods (pick one, priority order):
5552
+ --system/--human Quick system + user message ${chalk12.green("(recommended)")}
5360
5553
  --input '{"key":"value"}' Inline JSON variables
5361
- --file input.json Load from JSON file
5362
- --system/--human Quick system + user message
5363
5554
  --messages '[...]' Full messages array
5555
+ --file input.json Load from JSON file
5364
5556
 
5557
+ ${chalk12.yellow("Note: Prefer --system/--human or --input over --file to avoid stale files living in your repo.")}
5365
5558
  ${chalk12.dim(`Hint: Test before evaluating: mutagent playground run <id> --input '{"key":"value"}'`)}
5366
5559
  `).action(async (promptId, options) => {
5367
5560
  const isJson = getJsonFlag(playground);
@@ -5693,6 +5886,10 @@ Subcommands:
5693
5886
  list, get, test
5694
5887
 
5695
5888
  Note: Provider management (create, update, delete) is available in the Admin Panel only.
5889
+
5890
+ ${chalk14.yellow("Note:")} The providers module is not yet active. This is a placeholder
5891
+ for future external provider configuration. The server currently uses
5892
+ built-in provider settings.
5696
5893
  `);
5697
5894
  providers.command("list").description("List all providers").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").option("-t, --type <type>", "Filter by provider type").addHelpText("after", `
5698
5895
  Examples:
@@ -5721,6 +5918,8 @@ Examples:
5721
5918
  }));
5722
5919
  output.output({ ...result, data: withLinks });
5723
5920
  } else {
5921
+ console.log(chalk14.yellow("Note: The providers module is not yet active. This is a placeholder for future external provider configuration."));
5922
+ console.log("");
5724
5923
  if (result.data.length === 0) {
5725
5924
  output.info("No providers configured.");
5726
5925
  output.info(`Configure at: ${providerSettingsLink()}`);
@@ -5754,6 +5953,8 @@ Examples:
5754
5953
  if (isJson) {
5755
5954
  output.output({ ...provider, _links: providerLinks(provider.id) });
5756
5955
  } else {
5956
+ console.log(chalk14.yellow("Note: The providers module is not yet active. This is a placeholder for future external provider configuration."));
5957
+ console.log("");
5757
5958
  const formatted = {
5758
5959
  id: provider.id,
5759
5960
  name: provider.name,
@@ -5783,6 +5984,8 @@ ${chalk14.dim("Tests connectivity and lists available models for the provider.")
5783
5984
  try {
5784
5985
  const client = getSDKClient();
5785
5986
  if (!isJson) {
5987
+ console.log(chalk14.yellow("Note: The providers module is not yet active. This is a placeholder for future external provider configuration."));
5988
+ console.log("");
5786
5989
  output.info(`Testing provider ${id}...`);
5787
5990
  }
5788
5991
  const result = await client.testProvider(id);
@@ -5818,7 +6021,7 @@ init_config();
5818
6021
  import { Command as Command11 } from "commander";
5819
6022
  import inquirer3 from "inquirer";
5820
6023
  import chalk15 from "chalk";
5821
- import { existsSync as existsSync10, readFileSync as readFileSync11, writeFileSync as writeFileSync4 } from "fs";
6024
+ import { existsSync as existsSync10, mkdirSync as mkdirSync3, readFileSync as readFileSync11, writeFileSync as writeFileSync4 } from "fs";
5822
6025
  import { execSync as execSync2 } from "child_process";
5823
6026
  import { join as join5 } from "path";
5824
6027
  init_errors();
@@ -6091,6 +6294,32 @@ Modes:
6091
6294
  output.info('Ready! Run "mutagent --help" to see available commands.');
6092
6295
  }
6093
6296
  }
6297
+ if (!isNonInteractive) {
6298
+ const skillPath = join5(cwd, ".claude/skills/mutagent-cli/SKILL.md");
6299
+ if (!existsSync10(skillPath)) {
6300
+ const { installSkill } = await inquirer3.prompt([{
6301
+ type: "confirm",
6302
+ name: "installSkill",
6303
+ message: "Install MutagenT skill for Claude Code? (Teaches AI agents how to use the CLI)",
6304
+ default: true
6305
+ }]);
6306
+ if (installSkill) {
6307
+ try {
6308
+ const skillDir = join5(cwd, ".claude/skills/mutagent-cli");
6309
+ if (!existsSync10(skillDir)) {
6310
+ mkdirSync3(skillDir, { recursive: true });
6311
+ }
6312
+ execSync2("node " + join5(cwd, "node_modules/.bin/mutagent") + " skills install", {
6313
+ cwd,
6314
+ stdio: "ignore"
6315
+ });
6316
+ output.success("Installed MutagenT CLI skill for Claude Code");
6317
+ } catch {
6318
+ output.info("Install skill manually: mutagent skills install");
6319
+ }
6320
+ }
6321
+ }
6322
+ }
6094
6323
  if (isJson) {
6095
6324
  output.output({
6096
6325
  success: true,
@@ -6219,7 +6448,7 @@ Scanning ${scanPath}...
6219
6448
  // src/commands/skills.ts
6220
6449
  import { Command as Command13 } from "commander";
6221
6450
  import chalk17 from "chalk";
6222
- import { existsSync as existsSync11, mkdirSync as mkdirSync3, writeFileSync as writeFileSync5 } from "fs";
6451
+ import { existsSync as existsSync11, mkdirSync as mkdirSync4, writeFileSync as writeFileSync5 } from "fs";
6223
6452
  import { join as join6 } from "path";
6224
6453
  var SKILL_FRONTMATTER = `---
6225
6454
  name: mutagent-cli
@@ -6234,6 +6463,22 @@ var SKILL_BODY = `# MutagenT CLI Skill
6234
6463
 
6235
6464
  ## Quick Reference
6236
6465
  Run \`mutagent --help\` for full command list and examples.
6466
+ Run \`mutagent <command> --help\` for on-the-fly instructions before using any command.
6467
+
6468
+ Key commands:
6469
+ - \`mutagent auth login\` — Authenticate
6470
+ - \`mutagent prompts list\` — List prompts
6471
+ - \`mutagent usage\` — Check plan limits and remaining optimization runs
6472
+ - \`mutagent explore\` — Discover prompts in codebase
6473
+ - \`mutagent integrate <framework>\` — Framework integration instructions
6474
+
6475
+ ## Agent Usage Guidelines
6476
+
6477
+ - **ALWAYS** run \`mutagent prompts list --json\` and \`mutagent usage --json\` BEFORE creating or modifying resources
6478
+ - **ALWAYS** use the \`--json\` flag for machine-readable output
6479
+ - **ALWAYS** confirm with the user before write operations (create, update, delete, optimize)
6480
+ - **CHECK** \`mutagent usage\` before running \`optimize start\` to verify remaining optimization runs
6481
+ - Format dashboard links as markdown: \`[View on Dashboard](url)\`
6237
6482
 
6238
6483
  ## Post-Onboarding Decision Tree
6239
6484
 
@@ -6252,8 +6497,9 @@ After \`mutagent auth login\`, the user lands in one of 3 paths:
6252
6497
  3. \`mutagent prompts create --data '{...}'\` — Upload with REQUIRED outputSchema
6253
6498
  4. Guided eval creator OR \`mutagent prompts evaluation create\`
6254
6499
  5. \`mutagent prompts dataset add\` — Upload/curate datasets (named)
6255
- 6. \`mutagent prompts optimize start\` — Run optimization
6256
- 7. Review scorecard Apply/Reject optimized prompt
6500
+ 6. \`mutagent usage --json\` — Verify remaining optimization runs
6501
+ 7. \`mutagent prompts optimize start\` Run optimization
6502
+ 8. Review scorecard → Apply/Reject optimized prompt
6257
6503
 
6258
6504
  ### Path C: Manual
6259
6505
  User uses CLI commands directly. Run \`mutagent --help\`.
@@ -6327,7 +6573,7 @@ that teaches coding agents how to use the MutagenT CLI effectively.
6327
6573
  const skillDir = join6(process.cwd(), SKILL_DIR);
6328
6574
  const skillPath = join6(skillDir, SKILL_FILE);
6329
6575
  if (!existsSync11(skillDir)) {
6330
- mkdirSync3(skillDir, { recursive: true });
6576
+ mkdirSync4(skillDir, { recursive: true });
6331
6577
  }
6332
6578
  const content = `${SKILL_FRONTMATTER}
6333
6579
 
@@ -6351,8 +6597,108 @@ ${SKILL_BODY}
6351
6597
  return skills;
6352
6598
  }
6353
6599
 
6600
+ // src/commands/usage.ts
6601
+ init_config();
6602
+ import { Command as Command14 } from "commander";
6603
+ import chalk18 from "chalk";
6604
+ init_errors();
6605
+ init_sdk_client();
6606
+ var TRIAL_OPTIMIZATION_LIMIT = 5;
6607
+ var BILLING_URL = "https://app.mutagent.io/settings/billing";
6608
+ function renderProgressBar(used, limit, width = 30) {
6609
+ const ratio = Math.min(used / limit, 1);
6610
+ const filled = Math.round(ratio * width);
6611
+ const empty = width - filled;
6612
+ const bar = "█".repeat(filled) + "░".repeat(empty);
6613
+ const percent = Math.round(ratio * 100);
6614
+ return `${bar} ${String(percent)}%`;
6615
+ }
6616
+ function createUsageCommand() {
6617
+ const usage = new Command14("usage").description("Show resource counts and optimization run limits").addHelpText("after", `
6618
+ Examples:
6619
+ ${chalk18.dim("$")} mutagent usage
6620
+ ${chalk18.dim("$")} mutagent usage --json
6621
+ `);
6622
+ usage.action(async () => {
6623
+ const isJson = getJsonFlag(usage);
6624
+ const output = new OutputFormatter(isJson ? "json" : "table");
6625
+ try {
6626
+ const apiKey = getApiKey();
6627
+ if (!apiKey) {
6628
+ throw new AuthenticationError;
6629
+ }
6630
+ const spinner = await createSpinner("Fetching usage data...", isJson).start();
6631
+ const client = getSDKClient();
6632
+ const prompts = await client.listPrompts();
6633
+ const promptCount = prompts.length;
6634
+ let datasetCount = 0;
6635
+ let evaluationCount = 0;
6636
+ if (promptCount > 0) {
6637
+ const results = await Promise.all(prompts.map(async (prompt) => {
6638
+ const id = String(prompt.id);
6639
+ const [datasets, evaluations] = await Promise.all([
6640
+ client.listDatasets(id).catch(() => []),
6641
+ client.listEvaluations(id).catch(() => [])
6642
+ ]);
6643
+ return { datasets: datasets.length, evaluations: evaluations.length };
6644
+ }));
6645
+ for (const r of results) {
6646
+ datasetCount += r.datasets;
6647
+ evaluationCount += r.evaluations;
6648
+ }
6649
+ }
6650
+ const optimizationLimit = TRIAL_OPTIMIZATION_LIMIT;
6651
+ const optimizationUsed = 0;
6652
+ const optimizationRemaining = optimizationLimit - optimizationUsed;
6653
+ if (spinner && typeof spinner.stop === "function") {
6654
+ spinner.stop();
6655
+ }
6656
+ if (isJson) {
6657
+ output.output({
6658
+ resources: {
6659
+ prompts: promptCount,
6660
+ datasets: datasetCount,
6661
+ evaluations: evaluationCount
6662
+ },
6663
+ optimizationRuns: {
6664
+ used: optimizationUsed,
6665
+ limit: optimizationLimit,
6666
+ remaining: optimizationRemaining,
6667
+ plan: "trial"
6668
+ },
6669
+ _links: {
6670
+ billing: BILLING_URL
6671
+ }
6672
+ });
6673
+ } else {
6674
+ console.log("");
6675
+ console.log(chalk18.bold("\uD83D\uDCCA MutagenT Usage"));
6676
+ console.log(chalk18.dim("─".repeat(45)));
6677
+ console.log("");
6678
+ console.log(chalk18.bold("Resources:"));
6679
+ console.log(` Prompts: ${chalk18.cyan(String(promptCount))}`);
6680
+ console.log(` Datasets: ${chalk18.cyan(String(datasetCount))}`);
6681
+ console.log(` Evaluations: ${chalk18.cyan(String(evaluationCount))}`);
6682
+ console.log("");
6683
+ console.log(chalk18.bold(`Optimization Runs (${chalk18.yellow("trial")} plan):`));
6684
+ console.log(` Remaining: ${chalk18.cyan(String(optimizationRemaining))} / ${String(optimizationLimit)}`);
6685
+ console.log(` ${renderProgressBar(optimizationUsed, optimizationLimit)}`);
6686
+ console.log("");
6687
+ console.log(chalk18.yellow(` ⚠ ${String(optimizationRemaining)} optimization runs remaining`));
6688
+ console.log(chalk18.dim(` ℹ Optimization run counts are approximate`));
6689
+ console.log(` Upgrade: ${chalk18.underline(BILLING_URL)}`);
6690
+ console.log("");
6691
+ }
6692
+ } catch (error) {
6693
+ handleError(error, isJson);
6694
+ }
6695
+ });
6696
+ return usage;
6697
+ }
6698
+
6354
6699
  // src/bin/cli.ts
6355
6700
  init_config();
6701
+ import { existsSync as existsSync12 } from "fs";
6356
6702
  var cliVersion = "0.1.1";
6357
6703
  if (process.env.CLI_VERSION) {
6358
6704
  cliVersion = process.env.CLI_VERSION;
@@ -6364,7 +6710,7 @@ if (process.env.CLI_VERSION) {
6364
6710
  cliVersion = pkg.version ?? cliVersion;
6365
6711
  } catch {}
6366
6712
  }
6367
- var program = new Command14;
6713
+ var program = new Command15;
6368
6714
  program.name("mutagent").description(`MutagenT CLI - AI-native prompt optimization platform
6369
6715
 
6370
6716
  Documentation: https://docs.mutagent.io/cli
@@ -6373,45 +6719,47 @@ program.name("mutagent").description(`MutagenT CLI - AI-native prompt optimizati
6373
6719
  showGlobalOptions: true
6374
6720
  });
6375
6721
  program.addHelpText("after", `
6376
- ${chalk18.yellow("Non-Interactive Mode (CI/CD & Coding Agents):")}
6377
- Export your API key: ${chalk18.green("export MUTAGENT_API_KEY=mt_your_key_here")}
6378
- Or pass inline: ${chalk18.green("mutagent prompts list --api-key mt_your_key")}
6379
- Machine-readable output: ${chalk18.green("mutagent prompts list --json")}
6380
- Disable prompts: ${chalk18.green("mutagent prompts list --non-interactive")}
6381
- Set default workspace: ${chalk18.green("mutagent config set workspace <workspace-id>")}
6382
- Set default org: ${chalk18.green("mutagent config set org <org-id>")}
6722
+ ${chalk19.yellow("Non-Interactive Mode (CI/CD & Coding Agents):")}
6723
+ Export your API key: ${chalk19.green("export MUTAGENT_API_KEY=mt_your_key_here")}
6724
+ Or pass inline: ${chalk19.green("mutagent prompts list --api-key mt_your_key")}
6725
+ Machine-readable output: ${chalk19.green("mutagent prompts list --json")}
6726
+ Disable prompts: ${chalk19.green("mutagent prompts list --non-interactive")}
6727
+ Set default workspace: ${chalk19.green("mutagent config set workspace <workspace-id>")}
6728
+ Set default org: ${chalk19.green("mutagent config set org <org-id>")}
6383
6729
  `);
6384
6730
  program.addHelpText("after", `
6385
- ${chalk18.yellow("Workflows:")}
6386
- ${chalk18.bold("Evaluate → Optimize Loop:")}
6731
+ ${chalk19.yellow("Workflows:")}
6732
+ ${chalk19.bold("Evaluate → Optimize Loop:")}
6387
6733
  1. mutagent prompts create --name "..." --raw-file prompt.txt
6388
6734
  2. mutagent prompts dataset add <prompt-id> --name "..." --file data.json
6389
6735
  3. mutagent prompts evaluation create <prompt-id> --name "..." --file criteria.json
6390
6736
  4. mutagent prompts optimize start <prompt-id> --dataset <id> --max-iterations 3
6391
6737
 
6392
- ${chalk18.bold("Quick Test:")}
6738
+ ${chalk19.bold("Quick Test:")}
6393
6739
  mutagent playground run <prompt-id> --input '{"key":"value"}'
6394
6740
 
6395
- ${chalk18.bold("Prerequisites for Optimization:")}
6396
- ${chalk18.green("✓")} Prompt with input/output parameters
6397
- ${chalk18.green("✓")} Dataset with items (input + expectedOutput pairs)
6398
- ${chalk18.green("✓")} Evaluation with criteria (field-level, input/output focused)
6399
- ${chalk18.dim("•")} LLM provider ${chalk18.dim("(only when server uses external providers)")}
6741
+ ${chalk19.bold("Prerequisites for Optimization:")}
6742
+ ${chalk19.green("✓")} Prompt with input/output parameters
6743
+ ${chalk19.green("✓")} Dataset with items (input + expectedOutput pairs)
6744
+ ${chalk19.green("✓")} Evaluation with criteria (field-level, input/output focused)
6745
+ ${chalk19.dim("•")} LLM provider ${chalk19.dim("(only when server uses external providers)")}
6400
6746
  `);
6401
6747
  program.addHelpText("after", `
6402
- ${chalk18.cyan("┌─ AI AGENT INTEGRATION HINT ────────────────────────────────────────────────┐")}
6403
- ${chalk18.cyan("│")} ${chalk18.cyan("│")}
6404
- ${chalk18.cyan("│")} Frameworks: langchain, langgraph, vercel-ai, openai ${chalk18.cyan("│")}
6405
- ${chalk18.cyan("│")} ${chalk18.cyan("│")}
6406
- ${chalk18.cyan("│")} Get integration guide: mutagent integrate <framework> ${chalk18.cyan("│")}
6407
- ${chalk18.cyan("│")} Verify setup: mutagent integrate <framework> --verify ${chalk18.cyan("│")}
6408
- ${chalk18.cyan("│")} Use --json for AI parsing: mutagent <command> --json ${chalk18.cyan("│")}
6409
- ${chalk18.cyan("│")} ${chalk18.cyan("│")}
6410
- ${chalk18.cyan("└────────────────────────────────────────────────────────────────────────────┘")}
6411
- ${!hasCredentials() ? chalk18.yellow(`
6748
+ ${chalk19.cyan("┌─ AI AGENT INTEGRATION HINT ────────────────────────────────────────────────┐")}
6749
+ ${chalk19.cyan("│")} ${chalk19.cyan("│")}
6750
+ ${chalk19.cyan("│")} Frameworks: langchain, langgraph, vercel-ai, openai ${chalk19.cyan("│")}
6751
+ ${chalk19.cyan("│")} ${chalk19.cyan("│")}
6752
+ ${chalk19.cyan("│")} Get integration guide: mutagent integrate <framework> ${chalk19.cyan("│")}
6753
+ ${chalk19.cyan("│")} Verify setup: mutagent integrate <framework> --verify ${chalk19.cyan("│")}
6754
+ ${chalk19.cyan("│")} Use --json for AI parsing: mutagent <command> --json ${chalk19.cyan("│")}
6755
+ ${chalk19.cyan("│")} ${chalk19.cyan("│")}
6756
+ ${chalk19.cyan("└────────────────────────────────────────────────────────────────────────────┘")}
6757
+ ${!hasCredentials() ? chalk19.yellow(`
6412
6758
  Warning: Not authenticated. Run: mutagent auth login --browser
6413
- `) : ""}${!hasRcConfig() ? chalk18.green(`
6759
+ `) : ""}${!hasRcConfig() ? chalk19.green(`
6414
6760
  Get started: mutagent init
6761
+ `) : ""}${!existsSync12(join7(process.cwd(), ".claude/skills/mutagent-cli/SKILL.md")) ? chalk19.magenta(`
6762
+ Using Claude Code? Install the MutagenT skill: mutagent skills install
6415
6763
  `) : ""}`);
6416
6764
  program.hook("preAction", (thisCommand) => {
6417
6765
  const globalOpts = thisCommand.optsWithGlobals();
@@ -6438,7 +6786,8 @@ program.addCommand(createWorkspacesCommand());
6438
6786
  program.addCommand(createProvidersCommand());
6439
6787
  program.addCommand(createExploreCommand());
6440
6788
  program.addCommand(createSkillsCommand());
6789
+ program.addCommand(createUsageCommand());
6441
6790
  program.parse();
6442
6791
 
6443
- //# debugId=8F5D422F7273E03E64756E2164756E21
6792
+ //# debugId=88B4F3961CA6D43E64756E2164756E21
6444
6793
  //# sourceMappingURL=cli.js.map