@mutagent/cli 0.1.21 → 0.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cli.js +240 -282
- package/dist/bin/cli.js.map +12 -12
- package/dist/index.js +12 -5
- package/dist/index.js.map +3 -3
- package/package.json +4 -3
package/dist/bin/cli.js
CHANGED
|
@@ -21,7 +21,7 @@ var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
|
21
21
|
// src/bin/cli.ts
|
|
22
22
|
import { Command as Command15 } from "commander";
|
|
23
23
|
import chalk18 from "chalk";
|
|
24
|
-
import { readFileSync as
|
|
24
|
+
import { readFileSync as readFileSync10 } from "fs";
|
|
25
25
|
import { join as join7, dirname } from "path";
|
|
26
26
|
import { fileURLToPath } from "url";
|
|
27
27
|
|
|
@@ -515,10 +515,10 @@ class SDKClientWrapper {
|
|
|
515
515
|
});
|
|
516
516
|
let itemCount = 0;
|
|
517
517
|
if (items && items.length > 0) {
|
|
518
|
-
const mappedItems = items.map((item) => ({
|
|
518
|
+
const mappedItems = items.map((item, index) => ({
|
|
519
519
|
input: item.input ?? item,
|
|
520
520
|
expectedOutput: item.expectedOutput ?? item.expected_output ?? undefined,
|
|
521
|
-
name: item.name
|
|
521
|
+
name: item.name ?? `Item ${String(index + 1)}`,
|
|
522
522
|
userFeedback: item.userFeedback,
|
|
523
523
|
systemFeedback: item.systemFeedback,
|
|
524
524
|
labels: item.labels,
|
|
@@ -552,13 +552,19 @@ class SDKClientWrapper {
|
|
|
552
552
|
this.handleError(error);
|
|
553
553
|
}
|
|
554
554
|
}
|
|
555
|
+
async getEvaluation(evaluationId) {
|
|
556
|
+
try {
|
|
557
|
+
return await this.request(`/api/prompts/evaluations/${evaluationId}`);
|
|
558
|
+
} catch (error) {
|
|
559
|
+
this.handleError(error);
|
|
560
|
+
}
|
|
561
|
+
}
|
|
555
562
|
async createEvaluation(promptId, data) {
|
|
556
563
|
try {
|
|
557
564
|
return await this.request("/api/prompts/evaluations", {
|
|
558
565
|
method: "POST",
|
|
559
566
|
body: JSON.stringify({
|
|
560
567
|
promptId: parseInt(promptId, 10),
|
|
561
|
-
datasetId: data.datasetId,
|
|
562
568
|
name: data.name,
|
|
563
569
|
description: data.description,
|
|
564
570
|
evalConfig: data.evalConfig,
|
|
@@ -578,12 +584,13 @@ class SDKClientWrapper {
|
|
|
578
584
|
this.handleError(error);
|
|
579
585
|
}
|
|
580
586
|
}
|
|
581
|
-
async startOptimization(promptId, datasetId, config) {
|
|
587
|
+
async startOptimization(promptId, datasetId, evaluationId, config) {
|
|
582
588
|
try {
|
|
583
589
|
return await this.request(`/api/prompt/${promptId}/optimize`, {
|
|
584
590
|
method: "POST",
|
|
585
591
|
body: JSON.stringify({
|
|
586
592
|
datasetId: parseInt(datasetId, 10),
|
|
593
|
+
evaluationId: parseInt(evaluationId, 10),
|
|
587
594
|
config: {
|
|
588
595
|
maxIterations: config?.maxIterations ?? 1,
|
|
589
596
|
targetScore: config?.targetScore ?? 0.8,
|
|
@@ -1812,16 +1819,16 @@ function runOptimizationPath(promptCount, datasetCount) {
|
|
|
1812
1819
|
console.log("");
|
|
1813
1820
|
}
|
|
1814
1821
|
if (datasetCount === 0) {
|
|
1815
|
-
console.log(` ${String(step)}. ${chalk2.green(
|
|
1822
|
+
console.log(` ${String(step)}. ${chalk2.green(`mutagent prompts dataset add <prompt-id> -d '[{"input":{...},"expectedOutput":{...}}]' --name "my-dataset"`)}`);
|
|
1816
1823
|
console.log(" Upload a dataset with input/output pairs");
|
|
1817
1824
|
console.log("");
|
|
1818
1825
|
step++;
|
|
1819
1826
|
} else {
|
|
1820
1827
|
console.log(chalk2.dim(` ${chalk2.green("✓")} Datasets detected in codebase (${String(datasetCount)} found)`));
|
|
1821
|
-
console.log(chalk2.dim(` Upload one: mutagent prompts dataset add <prompt-id> --
|
|
1828
|
+
console.log(chalk2.dim(` Upload one: mutagent prompts dataset add <prompt-id> -d '[...]' --name "my-dataset"`));
|
|
1822
1829
|
console.log("");
|
|
1823
1830
|
}
|
|
1824
|
-
console.log(` ${String(step)}. ${chalk2.green(
|
|
1831
|
+
console.log(` ${String(step)}. ${chalk2.green(`mutagent prompts evaluation create <prompt-id> --name "My Eval" -d '{"evalConfig":{"criteria":[...]}}'`)}`);
|
|
1825
1832
|
console.log(" Define evaluation criteria");
|
|
1826
1833
|
console.log("");
|
|
1827
1834
|
step++;
|
|
@@ -1838,15 +1845,13 @@ function createAuthCommand() {
|
|
|
1838
1845
|
Examples:
|
|
1839
1846
|
${chalk3.dim("$")} mutagent auth login
|
|
1840
1847
|
${chalk3.dim("$")} mutagent auth login --browser
|
|
1841
|
-
${chalk3.dim("$")} mutagent auth login --api-key <key>
|
|
1842
1848
|
${chalk3.dim("$")} mutagent auth status
|
|
1843
1849
|
${chalk3.dim("$")} mutagent auth logout
|
|
1844
1850
|
`);
|
|
1845
|
-
auth.command("login").description("Authenticate and store API key").option("--
|
|
1851
|
+
auth.command("login").description("Authenticate and store API key").option("--browser", "Force browser-based authentication").option("--non-interactive", "Disable interactive prompts (auto-selects browser auth)").option("--endpoint <url>", "API endpoint", "https://api.mutagent.io").addHelpText("after", `
|
|
1846
1852
|
Examples:
|
|
1847
1853
|
${chalk3.dim("$")} mutagent auth login ${chalk3.dim("# Interactive (choose method)")}
|
|
1848
1854
|
${chalk3.dim("$")} mutagent auth login --browser ${chalk3.dim("# Browser OAuth flow")}
|
|
1849
|
-
${chalk3.dim("$")} mutagent auth login --api-key mg_live_xxx ${chalk3.dim("# Direct API key (CI/CD)")}
|
|
1850
1855
|
${chalk3.dim("$")} mutagent auth login --non-interactive ${chalk3.dim("# Auto browser flow (AI agents)")}
|
|
1851
1856
|
|
|
1852
1857
|
Environment Variables:
|
|
@@ -1859,7 +1864,7 @@ Environment Variables:
|
|
|
1859
1864
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
1860
1865
|
try {
|
|
1861
1866
|
const wasFirstLogin = !hasCredentials();
|
|
1862
|
-
let apiKey =
|
|
1867
|
+
let apiKey = process.env.MUTAGENT_API_KEY;
|
|
1863
1868
|
let endpoint = process.env.MUTAGENT_ENDPOINT ?? options.endpoint;
|
|
1864
1869
|
let workspaceName;
|
|
1865
1870
|
let organizationName;
|
|
@@ -1892,7 +1897,7 @@ Environment Variables:
|
|
|
1892
1897
|
}
|
|
1893
1898
|
const isNonInteractive = options.nonInteractive === true || process.env.MUTAGENT_NON_INTERACTIVE === "true" || process.env.CI === "true" || !process.stdin.isTTY;
|
|
1894
1899
|
if (isJson && !options.browser && isNonInteractive) {
|
|
1895
|
-
throw new MutagentError("INTERACTIVE_REQUIRED", "No API key provided.
|
|
1900
|
+
throw new MutagentError("INTERACTIVE_REQUIRED", "No API key provided. Set MUTAGENT_API_KEY env var or add --browser for browser auth.", "Run: export MUTAGENT_API_KEY=<key> or mutagent auth login --browser --non-interactive");
|
|
1896
1901
|
}
|
|
1897
1902
|
if (!isNonInteractive && !hasCredentials()) {
|
|
1898
1903
|
console.log(`
|
|
@@ -1906,7 +1911,7 @@ Environment Variables:
|
|
|
1906
1911
|
output.info("Non-interactive environment detected. Using browser authentication.");
|
|
1907
1912
|
useLocalBrowserAuth = true;
|
|
1908
1913
|
}
|
|
1909
|
-
if (!useLocalBrowserAuth
|
|
1914
|
+
if (!useLocalBrowserAuth) {
|
|
1910
1915
|
const methodAnswer = await inquirer.prompt([
|
|
1911
1916
|
{
|
|
1912
1917
|
type: "list",
|
|
@@ -2093,7 +2098,7 @@ Examples:
|
|
|
2093
2098
|
statusResult.remediation = {
|
|
2094
2099
|
interactive: "mutagent auth login --browser",
|
|
2095
2100
|
nonInteractive: "export MUTAGENT_API_KEY=<your-key>",
|
|
2096
|
-
ciCd: "
|
|
2101
|
+
ciCd: "export MUTAGENT_API_KEY=<key>"
|
|
2097
2102
|
};
|
|
2098
2103
|
}
|
|
2099
2104
|
output.output(statusResult);
|
|
@@ -2114,7 +2119,7 @@ Examples:
|
|
|
2114
2119
|
console.error("Authentication required. Options:");
|
|
2115
2120
|
console.error(" Interactive: mutagent auth login --browser");
|
|
2116
2121
|
console.error(" Non-interactive: export MUTAGENT_API_KEY=<your-key>");
|
|
2117
|
-
console.error(" CI/CD:
|
|
2122
|
+
console.error(" CI/CD: export MUTAGENT_API_KEY=<key>");
|
|
2118
2123
|
}
|
|
2119
2124
|
if (hasOnboarding) {
|
|
2120
2125
|
output.success("Onboarding: Complete (.mutagentrc.json found)");
|
|
@@ -2156,17 +2161,20 @@ import inquirer2 from "inquirer";
|
|
|
2156
2161
|
import chalk4 from "chalk";
|
|
2157
2162
|
import ora2 from "ora";
|
|
2158
2163
|
function createLoginCommand() {
|
|
2159
|
-
const login = new Command2("login").description("Login to MutagenT platform").option("--
|
|
2164
|
+
const login = new Command2("login").description("Login to MutagenT platform").option("--browser", "Force browser-based authentication").option("--non-interactive", "Disable interactive prompts (auto-selects browser auth)").option("--endpoint <url>", "API endpoint", "https://api.mutagent.io").addHelpText("after", `
|
|
2160
2165
|
Examples:
|
|
2161
2166
|
${chalk4.dim("$")} mutagent login
|
|
2162
2167
|
${chalk4.dim("$")} mutagent login --browser
|
|
2163
|
-
${chalk4.dim("$")} mutagent login --api-key <key>
|
|
2164
2168
|
${chalk4.dim("$")} mutagent login --non-interactive ${chalk4.dim("# Auto browser flow (AI agents)")}
|
|
2169
|
+
|
|
2170
|
+
Environment Variables:
|
|
2171
|
+
MUTAGENT_API_KEY API key (skips interactive login)
|
|
2172
|
+
MUTAGENT_ENDPOINT Custom API endpoint
|
|
2165
2173
|
`).action(async (options) => {
|
|
2166
2174
|
const isJson = getJsonFlag(login);
|
|
2167
2175
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
2168
2176
|
try {
|
|
2169
|
-
const apiKey =
|
|
2177
|
+
const apiKey = process.env.MUTAGENT_API_KEY;
|
|
2170
2178
|
const endpoint = process.env.MUTAGENT_ENDPOINT ?? options.endpoint;
|
|
2171
2179
|
if (apiKey) {
|
|
2172
2180
|
output.info("Validating API key...");
|
|
@@ -2183,7 +2191,7 @@ Examples:
|
|
|
2183
2191
|
}
|
|
2184
2192
|
const isNonInteractive = options.nonInteractive === true || process.env.MUTAGENT_NON_INTERACTIVE === "true" || process.env.CI === "true" || !process.stdin.isTTY;
|
|
2185
2193
|
if (isJson && !options.browser && isNonInteractive) {
|
|
2186
|
-
throw new MutagentError("INTERACTIVE_REQUIRED", "No API key provided.
|
|
2194
|
+
throw new MutagentError("INTERACTIVE_REQUIRED", "No API key provided. Set MUTAGENT_API_KEY env var or add --browser for browser auth.", "Run: export MUTAGENT_API_KEY=<key> or mutagent login --browser --non-interactive");
|
|
2187
2195
|
}
|
|
2188
2196
|
if (!isNonInteractive && !hasCredentials()) {
|
|
2189
2197
|
console.log(`
|
|
@@ -2197,7 +2205,7 @@ Examples:
|
|
|
2197
2205
|
output.info("Non-interactive environment detected. Using browser authentication.");
|
|
2198
2206
|
useBrowserAuth = true;
|
|
2199
2207
|
}
|
|
2200
|
-
if (!useBrowserAuth
|
|
2208
|
+
if (!useBrowserAuth) {
|
|
2201
2209
|
const methodAnswer = await inquirer2.prompt([
|
|
2202
2210
|
{
|
|
2203
2211
|
type: "list",
|
|
@@ -2408,7 +2416,7 @@ function promptCreatedDirective(promptId, name) {
|
|
|
2408
2416
|
},
|
|
2409
2417
|
next: [
|
|
2410
2418
|
`mutagent prompts evaluation create ${String(promptId)} --guided`,
|
|
2411
|
-
`mutagent prompts dataset add ${String(promptId)} --
|
|
2419
|
+
`mutagent prompts dataset add ${String(promptId)} -d '[...]' --name "my-dataset"`
|
|
2412
2420
|
],
|
|
2413
2421
|
instruction: CARD_INSTRUCTION
|
|
2414
2422
|
};
|
|
@@ -2487,7 +2495,6 @@ function evaluationCreatedDirective(promptId, evalId, name, criteriaCount) {
|
|
|
2487
2495
|
api: `/api/prompts/${String(promptId)}/evaluations/${String(evalId)}`
|
|
2488
2496
|
},
|
|
2489
2497
|
next: [
|
|
2490
|
-
`mutagent prompts dataset add ${String(promptId)} --file data.jsonl`,
|
|
2491
2498
|
`mutagent prompts optimize start ${String(promptId)} --dataset <dataset-id>`
|
|
2492
2499
|
],
|
|
2493
2500
|
instruction: CARD_INSTRUCTION
|
|
@@ -2558,7 +2565,7 @@ function formatSchemaWarning(fieldName) {
|
|
|
2558
2565
|
" }",
|
|
2559
2566
|
" }",
|
|
2560
2567
|
"",
|
|
2561
|
-
|
|
2568
|
+
` You can also provide a schema via --data: mutagent prompts create -d '{"inputSchema":{...},"outputSchema":{...}}'`
|
|
2562
2569
|
].join(`
|
|
2563
2570
|
`);
|
|
2564
2571
|
}
|
|
@@ -2842,49 +2849,15 @@ function warnSingleBraceVariables(content, output) {
|
|
|
2842
2849
|
}
|
|
2843
2850
|
}
|
|
2844
2851
|
}
|
|
2845
|
-
function parseDatasetFile(rawContent, filePath) {
|
|
2846
|
-
const trimmed = rawContent.trim();
|
|
2847
|
-
if (filePath.endsWith(".csv")) {
|
|
2848
|
-
return rawContent;
|
|
2849
|
-
}
|
|
2850
|
-
if (trimmed.startsWith("[")) {
|
|
2851
|
-
try {
|
|
2852
|
-
const parsed = JSON.parse(trimmed);
|
|
2853
|
-
if (!Array.isArray(parsed)) {
|
|
2854
|
-
throw new MutagentError("INVALID_JSON", "Expected a JSON array in dataset file", "Dataset JSON files should contain an array of objects: [{...}, {...}]");
|
|
2855
|
-
}
|
|
2856
|
-
return trimmed;
|
|
2857
|
-
} catch (e) {
|
|
2858
|
-
if (e instanceof MutagentError)
|
|
2859
|
-
throw e;
|
|
2860
|
-
throw new MutagentError("INVALID_JSON", `Failed to parse JSON array from ${filePath}`, "Ensure the file contains valid JSON. For line-delimited JSON, use .jsonl extension or put one object per line.");
|
|
2861
|
-
}
|
|
2862
|
-
}
|
|
2863
|
-
const lines = trimmed.split(`
|
|
2864
|
-
`).filter((line) => line.trim().length > 0);
|
|
2865
|
-
const items = [];
|
|
2866
|
-
for (let i = 0;i < lines.length; i++) {
|
|
2867
|
-
const line = lines[i];
|
|
2868
|
-
if (!line)
|
|
2869
|
-
continue;
|
|
2870
|
-
const trimmedLine = line.trim();
|
|
2871
|
-
try {
|
|
2872
|
-
items.push(JSON.parse(trimmedLine));
|
|
2873
|
-
} catch {
|
|
2874
|
-
throw new MutagentError("INVALID_JSONL", `Invalid JSON on line ${String(i + 1)} of ${filePath}`, `Each line must be valid JSON. Problem line: "${trimmedLine.substring(0, 80)}${trimmedLine.length > 80 ? "..." : ""}"`);
|
|
2875
|
-
}
|
|
2876
|
-
}
|
|
2877
|
-
return JSON.stringify(items);
|
|
2878
|
-
}
|
|
2879
2852
|
function createPromptsCommand() {
|
|
2880
2853
|
const prompts = new Command3("prompts").description("Manage prompts, datasets, evaluations, and optimizations").addHelpText("after", `
|
|
2881
2854
|
Examples:
|
|
2882
2855
|
${chalk6.dim("$")} mutagent prompts list
|
|
2883
2856
|
${chalk6.dim("$")} mutagent prompts get <prompt-id>
|
|
2884
|
-
${chalk6.dim("$")} mutagent prompts create --
|
|
2857
|
+
${chalk6.dim("$")} mutagent prompts create --name "my-prompt" --system "You are helpful" --human "{{input}}"
|
|
2885
2858
|
${chalk6.dim("$")} mutagent prompts dataset list <prompt-id>
|
|
2886
2859
|
${chalk6.dim("$")} mutagent prompts evaluation create <prompt-id> --name "My Eval"
|
|
2887
|
-
${chalk6.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id>
|
|
2860
|
+
${chalk6.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id>
|
|
2888
2861
|
|
|
2889
2862
|
Subcommands:
|
|
2890
2863
|
list, get, create, update, delete
|
|
@@ -2972,12 +2945,11 @@ ${chalk6.dim("Tip: Combine --with-datasets and --with-evals to fetch all nested
|
|
|
2972
2945
|
handleError(error, isJson);
|
|
2973
2946
|
}
|
|
2974
2947
|
});
|
|
2975
|
-
prompts.command("create").description("Create a new prompt").option("-d, --data <json>", "Prompt as JSON string (recommended — curl-style inline)").option("
|
|
2948
|
+
prompts.command("create").description("Create a new prompt").option("-d, --data <json>", "Prompt as JSON string (recommended — curl-style inline)").option("--raw-file <path>", "Create from plain text file (used as rawPrompt)").option("-n, --name <name>", "Prompt name").option("--description <text>", "Prompt description (shown in dashboard)").option("-c, --content <content>", "Prompt content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).option("--output-schema <json>", "Output schema as JSON string (required for optimization)").addHelpText("after", `
|
|
2976
2949
|
Examples:
|
|
2977
|
-
${chalk6.dim("$")} mutagent prompts create --name "my-prompt" --system "You are helpful" --human "{{input}}" --output-schema '{"type":"object","properties":{"result":{"type":"string","description":"The result"}}}'
|
|
2950
|
+
${chalk6.dim("$")} mutagent prompts create --name "my-prompt" --description "Greeting prompt for customers" --system "You are helpful" --human "{{input}}" --output-schema '{"type":"object","properties":{"result":{"type":"string","description":"The result"}}}'
|
|
2978
2951
|
${chalk6.dim("$")} mutagent prompts create --name "raw-prompt" --raw "Summarize: {{text}}" --output-schema '{"type":"object","properties":{"summary":{"type":"string","description":"Summary"}}}'
|
|
2979
2952
|
${chalk6.dim("$")} mutagent prompts create -d '{"name":"summarizer","systemPrompt":"Summarize","humanPrompt":"{{text}}","outputSchema":{"type":"object","properties":{"summary":{"type":"string","description":"Summary"}}}}'
|
|
2980
|
-
${chalk6.dim("$")} mutagent prompts create --file prompt.json ${chalk6.dim("# full prompt object as JSON file")}
|
|
2981
2953
|
|
|
2982
2954
|
Prompt Input Methods (pick one, priority order):
|
|
2983
2955
|
--system/--human Structured system + user message pair ${chalk6.green("(recommended)")}
|
|
@@ -2985,21 +2957,16 @@ Prompt Input Methods (pick one, priority order):
|
|
|
2985
2957
|
-d, --data Inline JSON object (CI/scripts/agents)
|
|
2986
2958
|
--messages Full messages array as JSON
|
|
2987
2959
|
--raw-file Load plain text file as raw prompt
|
|
2988
|
-
--file Load from JSON file (full prompt object)
|
|
2989
2960
|
|
|
2990
2961
|
Expected JSON (--data):
|
|
2991
2962
|
${chalk6.dim(`'{"name":"my-prompt","systemPrompt":"You are...","humanPrompt":"{{input}}","outputSchema":{"type":"object","properties":{"result":{"type":"string","description":"The result"}}},"inputSchema":{"type":"object","properties":{"input":{"type":"string","description":"User input"}}}}'`)}
|
|
2992
2963
|
|
|
2993
|
-
${chalk6.
|
|
2994
|
-
${chalk6.red("outputSchema is required.")} ${chalk6.dim("--data and --file are mutually exclusive.")}
|
|
2964
|
+
${chalk6.red("outputSchema is required.")}
|
|
2995
2965
|
`).action(async (options) => {
|
|
2996
2966
|
const isJson = getJsonFlag(prompts);
|
|
2997
2967
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
2998
2968
|
try {
|
|
2999
2969
|
let data;
|
|
3000
|
-
if (options.data && options.file) {
|
|
3001
|
-
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --data and --file together", "Use --data for inline JSON or --file for file-based input, not both");
|
|
3002
|
-
}
|
|
3003
2970
|
if (options.data) {
|
|
3004
2971
|
try {
|
|
3005
2972
|
data = JSON.parse(options.data);
|
|
@@ -3008,9 +2975,8 @@ ${chalk6.red("outputSchema is required.")} ${chalk6.dim("--data and --file are m
|
|
|
3008
2975
|
}
|
|
3009
2976
|
if (options.name)
|
|
3010
2977
|
data.name = options.name;
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
data = JSON.parse(content);
|
|
2978
|
+
if (options.description)
|
|
2979
|
+
data.description = options.description;
|
|
3014
2980
|
} else if (options.rawFile) {
|
|
3015
2981
|
if (!existsSync4(options.rawFile)) {
|
|
3016
2982
|
throw new MutagentError("FILE_NOT_FOUND", `File not found: ${options.rawFile}`, "Check the file path and try again");
|
|
@@ -3018,10 +2984,11 @@ ${chalk6.red("outputSchema is required.")} ${chalk6.dim("--data and --file are m
|
|
|
3018
2984
|
const textContent = readFileSync4(options.rawFile, "utf-8");
|
|
3019
2985
|
data = {
|
|
3020
2986
|
name: options.name ?? options.rawFile.replace(/^.*[\\/]/, "").replace(/\.[^.]+$/, ""),
|
|
2987
|
+
description: options.description,
|
|
3021
2988
|
rawPrompt: textContent
|
|
3022
2989
|
};
|
|
3023
2990
|
} else if (options.name) {
|
|
3024
|
-
data = { name: options.name };
|
|
2991
|
+
data = { name: options.name, description: options.description };
|
|
3025
2992
|
if (options.messages) {
|
|
3026
2993
|
try {
|
|
3027
2994
|
const messages = JSON.parse(options.messages);
|
|
@@ -3044,10 +3011,10 @@ ${chalk6.red("outputSchema is required.")} ${chalk6.dim("--data and --file are m
|
|
|
3044
3011
|
} else if (options.content) {
|
|
3045
3012
|
data.rawPrompt = options.content;
|
|
3046
3013
|
} else {
|
|
3047
|
-
throw new MutagentError("MISSING_ARGUMENTS", "Prompt content required", "Use --data, --raw, --raw
|
|
3014
|
+
throw new MutagentError("MISSING_ARGUMENTS", "Prompt content required", "Use --data, --raw-file, --raw, --system/--human, or --messages to specify prompt content");
|
|
3048
3015
|
}
|
|
3049
3016
|
} else {
|
|
3050
|
-
throw new MutagentError("MISSING_ARGUMENTS", "Either --data, --
|
|
3017
|
+
throw new MutagentError("MISSING_ARGUMENTS", "Either --data, --raw-file, or --name with prompt content is required", "Use --data for inline JSON, --raw-file for plain text, or provide --name with --raw, --system/--human, or --messages");
|
|
3051
3018
|
}
|
|
3052
3019
|
const promptContent = data.rawPrompt ?? data.systemPrompt ?? data.humanPrompt ?? "";
|
|
3053
3020
|
if (promptContent && !isJson) {
|
|
@@ -3061,10 +3028,10 @@ ${chalk6.red("outputSchema is required.")} ${chalk6.dim("--data and --file are m
|
|
|
3061
3028
|
}
|
|
3062
3029
|
}
|
|
3063
3030
|
if (isSchemaEmpty(data.outputSchema)) {
|
|
3064
|
-
throw new MutagentError("MISSING_ARGUMENTS", "outputSchema is required for prompt creation", `Use --output-schema '{"type":"object","properties":{...}}' or include outputSchema in --data
|
|
3031
|
+
throw new MutagentError("MISSING_ARGUMENTS", "outputSchema is required for prompt creation", `Use --output-schema '{"type":"object","properties":{...}}' or include outputSchema in --data`);
|
|
3065
3032
|
}
|
|
3066
3033
|
if (isSchemaEmpty(data.inputSchema)) {
|
|
3067
|
-
throw new MutagentError("VALIDATION_ERROR", "inputSchema is required. Optimization cannot run without defined input variables.", `Provide inputSchema via --data
|
|
3034
|
+
throw new MutagentError("VALIDATION_ERROR", "inputSchema is required. Optimization cannot run without defined input variables.", `Provide inputSchema via --data. Example:
|
|
3068
3035
|
` + ` --data '{"inputSchema":{"type":"object","properties":{"query":{"type":"string","description":"User query"}}}}'`);
|
|
3069
3036
|
} else if (!isValidJsonSchema(data.inputSchema)) {
|
|
3070
3037
|
throw new MutagentError("VALIDATION_ERROR", "inputSchema is not a valid JSON Schema object.", formatSchemaWarning("inputSchema"));
|
|
@@ -3095,37 +3062,32 @@ ${chalk6.red("outputSchema is required.")} ${chalk6.dim("--data and --file are m
|
|
|
3095
3062
|
});
|
|
3096
3063
|
console.log(hints);
|
|
3097
3064
|
}
|
|
3098
|
-
|
|
3099
|
-
|
|
3065
|
+
if (options.rawFile) {
|
|
3066
|
+
const rawFilePath = options.rawFile;
|
|
3100
3067
|
updateMutationContext((ctx) => {
|
|
3101
3068
|
const preview = (data.rawPrompt ?? data.systemPrompt ?? data.name ?? "").slice(0, 50);
|
|
3102
|
-
ctx.addDiscoveredPrompt(
|
|
3103
|
-
ctx.markPromptUploaded(
|
|
3069
|
+
ctx.addDiscoveredPrompt(rawFilePath, 1, preview);
|
|
3070
|
+
ctx.markPromptUploaded(rawFilePath, String(prompt.id), prompt.version);
|
|
3104
3071
|
});
|
|
3105
3072
|
}
|
|
3106
3073
|
} catch (error) {
|
|
3107
3074
|
handleError(error, isJson);
|
|
3108
3075
|
}
|
|
3109
3076
|
});
|
|
3110
|
-
prompts.command("update").description("Update a prompt").argument("<id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Update fields as JSON string (recommended — curl-style inline)").option("
|
|
3077
|
+
prompts.command("update").description("Update a prompt").argument("<id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Update fields as JSON string (recommended — curl-style inline)").option("--raw-file <path>", "Update from plain text file (used as rawPrompt)").option("-n, --name <name>", "New name").option("--description <text>", "New description (shown in dashboard)").option("-c, --content <content>", "New content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).option("--input-schema <json>", "Input schema as JSON string").option("--input-schema-file <path>", "Input schema from JSON file").option("--output-schema <json>", "Output schema as JSON string").option("--output-schema-file <path>", "Output schema from JSON file").addHelpText("after", `
|
|
3111
3078
|
Examples:
|
|
3112
3079
|
${chalk6.dim("$")} mutagent prompts update <id> --system "Updated system prompt" --human "{{input}}"
|
|
3113
|
-
${chalk6.dim("$")} mutagent prompts update <id> --name "new-name"
|
|
3080
|
+
${chalk6.dim("$")} mutagent prompts update <id> --name "new-name" --description "Updated description"
|
|
3114
3081
|
${chalk6.dim("$")} mutagent prompts update <id> --raw "Summarize: {{text}}"
|
|
3115
3082
|
${chalk6.dim("$")} mutagent prompts update <id> -d '{"name":"new-name","systemPrompt":"Updated prompt"}'
|
|
3116
3083
|
${chalk6.dim("$")} mutagent prompts update <id> --input-schema '{"type":"object","properties":{"text":{"type":"string","description":"Input text"}}}'
|
|
3117
|
-
${chalk6.dim("$")} mutagent prompts update <id> --file updated-prompt.json ${chalk6.dim("# full prompt object")}
|
|
3118
3084
|
|
|
3119
|
-
${chalk6.
|
|
3120
|
-
${chalk6.dim("--data and --file are mutually exclusive. CLI flags (--name) override --data fields.")}
|
|
3085
|
+
${chalk6.dim("CLI flags (--name) override --data fields.")}
|
|
3121
3086
|
`).action(async (id, options) => {
|
|
3122
3087
|
const isJson = getJsonFlag(prompts);
|
|
3123
3088
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
3124
3089
|
try {
|
|
3125
3090
|
let data = {};
|
|
3126
|
-
if (options.data && options.file) {
|
|
3127
|
-
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --data and --file together", "Use --data for inline JSON or --file for file-based input, not both");
|
|
3128
|
-
}
|
|
3129
3091
|
if (options.data) {
|
|
3130
3092
|
try {
|
|
3131
3093
|
data = JSON.parse(options.data);
|
|
@@ -3134,9 +3096,8 @@ ${chalk6.dim("--data and --file are mutually exclusive. CLI flags (--name) overr
|
|
|
3134
3096
|
}
|
|
3135
3097
|
if (options.name)
|
|
3136
3098
|
data.name = options.name;
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
data = JSON.parse(content);
|
|
3099
|
+
if (options.description)
|
|
3100
|
+
data.description = options.description;
|
|
3140
3101
|
} else if (options.rawFile) {
|
|
3141
3102
|
if (!existsSync4(options.rawFile)) {
|
|
3142
3103
|
throw new MutagentError("FILE_NOT_FOUND", `File not found: ${options.rawFile}`, "Check the file path and try again");
|
|
@@ -3145,9 +3106,13 @@ ${chalk6.dim("--data and --file are mutually exclusive. CLI flags (--name) overr
|
|
|
3145
3106
|
data.rawPrompt = textContent;
|
|
3146
3107
|
if (options.name)
|
|
3147
3108
|
data.name = options.name;
|
|
3109
|
+
if (options.description)
|
|
3110
|
+
data.description = options.description;
|
|
3148
3111
|
} else {
|
|
3149
3112
|
if (options.name)
|
|
3150
3113
|
data.name = options.name;
|
|
3114
|
+
if (options.description)
|
|
3115
|
+
data.description = options.description;
|
|
3151
3116
|
if (options.messages) {
|
|
3152
3117
|
try {
|
|
3153
3118
|
const messages = JSON.parse(options.messages);
|
|
@@ -3216,7 +3181,7 @@ ${chalk6.dim("--data and --file are mutually exclusive. CLI flags (--name) overr
|
|
|
3216
3181
|
}
|
|
3217
3182
|
}
|
|
3218
3183
|
if (Object.keys(data).length === 0) {
|
|
3219
|
-
throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --data, --
|
|
3184
|
+
throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --data, --raw-file, --name, --raw, --system/--human, --messages, --input-schema, or --output-schema");
|
|
3220
3185
|
}
|
|
3221
3186
|
const promptContent = data.rawPrompt ?? data.systemPrompt ?? data.humanPrompt ?? "";
|
|
3222
3187
|
if (promptContent && !isJson) {
|
|
@@ -3237,10 +3202,10 @@ ${chalk6.dim("--data and --file are mutually exclusive. CLI flags (--name) overr
|
|
|
3237
3202
|
});
|
|
3238
3203
|
console.log(hints);
|
|
3239
3204
|
}
|
|
3240
|
-
|
|
3241
|
-
|
|
3205
|
+
if (options.rawFile) {
|
|
3206
|
+
const rawFilePath = options.rawFile;
|
|
3242
3207
|
updateMutationContext((ctx) => {
|
|
3243
|
-
ctx.markPromptUploaded(
|
|
3208
|
+
ctx.markPromptUploaded(rawFilePath, String(prompt.id), prompt.version);
|
|
3244
3209
|
});
|
|
3245
3210
|
}
|
|
3246
3211
|
} catch (error) {
|
|
@@ -3289,7 +3254,7 @@ ${chalk6.dim("Note: --force is required. The CLI is non-interactive — confirm
|
|
|
3289
3254
|
const dataset = new Command3("dataset").description("Manage datasets for prompts").addHelpText("after", `
|
|
3290
3255
|
Examples:
|
|
3291
3256
|
${chalk6.dim("$")} mutagent prompts dataset list <prompt-id>
|
|
3292
|
-
${chalk6.dim("$")} mutagent prompts dataset add <prompt-id>
|
|
3257
|
+
${chalk6.dim("$")} mutagent prompts dataset add <prompt-id> -d '[{"input":{...},"expectedOutput":{...}}]'
|
|
3293
3258
|
${chalk6.dim("$")} mutagent prompts dataset remove <prompt-id> <dataset-id>
|
|
3294
3259
|
`).action(() => {
|
|
3295
3260
|
dataset.help();
|
|
@@ -3328,16 +3293,10 @@ Examples:
|
|
|
3328
3293
|
handleError(error, isJson);
|
|
3329
3294
|
}
|
|
3330
3295
|
});
|
|
3331
|
-
dataset.command("add").description("Add dataset to a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").option("-
|
|
3296
|
+
dataset.command("add").description("Add dataset to a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Inline JSON array of dataset items").option("-n, --name <name>", "Dataset name").addHelpText("after", `
|
|
3332
3297
|
Examples:
|
|
3333
3298
|
${chalk6.dim("$")} mutagent prompts dataset add <prompt-id> -d '[{"input":{"text":"hello"},"expectedOutput":{"result":"world"}}]'
|
|
3334
3299
|
${chalk6.dim("$")} mutagent prompts dataset add <prompt-id> -d '[{"input":{"text":"hello"},"expectedOutput":{"result":"world"}}]' --name "My Dataset"
|
|
3335
|
-
${chalk6.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.jsonl ${chalk6.dim("# also supports .json, .csv")}
|
|
3336
|
-
|
|
3337
|
-
Supported file formats:
|
|
3338
|
-
${chalk6.dim(".json")} JSON array of objects: [{"input": {...}, "expectedOutput": {...}}, ...]
|
|
3339
|
-
${chalk6.dim(".jsonl")} One JSON object per line (newline-delimited JSON)
|
|
3340
|
-
${chalk6.dim(".csv")} Comma-separated values with header row
|
|
3341
3300
|
|
|
3342
3301
|
Inline data format (-d):
|
|
3343
3302
|
JSON array of objects, e.g.:
|
|
@@ -3346,44 +3305,30 @@ Inline data format (-d):
|
|
|
3346
3305
|
Expected item format:
|
|
3347
3306
|
${chalk6.dim('{"input": {"<field>": "<value>"}, "expectedOutput": {"<field>": "<value>"}}')}
|
|
3348
3307
|
|
|
3349
|
-
${chalk6.red("Required: --data
|
|
3350
|
-
${chalk6.yellow("Note: Prefer -d/--data for inline JSON over --file to avoid stale files living in your repo.")}
|
|
3351
|
-
${chalk6.dim("--file and -d are mutually exclusive.")}
|
|
3308
|
+
${chalk6.red("Required: --data must be provided.")}
|
|
3352
3309
|
`).action(async (promptId, options) => {
|
|
3353
3310
|
const isJson = getJsonFlag(prompts);
|
|
3354
3311
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
3355
3312
|
try {
|
|
3356
|
-
if (options.
|
|
3357
|
-
throw new MutagentError("
|
|
3358
|
-
}
|
|
3359
|
-
if (!options.file && !options.data) {
|
|
3360
|
-
throw new MutagentError("MISSING_ARGUMENTS", "Either --file or -d/--data is required", "Use --file <path> to upload from a file or -d '[{...}]' for inline JSON data");
|
|
3313
|
+
if (!options.data) {
|
|
3314
|
+
throw new MutagentError("MISSING_ARGUMENTS", "-d/--data is required", "Use -d '[{...}]' to provide inline JSON data");
|
|
3361
3315
|
}
|
|
3362
3316
|
let content;
|
|
3363
|
-
|
|
3364
|
-
|
|
3365
|
-
|
|
3366
|
-
|
|
3367
|
-
throw new MutagentError("INVALID_JSON", "Inline data must be a JSON array", `Provide a JSON array, e.g., '[{"input": {...}, "expectedOutput": {...}}]'`);
|
|
3368
|
-
}
|
|
3369
|
-
content = options.data;
|
|
3370
|
-
} catch (e) {
|
|
3371
|
-
if (e instanceof MutagentError)
|
|
3372
|
-
throw e;
|
|
3373
|
-
throw new MutagentError("INVALID_JSON", "Invalid JSON in -d/--data flag", `Provide a valid JSON array, e.g., '[{"input": {...}, "expectedOutput": {...}}]'`);
|
|
3317
|
+
try {
|
|
3318
|
+
const parsed = JSON.parse(options.data);
|
|
3319
|
+
if (!Array.isArray(parsed)) {
|
|
3320
|
+
throw new MutagentError("INVALID_JSON", "Inline data must be a JSON array", `Provide a JSON array, e.g., '[{"input": {...}, "expectedOutput": {...}}]'`);
|
|
3374
3321
|
}
|
|
3375
|
-
|
|
3376
|
-
|
|
3377
|
-
|
|
3378
|
-
|
|
3379
|
-
|
|
3380
|
-
let datasetName = options.name;
|
|
3381
|
-
if (!datasetName && options.file) {
|
|
3382
|
-
datasetName = options.file.replace(/^.*[\\/]/, "").replace(/\.[^.]+$/, "");
|
|
3322
|
+
content = options.data;
|
|
3323
|
+
} catch (e) {
|
|
3324
|
+
if (e instanceof MutagentError)
|
|
3325
|
+
throw e;
|
|
3326
|
+
throw new MutagentError("INVALID_JSON", "Invalid JSON in -d/--data flag", `Provide a valid JSON array, e.g., '[{"input": {...}, "expectedOutput": {...}}]'`);
|
|
3383
3327
|
}
|
|
3328
|
+
const datasetName = options.name;
|
|
3384
3329
|
if (!datasetName) {
|
|
3385
3330
|
throw new MutagentError("MISSING_ARGUMENTS", "Dataset name is required", `[Agent: Ask the user for a dataset name via AskUserQuestion, then pass --name]
|
|
3386
|
-
` + `Use --name <name>, e.g., mutagent prompts dataset add ${promptId} --name "my-dataset"
|
|
3331
|
+
` + `Use --name <name>, e.g., mutagent prompts dataset add ${promptId} --name "my-dataset" -d '[...]'`);
|
|
3387
3332
|
}
|
|
3388
3333
|
const client = getSDKClient();
|
|
3389
3334
|
const datasetResult = await client.addDataset(promptId, content, datasetName);
|
|
@@ -3410,10 +3355,9 @@ ${chalk6.dim("--file and -d are mutually exclusive.")}
|
|
|
3410
3355
|
});
|
|
3411
3356
|
console.log(hints);
|
|
3412
3357
|
}
|
|
3413
|
-
const datasetFile = options.file ?? "inline-data";
|
|
3414
3358
|
updateMutationContext((ctx) => {
|
|
3415
|
-
ctx.addDiscoveredDataset(
|
|
3416
|
-
ctx.markDatasetUploaded(
|
|
3359
|
+
ctx.addDiscoveredDataset("inline-data", datasetResult.name, datasetResult.itemCount ?? 0);
|
|
3360
|
+
ctx.markDatasetUploaded("inline-data", String(datasetResult.id), promptId);
|
|
3417
3361
|
});
|
|
3418
3362
|
} catch (error) {
|
|
3419
3363
|
handleError(error, isJson);
|
|
@@ -3459,6 +3403,7 @@ Examples:
|
|
|
3459
3403
|
const evaluation = new Command3("evaluation").description("Manage evaluations for prompts").addHelpText("after", `
|
|
3460
3404
|
Examples:
|
|
3461
3405
|
${chalk6.dim("$")} mutagent prompts evaluation list <prompt-id>
|
|
3406
|
+
${chalk6.dim("$")} mutagent prompts evaluation get <evaluation-id>
|
|
3462
3407
|
${chalk6.dim("$")} mutagent prompts evaluation create <prompt-id> --name "My Eval"
|
|
3463
3408
|
${chalk6.dim("$")} mutagent prompts evaluation delete <evaluation-id>
|
|
3464
3409
|
`).action(() => {
|
|
@@ -3499,7 +3444,63 @@ Examples:
|
|
|
3499
3444
|
handleError(error, isJson);
|
|
3500
3445
|
}
|
|
3501
3446
|
});
|
|
3502
|
-
evaluation.command("
|
|
3447
|
+
evaluation.command("get").description("Get evaluation details including criteria").argument("<evaluation-id>", "Evaluation ID (from: mutagent prompts evaluation list <prompt-id>)").addHelpText("after", `
|
|
3448
|
+
Examples:
|
|
3449
|
+
${chalk6.dim("$")} mutagent prompts evaluation get <evaluation-id>
|
|
3450
|
+
${chalk6.dim("$")} mutagent prompts evaluation get <evaluation-id> --json
|
|
3451
|
+
`).action(async (evaluationId) => {
|
|
3452
|
+
const isJson = getJsonFlag(prompts);
|
|
3453
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
3454
|
+
try {
|
|
3455
|
+
const client = getSDKClient();
|
|
3456
|
+
const evalObj = await client.getEvaluation(evaluationId);
|
|
3457
|
+
if (isJson) {
|
|
3458
|
+
output.output({
|
|
3459
|
+
...evalObj,
|
|
3460
|
+
_links: evaluationLinks(evalObj.promptGroupId, evalObj.id)
|
|
3461
|
+
});
|
|
3462
|
+
} else {
|
|
3463
|
+
output.success(`Evaluation: ${evalObj.name} (ID: ${String(evalObj.id)})`);
|
|
3464
|
+
if (evalObj.description) {
|
|
3465
|
+
console.log(` Description: ${evalObj.description}`);
|
|
3466
|
+
}
|
|
3467
|
+
if (evalObj.datasetId != null) {
|
|
3468
|
+
console.log(` Dataset ID: ${String(evalObj.datasetId)}`);
|
|
3469
|
+
}
|
|
3470
|
+
console.log(` Created: ${evalObj.createdAt ? new Date(String(evalObj.createdAt)).toLocaleDateString() : "N/A"}`);
|
|
3471
|
+
console.log("");
|
|
3472
|
+
const config = evalObj.evalConfig;
|
|
3473
|
+
const criteria = config?.criteria;
|
|
3474
|
+
if (criteria && Array.isArray(criteria) && criteria.length > 0) {
|
|
3475
|
+
console.log(" Criteria:");
|
|
3476
|
+
for (const c of criteria) {
|
|
3477
|
+
console.log(` ${chalk6.cyan(c.name)}`);
|
|
3478
|
+
if (c.description) {
|
|
3479
|
+
console.log(` Description: ${c.description}`);
|
|
3480
|
+
}
|
|
3481
|
+
if (c.evaluationParameter) {
|
|
3482
|
+
console.log(` Parameter: ${c.evaluationParameter}`);
|
|
3483
|
+
}
|
|
3484
|
+
console.log("");
|
|
3485
|
+
}
|
|
3486
|
+
} else {
|
|
3487
|
+
console.log(" No criteria defined.");
|
|
3488
|
+
}
|
|
3489
|
+
const llmConfig = evalObj.llmConfig;
|
|
3490
|
+
if (llmConfig && Object.keys(llmConfig).length > 0) {
|
|
3491
|
+
console.log(" LLM Config:");
|
|
3492
|
+
for (const [key, value] of Object.entries(llmConfig)) {
|
|
3493
|
+
console.log(` ${key}: ${String(value)}`);
|
|
3494
|
+
}
|
|
3495
|
+
console.log("");
|
|
3496
|
+
}
|
|
3497
|
+
output.info(`View in dashboard: ${evaluationLink(evalObj.promptGroupId, evalObj.id)}`);
|
|
3498
|
+
}
|
|
3499
|
+
} catch (error) {
|
|
3500
|
+
handleError(error, isJson);
|
|
3501
|
+
}
|
|
3502
|
+
});
|
|
3503
|
+
evaluation.command("create").description("Create an evaluation configuration for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Evaluation as JSON string (for pre-validated criteria only)").option("-n, --name <name>", "Evaluation name (required unless --guided)").option("--description <text>", "Evaluation description").option("--guided", "Show evaluation workflow guide with prompt schema fields and criteria template (recommended)").addHelpText("after", `
|
|
3503
3504
|
Examples:
|
|
3504
3505
|
${chalk6.dim("$")} mutagent prompts evaluation create <prompt-id> --guided ${chalk6.dim("# recommended: shows workflow guide + schema fields")}
|
|
3505
3506
|
${chalk6.dim("$")} mutagent prompts evaluation create <prompt-id> --guided --json ${chalk6.dim("# structured workflow for AI agents")}
|
|
@@ -3518,16 +3519,14 @@ AI Agent Directive:
|
|
|
3518
3519
|
Before creating an eval, CHECK if criteria already exist in the user's code:
|
|
3519
3520
|
- If criteria match expected shape (name + description + evaluationParameter targeting schema fields) → use --data
|
|
3520
3521
|
- If criteria are missing or malformed → use --guided
|
|
3521
|
-
- NEVER use --file unless the user explicitly provides a file path
|
|
3522
3522
|
|
|
3523
3523
|
Expected Criteria Shape (--data):
|
|
3524
3524
|
${chalk6.dim('{"evalConfig":{"criteria":[{"name":"<name>","description":"<scoring rubric>","evaluationParameter":"<schema field>"}]}}')}
|
|
3525
3525
|
evaluationParameter must target an outputSchema OR inputSchema field.
|
|
3526
3526
|
|
|
3527
|
-
${chalk6.yellow("Note: Prefer --guided or --data. Avoid --file unless you have a pre-existing criteria file.")}
|
|
3528
3527
|
${chalk6.red("Required: --name (unless --guided). Criteria must include evaluationParameter.")}
|
|
3529
|
-
${chalk6.dim("
|
|
3530
|
-
${chalk6.dim("Get prompt IDs: mutagent prompts list
|
|
3528
|
+
${chalk6.dim("CLI flags (--name, --description) override --data fields.")}
|
|
3529
|
+
${chalk6.dim("Get prompt IDs: mutagent prompts list")}
|
|
3531
3530
|
`).action(async (promptId, options) => {
|
|
3532
3531
|
const isJson = getJsonFlag(prompts);
|
|
3533
3532
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3537,6 +3536,29 @@ ${chalk6.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent
|
|
|
3537
3536
|
const prompt = await client2.getPrompt(promptId);
|
|
3538
3537
|
const inputFields = prompt.inputSchema && typeof prompt.inputSchema === "object" ? Object.keys(prompt.inputSchema.properties ?? {}) : [];
|
|
3539
3538
|
const outputFields = prompt.outputSchema && typeof prompt.outputSchema === "object" ? Object.keys(prompt.outputSchema.properties ?? {}) : [];
|
|
3539
|
+
const allFields = [
|
|
3540
|
+
...outputFields.map((f) => ({ field: f, source: "outputSchema" })),
|
|
3541
|
+
...inputFields.map((f) => ({ field: f, source: "inputSchema" }))
|
|
3542
|
+
];
|
|
3543
|
+
const askUserQuestions = allFields.map(({ field, source }) => ({
|
|
3544
|
+
question: `How should "${field}" (${source}) be evaluated? Describe the scoring rubric — what does 1.0 (perfect) vs 0.0 (fail) mean for this field?`,
|
|
3545
|
+
header: field,
|
|
3546
|
+
options: [
|
|
3547
|
+
{
|
|
3548
|
+
label: "Exact Match",
|
|
3549
|
+
description: `Score 1.0 if the "${field}" output exactly matches the expected value. Score 0.5 for same meaning but different format. Score 0.0 if substantively different.`
|
|
3550
|
+
},
|
|
3551
|
+
{
|
|
3552
|
+
label: "Semantic Similarity",
|
|
3553
|
+
description: `Score 0.0-1.0 based on how closely the "${field}" output matches the expected meaning. 1.0 = identical meaning, 0.5 = partially relevant, 0.0 = unrelated or contradictory.`
|
|
3554
|
+
},
|
|
3555
|
+
{
|
|
3556
|
+
label: "Custom rubric",
|
|
3557
|
+
description: `Write a detailed custom scoring rubric for "${field}" — explain what makes a 1.0 vs 0.0 score.`
|
|
3558
|
+
}
|
|
3559
|
+
],
|
|
3560
|
+
multiSelect: false
|
|
3561
|
+
}));
|
|
3540
3562
|
const workflow = {
|
|
3541
3563
|
prompt: { id: promptId, name: prompt.name },
|
|
3542
3564
|
inputSchema: { fields: inputFields },
|
|
@@ -3551,18 +3573,23 @@ ${chalk6.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent
|
|
|
3551
3573
|
},
|
|
3552
3574
|
{
|
|
3553
3575
|
step: 2,
|
|
3554
|
-
action: "
|
|
3555
|
-
detail: "
|
|
3576
|
+
action: "Ask the user about each schema field using AskUserQuestion",
|
|
3577
|
+
detail: "For EACH field listed in askUserQuestions below, present the question to the user. Collect their scoring rubric choice or custom description."
|
|
3556
3578
|
},
|
|
3557
3579
|
{
|
|
3558
3580
|
step: 3,
|
|
3559
|
-
action: "Build criteria JSON",
|
|
3560
|
-
detail:
|
|
3581
|
+
action: "Build criteria JSON from user answers",
|
|
3582
|
+
detail: `Map each user answer to a criterion: { name: "<field> <rubric>", description: "<user's rubric>", evaluationParameter: "<field>" }`
|
|
3561
3583
|
},
|
|
3562
3584
|
{
|
|
3563
3585
|
step: 4,
|
|
3586
|
+
action: "Ask for evaluation name",
|
|
3587
|
+
detail: 'Ask the user what to name this evaluation (e.g., "Accuracy Check", "Quality Eval")'
|
|
3588
|
+
},
|
|
3589
|
+
{
|
|
3590
|
+
step: 5,
|
|
3564
3591
|
action: "Create the evaluation",
|
|
3565
|
-
detail:
|
|
3592
|
+
detail: `Run: mutagent prompts evaluation create <prompt-id> --name "<name>" -d '{"evalConfig":{"criteria":[...]}}'`
|
|
3566
3593
|
}
|
|
3567
3594
|
],
|
|
3568
3595
|
criteriaTemplate: {
|
|
@@ -3571,7 +3598,7 @@ ${chalk6.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent
|
|
|
3571
3598
|
{
|
|
3572
3599
|
name: "<criterion_name>",
|
|
3573
3600
|
description: "<scoring rubric - describe what 1.0 vs 0.0 means>",
|
|
3574
|
-
evaluationParameter: `<one of: ${
|
|
3601
|
+
evaluationParameter: `<one of: ${allFields.length > 0 ? allFields.map((f) => f.field).join(", ") : "no fields detected - check prompt schemas"}>`
|
|
3575
3602
|
}
|
|
3576
3603
|
]
|
|
3577
3604
|
}
|
|
@@ -3586,9 +3613,15 @@ ${chalk6.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent
|
|
|
3586
3613
|
}
|
|
3587
3614
|
})}'`
|
|
3588
3615
|
},
|
|
3616
|
+
askUserQuestions,
|
|
3589
3617
|
_directive: {
|
|
3590
3618
|
display: "workflow_guide",
|
|
3591
|
-
instruction: "STOP.
|
|
3619
|
+
instruction: "STOP. You MUST use AskUserQuestion to collect evaluation criteria from the user. For EACH field in askUserQuestions, present the question. Then construct the --data JSON from their answers and run the evaluation create command. Do NOT skip fields or auto-generate criteria without user input.",
|
|
3620
|
+
aiAgentDecisionTree: {
|
|
3621
|
+
step1: "Check if criteria already exist in the user's code. If criteria match expected shape (name + description + evaluationParameter targeting schema fields), use --data directly.",
|
|
3622
|
+
step2: "If criteria are missing or malformed, use the askUserQuestions payloads to collect them via AskUserQuestion.",
|
|
3623
|
+
step3: `After collecting all criteria, construct the --data JSON and run: mutagent prompts evaluation create <prompt-id> --name "<name>" -d '<json>'`
|
|
3624
|
+
}
|
|
3592
3625
|
}
|
|
3593
3626
|
};
|
|
3594
3627
|
if (isJson) {
|
|
@@ -3603,20 +3636,20 @@ ${chalk6.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent
|
|
|
3603
3636
|
console.log(` Input Schema Fields: ${inputFields.join(", ")}`);
|
|
3604
3637
|
}
|
|
3605
3638
|
console.log("");
|
|
3606
|
-
console.log("
|
|
3607
|
-
|
|
3608
|
-
|
|
3609
|
-
|
|
3610
|
-
|
|
3639
|
+
console.log(" For each field, define a scoring criterion:");
|
|
3640
|
+
for (const { field, source } of allFields) {
|
|
3641
|
+
console.log(` ${chalk6.cyan(field)} (${source})`);
|
|
3642
|
+
console.log(` → What does 1.0 vs 0.0 mean for "${field}"?`);
|
|
3643
|
+
}
|
|
3611
3644
|
console.log("");
|
|
3612
|
-
console.log("
|
|
3645
|
+
console.log(" Then create the evaluation:");
|
|
3613
3646
|
console.log(` mutagent prompts evaluation create ${promptId} --name "My Eval" \\`);
|
|
3614
3647
|
console.log(` -d '{"evalConfig":{"criteria":[{"name":"accuracy","description":"Score 1.0 if correct","evaluationParameter":"${outputFields[0] ?? "<field>"}"}]}}'`);
|
|
3615
3648
|
console.log("");
|
|
3616
3649
|
}
|
|
3617
3650
|
return;
|
|
3618
3651
|
}
|
|
3619
|
-
if (!options.data
|
|
3652
|
+
if (!options.data) {
|
|
3620
3653
|
throw new MutagentError("MISSING_ARGUMENTS", "No criteria source provided", `[Agent: Run --guided first to see available schema fields, then ask the user to define criteria via AskUserQuestion]
|
|
3621
3654
|
` + `Use --guided to see the evaluation workflow and available schema fields, or provide criteria via --data.
|
|
3622
3655
|
` + "Example: mutagent prompts evaluation create " + promptId + " --guided");
|
|
@@ -3625,47 +3658,22 @@ ${chalk6.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent
|
|
|
3625
3658
|
throw new MutagentError("MISSING_ARGUMENTS", "Evaluation name is required", `[Agent: Ask the user for an evaluation name via AskUserQuestion, then pass --name]
|
|
3626
3659
|
` + "Use --name <name>, e.g., mutagent prompts evaluation create " + promptId + ` --name "My Eval" --data '...'`);
|
|
3627
3660
|
}
|
|
3628
|
-
if (options.data && options.file) {
|
|
3629
|
-
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --data and --file together", "Use --data for inline JSON or --file for file-based input, not both");
|
|
3630
|
-
}
|
|
3631
3661
|
const evalData = {
|
|
3632
3662
|
name: options.name,
|
|
3633
3663
|
description: options.description
|
|
3634
3664
|
};
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
|
|
3638
|
-
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
evalData.datasetId = parsed.datasetId;
|
|
3648
|
-
} catch {
|
|
3649
|
-
throw new MutagentError("INVALID_JSON", "Invalid JSON in --data flag", `Provide a valid JSON object, e.g., '{"evalConfig":{"criteria":[...]},"llmConfig":{"model":"gpt-4"}}'`);
|
|
3650
|
-
}
|
|
3651
|
-
} else if (options.file) {
|
|
3652
|
-
if (!existsSync4(options.file)) {
|
|
3653
|
-
throw new MutagentError("FILE_NOT_FOUND", `File not found: ${options.file}`, "Check the file path and try again");
|
|
3654
|
-
}
|
|
3655
|
-
const fileContent = readFileSync4(options.file, "utf-8");
|
|
3656
|
-
try {
|
|
3657
|
-
const parsed = JSON.parse(fileContent);
|
|
3658
|
-
if (parsed.evalConfig)
|
|
3659
|
-
evalData.evalConfig = parsed.evalConfig;
|
|
3660
|
-
if (parsed.llmConfig)
|
|
3661
|
-
evalData.llmConfig = parsed.llmConfig;
|
|
3662
|
-
if (parsed.tags)
|
|
3663
|
-
evalData.tags = parsed.tags;
|
|
3664
|
-
if (parsed.metadata)
|
|
3665
|
-
evalData.metadata = parsed.metadata;
|
|
3666
|
-
} catch {
|
|
3667
|
-
throw new MutagentError("INVALID_JSON", `Failed to parse criteria file: ${options.file}`, "Ensure the file contains valid JSON with evalConfig and/or llmConfig");
|
|
3668
|
-
}
|
|
3665
|
+
try {
|
|
3666
|
+
const parsed = JSON.parse(options.data);
|
|
3667
|
+
if (parsed.evalConfig)
|
|
3668
|
+
evalData.evalConfig = parsed.evalConfig;
|
|
3669
|
+
if (parsed.llmConfig)
|
|
3670
|
+
evalData.llmConfig = parsed.llmConfig;
|
|
3671
|
+
if (parsed.tags)
|
|
3672
|
+
evalData.tags = parsed.tags;
|
|
3673
|
+
if (parsed.metadata)
|
|
3674
|
+
evalData.metadata = parsed.metadata;
|
|
3675
|
+
} catch {
|
|
3676
|
+
throw new MutagentError("INVALID_JSON", "Invalid JSON in --data flag", `Provide a valid JSON object, e.g., '{"evalConfig":{"criteria":[...]},"llmConfig":{"model":"gpt-4"}}'`);
|
|
3669
3677
|
}
|
|
3670
3678
|
if (options.name)
|
|
3671
3679
|
evalData.name = options.name;
|
|
@@ -3689,7 +3697,7 @@ Detected output fields from prompt schema: ${fields.join(", ")}
|
|
|
3689
3697
|
}
|
|
3690
3698
|
}
|
|
3691
3699
|
} catch {}
|
|
3692
|
-
throw new MutagentError("VALIDATION_ERROR", "Evaluation criteria are required. Provide criteria via --data
|
|
3700
|
+
throw new MutagentError("VALIDATION_ERROR", "Evaluation criteria are required. Provide criteria via --data.", `Run with --guided to see available schema fields and criteria template.
|
|
3693
3701
|
` + fieldsHint + `
|
|
3694
3702
|
Example JSON (--data flag):
|
|
3695
3703
|
` + ` --data '{"evalConfig":{"criteria":[{"name":"Accuracy","description":"...","evaluationParameter":"classification"}]}}'
|
|
@@ -3701,8 +3709,7 @@ Example JSON (--data flag):
|
|
|
3701
3709
|
delete c.scoringRubric;
|
|
3702
3710
|
}
|
|
3703
3711
|
if (c.targetField && !c.evaluationParameter) {
|
|
3704
|
-
|
|
3705
|
-
c.evaluationParameter = tf.startsWith("output.") ? tf.slice("output.".length) : tf.startsWith("input.") ? tf.slice("input.".length) : tf;
|
|
3712
|
+
c.evaluationParameter = c.targetField;
|
|
3706
3713
|
delete c.targetField;
|
|
3707
3714
|
}
|
|
3708
3715
|
const hasDescription = c.description;
|
|
@@ -3743,12 +3750,6 @@ Available output fields: ${availableFields.join(", ")}` : "";
|
|
|
3743
3750
|
Example:
|
|
3744
3751
|
--data '{"evalConfig":{"criteria":[` + '{"name":"Accuracy","description":"...","evaluationParameter":"classification"},' + '{"name":"Confidence","description":"...","evaluationParameter":"confidence"}' + "]}}'");
|
|
3745
3752
|
}
|
|
3746
|
-
if (options.dataset) {
|
|
3747
|
-
evalData.datasetId = parseInt(options.dataset, 10);
|
|
3748
|
-
if (isNaN(evalData.datasetId)) {
|
|
3749
|
-
throw new MutagentError("VALIDATION_ERROR", `Invalid dataset ID: ${options.dataset}`, "Dataset ID must be a number. Get IDs with: mutagent prompts dataset list <prompt-id>");
|
|
3750
|
-
}
|
|
3751
|
-
}
|
|
3752
3753
|
const client = getSDKClient();
|
|
3753
3754
|
const evalResult = await client.createEvaluation(promptId, evalData);
|
|
3754
3755
|
if (isJson) {
|
|
@@ -3816,7 +3817,7 @@ Examples:
|
|
|
3816
3817
|
});
|
|
3817
3818
|
const optimize = new Command3("optimize").description("Manage prompt optimization jobs").addHelpText("after", `
|
|
3818
3819
|
Examples:
|
|
3819
|
-
${chalk6.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id>
|
|
3820
|
+
${chalk6.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id>
|
|
3820
3821
|
${chalk6.dim("$")} mutagent prompts optimize status <job-id>
|
|
3821
3822
|
${chalk6.dim("$")} mutagent prompts optimize results <job-id>
|
|
3822
3823
|
|
|
@@ -3825,17 +3826,14 @@ Workflow: start -> status (poll) -> results
|
|
|
3825
3826
|
optimize.help();
|
|
3826
3827
|
});
|
|
3827
3828
|
prompts.addCommand(optimize);
|
|
3828
|
-
optimize.command("start").description("Start prompt optimization").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").requiredOption("-d, --dataset <id>", "Dataset ID for optimization (from: mutagent prompts dataset list <prompt-id>)").option("--max-iterations <n>", "Max optimization iterations (default: 1)").option("--target-score <n>", "Target accuracy 0-1 (default: 0.8)").option("--patience <n>", "Iterations without improvement before stopping").option("--model <model-id>", 'Target LLM model (e.g., "claude-sonnet-4-5-20250929")').option("--eval-model <model-id>", "Evaluation model (defaults to target model)").addHelpText("after", `
|
|
3829
|
+
optimize.command("start").description("Start prompt optimization").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").requiredOption("-d, --dataset <id>", "Dataset ID for optimization (from: mutagent prompts dataset list <prompt-id>)").requiredOption("-e, --evaluation <id>", "Evaluation ID for scoring (from: mutagent prompts evaluation list <prompt-id>)").option("--max-iterations <n>", "Max optimization iterations (default: 1)").option("--target-score <n>", "Target accuracy 0-1 (default: 0.8)").option("--patience <n>", "Iterations without improvement before stopping").option("--model <model-id>", 'Target LLM model (e.g., "claude-sonnet-4-5-20250929")').option("--eval-model <model-id>", "Evaluation model (defaults to target model)").addHelpText("after", `
|
|
3829
3830
|
Examples:
|
|
3830
|
-
${chalk6.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id>
|
|
3831
|
-
${chalk6.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --max-iterations 5
|
|
3832
|
-
${chalk6.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --target-score 0.95 --model claude-sonnet-4-5-20250929
|
|
3833
|
-
${chalk6.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --json
|
|
3831
|
+
${chalk6.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id>
|
|
3832
|
+
${chalk6.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id> --max-iterations 5
|
|
3833
|
+
${chalk6.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id> --target-score 0.95 --model claude-sonnet-4-5-20250929
|
|
3834
|
+
${chalk6.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id> --json
|
|
3834
3835
|
${PREREQUISITES_TEXT}
|
|
3835
3836
|
|
|
3836
|
-
${chalk6.yellow("Important: Ensure exactly 1 evaluation exists for this prompt before optimizing.")}
|
|
3837
|
-
${chalk6.yellow("Multiple evaluations produce conflicting scores. Delete extras first.")}
|
|
3838
|
-
|
|
3839
3837
|
${chalk6.dim("Monitor progress with: mutagent prompts optimize status <job-id>")}
|
|
3840
3838
|
|
|
3841
3839
|
${chalk6.dim("AI Agent Note: After running commands, present CLI output to the user as a status report. Use --json for structured parsing.")}
|
|
@@ -3854,31 +3852,26 @@ ${chalk6.dim("AI Agent Note: After running commands, present CLI output to the u
|
|
|
3854
3852
|
output.warn("Prompt has no inputSchema. Optimization works best with defined input variables.");
|
|
3855
3853
|
}
|
|
3856
3854
|
try {
|
|
3857
|
-
const
|
|
3858
|
-
|
|
3859
|
-
|
|
3860
|
-
|
|
3861
|
-
|
|
3862
|
-
const config = e.evalConfig;
|
|
3863
|
-
const criteria = config?.criteria;
|
|
3864
|
-
if (!criteria || criteria.length === 0)
|
|
3865
|
-
return false;
|
|
3866
|
-
return criteria.some((c) => c.evaluationParameter ?? c.targetField);
|
|
3867
|
-
});
|
|
3868
|
-
if (validEvals.length === 0) {
|
|
3869
|
-
output.warn("Evaluations exist but none have evaluationParameter set. " + "Optimization works best with criteria targeting specific output fields. " + `Re-create: mutagent prompts evaluation create ${promptId} --guided`);
|
|
3870
|
-
} else if (validEvals.length > 1) {
|
|
3871
|
-
output.warn(`Found ${String(validEvals.length)} evaluations. Optimization works best with exactly 1 evaluation. ` + "Multiple evaluations may produce conflicting scoring. " + `Delete extras: mutagent prompts evaluation delete <evaluation-id> --force`);
|
|
3855
|
+
const evalObj = await client.getEvaluation(options.evaluation);
|
|
3856
|
+
const evalConfig = evalObj.evalConfig;
|
|
3857
|
+
const criteria = evalConfig?.criteria;
|
|
3858
|
+
if (!criteria || criteria.length === 0) {
|
|
3859
|
+
output.warn(`Evaluation ${options.evaluation} has no criteria defined. ` + "Optimization works best with criteria targeting specific output fields. " + `Re-create: mutagent prompts evaluation create ${promptId} --guided`);
|
|
3872
3860
|
} else {
|
|
3873
|
-
|
|
3861
|
+
const hasCriteriaWithParam = criteria.some((c) => c.evaluationParameter ?? c.targetField);
|
|
3862
|
+
if (!hasCriteriaWithParam) {
|
|
3863
|
+
output.warn(`Evaluation ${options.evaluation} has criteria but none have evaluationParameter set. ` + `Re-create: mutagent prompts evaluation create ${promptId} --guided`);
|
|
3864
|
+
} else {
|
|
3865
|
+
output.info(`Using evaluation ${options.evaluation} with ${String(criteria.length)} criteria.`);
|
|
3866
|
+
}
|
|
3874
3867
|
}
|
|
3875
3868
|
} catch (e) {
|
|
3876
3869
|
if (e instanceof MutagentError)
|
|
3877
3870
|
throw e;
|
|
3878
|
-
output.warn(
|
|
3871
|
+
output.warn(`Could not validate evaluation ${options.evaluation}. Proceeding anyway.`);
|
|
3879
3872
|
}
|
|
3880
3873
|
}
|
|
3881
|
-
const job = await client.startOptimization(promptId, options.dataset, {
|
|
3874
|
+
const job = await client.startOptimization(promptId, options.dataset, options.evaluation, {
|
|
3882
3875
|
maxIterations: options.maxIterations ? parseInt(options.maxIterations, 10) : 1,
|
|
3883
3876
|
targetScore: options.targetScore ? parseFloat(options.targetScore) : undefined,
|
|
3884
3877
|
patience: options.patience ? parseInt(options.patience, 10) : undefined,
|
|
@@ -3912,7 +3905,7 @@ ${chalk6.dim("AI Agent Note: After running commands, present CLI output to the u
|
|
|
3912
3905
|
suggestions.push("Dataset item inputs don't match the prompt's inputSchema. Verify field names and types.");
|
|
3913
3906
|
}
|
|
3914
3907
|
if (errorText.includes("has no items") || errorText.includes("Dataset") && errorText.includes("not found")) {
|
|
3915
|
-
suggestions.push(`Upload data: mutagent prompts dataset add ${promptId} --
|
|
3908
|
+
suggestions.push(`Upload data: mutagent prompts dataset add ${promptId} -d '[...]' --name "my-dataset"`);
|
|
3916
3909
|
}
|
|
3917
3910
|
if (errorText.includes("Evaluations are not present")) {
|
|
3918
3911
|
suggestions.push(`Create evaluation: mutagent prompts evaluation create ${promptId} --guided`);
|
|
@@ -4939,14 +4932,13 @@ AI AGENT: Execute these steps using Bash for install, Write/Edit for code change
|
|
|
4939
4932
|
// src/commands/agents.ts
|
|
4940
4933
|
import { Command as Command6 } from "commander";
|
|
4941
4934
|
import chalk9 from "chalk";
|
|
4942
|
-
import { readFileSync as readFileSync9 } from "fs";
|
|
4943
4935
|
function createAgentsCommand() {
|
|
4944
4936
|
const agents = new Command6("agents").description("Manage AI agents").addHelpText("after", `
|
|
4945
4937
|
Examples:
|
|
4946
4938
|
${chalk9.dim("$")} mutagent agents list
|
|
4947
4939
|
${chalk9.dim("$")} mutagent agents get <agent-id>
|
|
4948
|
-
${chalk9.dim("$")} mutagent agents create --file agent.json
|
|
4949
4940
|
${chalk9.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "You are a code reviewer..."
|
|
4941
|
+
${chalk9.dim("$")} mutagent agents create -d '{"name":"Code Reviewer","slug":"code-reviewer","systemPrompt":"You are..."}'
|
|
4950
4942
|
${chalk9.dim("$")} mutagent agents update <agent-id> --name "Updated Name"
|
|
4951
4943
|
${chalk9.dim("$")} mutagent agents delete <agent-id> --force
|
|
4952
4944
|
${chalk9.dim("$")} mutagent agents conversations list <agent-id>
|
|
@@ -5055,11 +5047,10 @@ System Prompt:`));
|
|
|
5055
5047
|
handleError(error, isJson);
|
|
5056
5048
|
}
|
|
5057
5049
|
});
|
|
5058
|
-
agents.command("create").description("Create a new agent").option("-d, --data <json>", "Agent as JSON string (recommended for CI/scripts/agents)").option("-
|
|
5050
|
+
agents.command("create").description("Create a new agent").option("-d, --data <json>", "Agent as JSON string (recommended for CI/scripts/agents)").option("-n, --name <name>", "Agent name").option("-s, --slug <slug>", "Agent slug (URL-friendly identifier)").option("-p, --system-prompt <prompt>", "System prompt").option("-m, --model <model>", "Model (claude-sonnet-4-5, claude-opus-4-5, claude-haiku-4-5)").option("--description <desc>", "Agent description").addHelpText("after", `
|
|
5059
5051
|
Examples:
|
|
5060
5052
|
${chalk9.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "You are a code reviewer..."
|
|
5061
5053
|
${chalk9.dim("$")} mutagent agents create -d '{"name":"Code Reviewer","slug":"code-reviewer","systemPrompt":"You are a code reviewer..."}'
|
|
5062
|
-
${chalk9.dim("$")} mutagent agents create --file agent.json ${chalk9.dim("# full agent object as JSON file")}
|
|
5063
5054
|
|
|
5064
5055
|
Expected JSON (--data):
|
|
5065
5056
|
${chalk9.dim('{"name":"<name>","slug":"<slug>","systemPrompt":"<system prompt>","model":"<model-id>","description":"<description>"}')}
|
|
@@ -5067,18 +5058,13 @@ Expected JSON (--data):
|
|
|
5067
5058
|
Input Methods (pick one, priority order):
|
|
5068
5059
|
--name/--slug/... Individual flags ${chalk9.green("(recommended)")}
|
|
5069
5060
|
-d, --data Inline JSON object (CI/scripts/agents)
|
|
5070
|
-
--file Load from JSON file
|
|
5071
5061
|
|
|
5072
|
-
${chalk9.
|
|
5073
|
-
${chalk9.red("Required: name, slug, systemPrompt.")} ${chalk9.dim("--data and --file are mutually exclusive. CLI flags override --data fields.")}
|
|
5062
|
+
${chalk9.red("Required: name, slug, systemPrompt.")} ${chalk9.dim("CLI flags override --data fields.")}
|
|
5074
5063
|
`).action(async (options) => {
|
|
5075
5064
|
const isJson = getJsonFlag(agents);
|
|
5076
5065
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5077
5066
|
try {
|
|
5078
5067
|
let data;
|
|
5079
|
-
if (options.data && options.file) {
|
|
5080
|
-
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --data and --file together", "Use --data for inline JSON or --file for file-based input, not both");
|
|
5081
|
-
}
|
|
5082
5068
|
if (options.data) {
|
|
5083
5069
|
try {
|
|
5084
5070
|
data = JSON.parse(options.data);
|
|
@@ -5095,9 +5081,6 @@ ${chalk9.red("Required: name, slug, systemPrompt.")} ${chalk9.dim("--data and --
|
|
|
5095
5081
|
data.model = options.model;
|
|
5096
5082
|
if (options.description)
|
|
5097
5083
|
data.description = options.description;
|
|
5098
|
-
} else if (options.file) {
|
|
5099
|
-
const content = readFileSync9(options.file, "utf-8");
|
|
5100
|
-
data = JSON.parse(content);
|
|
5101
5084
|
} else if (options.name && options.slug && options.systemPrompt) {
|
|
5102
5085
|
data = {
|
|
5103
5086
|
name: options.name,
|
|
@@ -5109,7 +5092,7 @@ ${chalk9.red("Required: name, slug, systemPrompt.")} ${chalk9.dim("--data and --
|
|
|
5109
5092
|
if (options.description)
|
|
5110
5093
|
data.description = options.description;
|
|
5111
5094
|
} else {
|
|
5112
|
-
throw new MutagentError("MISSING_ARGUMENTS", "Either --data
|
|
5095
|
+
throw new MutagentError("MISSING_ARGUMENTS", "Either --data or (--name, --slug, and --system-prompt) are required", "Use --data for inline JSON or provide --name, --slug, and --system-prompt");
|
|
5113
5096
|
}
|
|
5114
5097
|
const client = getSDKClient();
|
|
5115
5098
|
const agent = await client.createAgent(data);
|
|
@@ -5119,28 +5102,22 @@ ${chalk9.red("Required: name, slug, systemPrompt.")} ${chalk9.dim("--data and --
|
|
|
5119
5102
|
handleError(error, isJson);
|
|
5120
5103
|
}
|
|
5121
5104
|
});
|
|
5122
|
-
agents.command("update").description("Update an agent").argument("<id>", "Agent ID").option("-d, --data <json>", "Agent updates as JSON string (CI/scripts/agents)").option("-
|
|
5105
|
+
agents.command("update").description("Update an agent").argument("<id>", "Agent ID").option("-d, --data <json>", "Agent updates as JSON string (CI/scripts/agents)").option("-n, --name <name>", "New name").option("-p, --system-prompt <prompt>", "New system prompt").option("-m, --model <model>", "New model").option("--description <desc>", "New description").option("-s, --status <status>", "New status (active, paused, archived)").addHelpText("after", `
|
|
5123
5106
|
Examples:
|
|
5124
5107
|
${chalk9.dim("$")} mutagent agents update <id> --name "New Name"
|
|
5125
5108
|
${chalk9.dim("$")} mutagent agents update <id> --system-prompt "Updated prompt" --status active
|
|
5126
5109
|
${chalk9.dim("$")} mutagent agents update <id> -d '{"name":"New Name","systemPrompt":"Updated prompt"}'
|
|
5127
|
-
${chalk9.dim("$")} mutagent agents update <id> --file updated-agent.json ${chalk9.dim("# full agent object")}
|
|
5128
5110
|
|
|
5129
5111
|
Input Methods (pick one, priority order):
|
|
5130
5112
|
--name/--system-prompt/... Individual flags ${chalk9.green("(recommended)")}
|
|
5131
5113
|
-d, --data Inline JSON object (CI/scripts/agents)
|
|
5132
|
-
--file Load from JSON file
|
|
5133
5114
|
|
|
5134
|
-
${chalk9.
|
|
5135
|
-
${chalk9.dim("--data and --file are mutually exclusive. CLI flags override --data fields.")}
|
|
5115
|
+
${chalk9.dim("CLI flags override --data fields.")}
|
|
5136
5116
|
`).action(async (id, options) => {
|
|
5137
5117
|
const isJson = getJsonFlag(agents);
|
|
5138
5118
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5139
5119
|
try {
|
|
5140
5120
|
let data = {};
|
|
5141
|
-
if (options.data && options.file) {
|
|
5142
|
-
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --data and --file together", "Use --data for inline JSON or --file for file-based input, not both");
|
|
5143
|
-
}
|
|
5144
5121
|
if (options.data) {
|
|
5145
5122
|
try {
|
|
5146
5123
|
data = JSON.parse(options.data);
|
|
@@ -5157,9 +5134,6 @@ ${chalk9.dim("--data and --file are mutually exclusive. CLI flags override --dat
|
|
|
5157
5134
|
data.description = options.description;
|
|
5158
5135
|
if (options.status)
|
|
5159
5136
|
data.status = options.status;
|
|
5160
|
-
} else if (options.file) {
|
|
5161
|
-
const content = readFileSync9(options.file, "utf-8");
|
|
5162
|
-
data = JSON.parse(content);
|
|
5163
5137
|
} else {
|
|
5164
5138
|
if (options.name)
|
|
5165
5139
|
data.name = options.name;
|
|
@@ -5173,7 +5147,7 @@ ${chalk9.dim("--data and --file are mutually exclusive. CLI flags override --dat
|
|
|
5173
5147
|
data.status = options.status;
|
|
5174
5148
|
}
|
|
5175
5149
|
if (Object.keys(data).length === 0) {
|
|
5176
|
-
throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --data, --
|
|
5150
|
+
throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --data, --name, --system-prompt, --model, --description, or --status");
|
|
5177
5151
|
}
|
|
5178
5152
|
const client = getSDKClient();
|
|
5179
5153
|
const agent = await client.updateAgent(id, data);
|
|
@@ -5461,7 +5435,6 @@ ${chalk10.dim("Persists organization ID for org-scoped API keys.")}
|
|
|
5461
5435
|
// src/commands/playground.ts
|
|
5462
5436
|
import { Command as Command8 } from "commander";
|
|
5463
5437
|
import chalk11 from "chalk";
|
|
5464
|
-
import { readFileSync as readFileSync10 } from "fs";
|
|
5465
5438
|
function parseSSELine(line) {
|
|
5466
5439
|
if (!line || line.startsWith(":")) {
|
|
5467
5440
|
return null;
|
|
@@ -5488,7 +5461,6 @@ function createPlaygroundCommand() {
|
|
|
5488
5461
|
const playground = new Command8("playground").description("Execute and test prompts interactively").addHelpText("after", `
|
|
5489
5462
|
Examples:
|
|
5490
5463
|
${chalk11.dim("$")} mutagent playground run <prompt-id> --input '{"name": "John"}'
|
|
5491
|
-
${chalk11.dim("$")} mutagent playground run <prompt-id> --file input.json
|
|
5492
5464
|
${chalk11.dim("$")} mutagent playground run <prompt-id> --input '{}' --stream
|
|
5493
5465
|
${chalk11.dim("$")} mutagent playground run <prompt-id> -i '{}' --model gpt-4-turbo
|
|
5494
5466
|
${chalk11.dim("$")} mutagent playground run <prompt-id> --system "You are helpful" --human "Hello"
|
|
@@ -5496,7 +5468,7 @@ Examples:
|
|
|
5496
5468
|
|
|
5497
5469
|
Input Format:
|
|
5498
5470
|
The input must be a valid JSON object matching the prompt's input schema.
|
|
5499
|
-
Use --input for inline JSON
|
|
5471
|
+
Use --input for inline JSON.
|
|
5500
5472
|
|
|
5501
5473
|
Prompt Style Flags:
|
|
5502
5474
|
Use --system and --human to quickly construct a messages array.
|
|
@@ -5506,10 +5478,10 @@ Streaming:
|
|
|
5506
5478
|
Use --stream to receive tokens as they are generated.
|
|
5507
5479
|
In JSON mode (--json), each token is output as a separate JSON object.
|
|
5508
5480
|
`);
|
|
5509
|
-
playground.command("run").description("Execute a prompt with input variables").argument("<prompt-id>", "Prompt ID to execute (from: mutagent prompts list)").option("-i, --input <json>", "Input variables as JSON").option("-
|
|
5481
|
+
playground.command("run").description("Execute a prompt with input variables").argument("<prompt-id>", "Prompt ID to execute (from: mutagent prompts list)").option("-i, --input <json>", "Input variables as JSON").option("-s, --stream", "Stream the response").option("-m, --model <model>", "Override model").option("--system <text>", "Set system prompt text").option("--human <text>", "Set human/user message text").option("--messages <json>", "Pass full messages array as JSON string").addHelpText("after", `
|
|
5510
5482
|
Examples:
|
|
5511
5483
|
${chalk11.dim("$")} mutagent playground run <prompt-id> --input '{"name": "John"}'
|
|
5512
|
-
${chalk11.dim("$")} mutagent playground run <prompt-id> --
|
|
5484
|
+
${chalk11.dim("$")} mutagent playground run <prompt-id> --input '{}' --stream
|
|
5513
5485
|
${chalk11.dim("$")} mutagent playground run <prompt-id> --system "You are helpful" --human "Hello"
|
|
5514
5486
|
${chalk11.dim("$")} mutagent playground run <prompt-id> --input '{}' --model gpt-4-turbo --json
|
|
5515
5487
|
|
|
@@ -5517,9 +5489,6 @@ Input Methods (pick one, priority order):
|
|
|
5517
5489
|
--system/--human Quick system + user message ${chalk11.green("(recommended)")}
|
|
5518
5490
|
--input '{"key":"value"}' Inline JSON variables
|
|
5519
5491
|
--messages '[...]' Full messages array
|
|
5520
|
-
--file input.json Load from JSON file
|
|
5521
|
-
|
|
5522
|
-
${chalk11.yellow("Note: Prefer --system/--human or --input over --file to avoid stale files living in your repo.")}
|
|
5523
5492
|
${chalk11.dim(`Hint: Test before evaluating: mutagent playground run <id> --input '{"key":"value"}'`)}
|
|
5524
5493
|
`).action(async (promptId, options) => {
|
|
5525
5494
|
const isJson = getJsonFlag(playground);
|
|
@@ -5567,13 +5536,9 @@ Execution Result:`));
|
|
|
5567
5536
|
return playground;
|
|
5568
5537
|
}
|
|
5569
5538
|
function parseInput(options) {
|
|
5570
|
-
const hasInputOrFile = options.input ?? options.file;
|
|
5571
5539
|
const hasPromptStyleFlags = options.system ?? options.human ?? options.messages;
|
|
5572
|
-
if (
|
|
5573
|
-
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --input
|
|
5574
|
-
}
|
|
5575
|
-
if (options.input && options.file) {
|
|
5576
|
-
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use both --input and --file options", "Use either --input for inline JSON or --file to load from a file");
|
|
5540
|
+
if (options.input && hasPromptStyleFlags) {
|
|
5541
|
+
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --input with prompt style flags (--system, --human, --messages)", "Use either --input OR prompt style flags, not both");
|
|
5577
5542
|
}
|
|
5578
5543
|
if (hasPromptStyleFlags) {
|
|
5579
5544
|
return parsePromptStyleInput(options);
|
|
@@ -5621,17 +5586,10 @@ function parsePromptStyleInput(options) {
|
|
|
5621
5586
|
return { messages };
|
|
5622
5587
|
}
|
|
5623
5588
|
function getInputJson(options) {
|
|
5624
|
-
if (options.file) {
|
|
5625
|
-
try {
|
|
5626
|
-
return readFileSync10(options.file, "utf-8");
|
|
5627
|
-
} catch (err) {
|
|
5628
|
-
throw new MutagentError("FILE_READ_ERROR", `Failed to read input file: ${options.file}`, err instanceof Error ? err.message : "Check the file path and permissions");
|
|
5629
|
-
}
|
|
5630
|
-
}
|
|
5631
5589
|
if (options.input) {
|
|
5632
5590
|
return options.input;
|
|
5633
5591
|
}
|
|
5634
|
-
throw new MutagentError("MISSING_ARGUMENTS", "Input is required", `Use --input '{"key": "value"}' or --
|
|
5592
|
+
throw new MutagentError("MISSING_ARGUMENTS", "Input is required", `Use --input '{"key": "value"}' or --system/--human/--messages flags`);
|
|
5635
5593
|
}
|
|
5636
5594
|
function parseJsonInput(inputJson) {
|
|
5637
5595
|
try {
|
|
@@ -5989,7 +5947,7 @@ Available Models:`));
|
|
|
5989
5947
|
import { Command as Command11 } from "commander";
|
|
5990
5948
|
import inquirer3 from "inquirer";
|
|
5991
5949
|
import chalk14 from "chalk";
|
|
5992
|
-
import { existsSync as existsSync10, mkdirSync as mkdirSync3, readFileSync as
|
|
5950
|
+
import { existsSync as existsSync10, mkdirSync as mkdirSync3, readFileSync as readFileSync9, writeFileSync as writeFileSync4 } from "fs";
|
|
5993
5951
|
import { execSync as execSync2 } from "child_process";
|
|
5994
5952
|
import { join as join5 } from "path";
|
|
5995
5953
|
var FRAMEWORK_DETECTION_MAP = {
|
|
@@ -6077,7 +6035,7 @@ function detectFrameworkFromPackageJson(cwd = process.cwd()) {
|
|
|
6077
6035
|
}
|
|
6078
6036
|
let pkg;
|
|
6079
6037
|
try {
|
|
6080
|
-
pkg = JSON.parse(
|
|
6038
|
+
pkg = JSON.parse(readFileSync9(pkgPath, "utf-8"));
|
|
6081
6039
|
} catch {
|
|
6082
6040
|
return null;
|
|
6083
6041
|
}
|
|
@@ -6677,7 +6635,7 @@ if (process.env.CLI_VERSION) {
|
|
|
6677
6635
|
try {
|
|
6678
6636
|
const __dirname2 = dirname(fileURLToPath(import.meta.url));
|
|
6679
6637
|
const pkgPath = join7(__dirname2, "..", "..", "package.json");
|
|
6680
|
-
const pkg = JSON.parse(
|
|
6638
|
+
const pkg = JSON.parse(readFileSync10(pkgPath, "utf-8"));
|
|
6681
6639
|
cliVersion = pkg.version ?? cliVersion;
|
|
6682
6640
|
} catch {}
|
|
6683
6641
|
}
|
|
@@ -6702,8 +6660,8 @@ program.addHelpText("after", `
|
|
|
6702
6660
|
${chalk18.yellow("Workflows:")}
|
|
6703
6661
|
${chalk18.bold("Evaluate → Optimize Loop:")}
|
|
6704
6662
|
1. mutagent prompts create --name "..." --raw-file prompt.txt
|
|
6705
|
-
2. mutagent prompts dataset add <prompt-id> --name "..."
|
|
6706
|
-
3. mutagent prompts evaluation create <prompt-id> --name "..."
|
|
6663
|
+
2. mutagent prompts dataset add <prompt-id> --name "..." -d '[{"input":{...},"expectedOutput":{...}}]'
|
|
6664
|
+
3. mutagent prompts evaluation create <prompt-id> --name "..." -d '{"evalConfig":{"criteria":[...]}}'
|
|
6707
6665
|
4. mutagent prompts optimize start <prompt-id> --dataset <id> --max-iterations 1
|
|
6708
6666
|
|
|
6709
6667
|
${chalk18.bold("Quick Test:")}
|
|
@@ -6778,5 +6736,5 @@ program.addCommand(createSkillsCommand());
|
|
|
6778
6736
|
program.addCommand(createUsageCommand());
|
|
6779
6737
|
program.parse();
|
|
6780
6738
|
|
|
6781
|
-
//# debugId=
|
|
6739
|
+
//# debugId=790DEAC174F0003F64756E2164756E21
|
|
6782
6740
|
//# sourceMappingURL=cli.js.map
|