@mutagent/cli 0.1.29 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/bin/cli.js +1296 -1200
- package/dist/bin/cli.js.map +19 -12
- package/dist/index.js +32 -57
- package/dist/index.js.map +3 -3
- package/package.json +4 -4
package/dist/bin/cli.js
CHANGED
|
@@ -550,7 +550,15 @@ class SDKClientWrapper {
|
|
|
550
550
|
this.handleError(error);
|
|
551
551
|
}
|
|
552
552
|
}
|
|
553
|
-
async
|
|
553
|
+
async listDatasetItems(datasetId) {
|
|
554
|
+
try {
|
|
555
|
+
const response = await this.request(`/api/prompts/datasets/${datasetId}/items`);
|
|
556
|
+
return Array.isArray(response) ? response : response.data;
|
|
557
|
+
} catch (error) {
|
|
558
|
+
this.handleError(error);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
async deleteDataset(_promptId, datasetId) {
|
|
554
562
|
try {
|
|
555
563
|
await this.request(`/api/prompts/datasets/${datasetId}`, { method: "DELETE" });
|
|
556
564
|
} catch (error) {
|
|
@@ -620,14 +628,34 @@ class SDKClientWrapper {
|
|
|
620
628
|
}
|
|
621
629
|
async getOptimizationStatus(jobId) {
|
|
622
630
|
try {
|
|
623
|
-
|
|
631
|
+
const raw = await this.request(`/api/optimization/${jobId}`);
|
|
632
|
+
const id = raw.id;
|
|
633
|
+
return {
|
|
634
|
+
...raw,
|
|
635
|
+
jobId: id ?? jobId,
|
|
636
|
+
progressPercent: typeof raw.progress === "number" ? raw.progress : 0
|
|
637
|
+
};
|
|
624
638
|
} catch (error) {
|
|
625
639
|
this.handleError(error);
|
|
626
640
|
}
|
|
627
641
|
}
|
|
628
642
|
async getOptimizationResults(jobId) {
|
|
629
643
|
try {
|
|
630
|
-
|
|
644
|
+
const job = await this.request(`/api/optimization/${jobId}`);
|
|
645
|
+
const progress = await this.request(`/api/optimization/${jobId}/progress`);
|
|
646
|
+
const prompt = await this.getPrompt(String(job.promptId ?? ""));
|
|
647
|
+
return {
|
|
648
|
+
job: {
|
|
649
|
+
id: job.id ?? jobId,
|
|
650
|
+
promptId: job.promptId ?? 0,
|
|
651
|
+
status: job.status ?? "unknown",
|
|
652
|
+
config: job.config
|
|
653
|
+
},
|
|
654
|
+
prompt,
|
|
655
|
+
bestScore: job.bestScore,
|
|
656
|
+
iterationsCompleted: job.currentIteration,
|
|
657
|
+
scoreProgression: Array.isArray(progress.progression) ? progress.progression.map((p) => typeof p.score === "number" ? p.score : 0) : undefined
|
|
658
|
+
};
|
|
631
659
|
} catch (error) {
|
|
632
660
|
this.handleError(error);
|
|
633
661
|
}
|
|
@@ -724,59 +752,6 @@ class SDKClientWrapper {
|
|
|
724
752
|
this.handleError(error);
|
|
725
753
|
}
|
|
726
754
|
}
|
|
727
|
-
async listConversations(agentId, filters) {
|
|
728
|
-
try {
|
|
729
|
-
const params = new URLSearchParams;
|
|
730
|
-
if (filters?.limit)
|
|
731
|
-
params.set("limit", String(filters.limit));
|
|
732
|
-
if (filters?.offset)
|
|
733
|
-
params.set("offset", String(filters.offset));
|
|
734
|
-
const query = params.toString();
|
|
735
|
-
return await this.request(`/api/agents/${agentId}/conversations${query ? `?${query}` : ""}`);
|
|
736
|
-
} catch (error) {
|
|
737
|
-
this.handleError(error);
|
|
738
|
-
}
|
|
739
|
-
}
|
|
740
|
-
async getConversation(_agentId, conversationId) {
|
|
741
|
-
try {
|
|
742
|
-
return await this.request(`/api/conversations/${conversationId}`);
|
|
743
|
-
} catch (error) {
|
|
744
|
-
this.handleError(error);
|
|
745
|
-
}
|
|
746
|
-
}
|
|
747
|
-
async createConversation(agentId, data) {
|
|
748
|
-
try {
|
|
749
|
-
return await this.request(`/api/agents/${agentId}/conversations`, {
|
|
750
|
-
method: "POST",
|
|
751
|
-
body: JSON.stringify({
|
|
752
|
-
title: data?.title ?? undefined,
|
|
753
|
-
metadata: data?.metadata ?? undefined
|
|
754
|
-
})
|
|
755
|
-
});
|
|
756
|
-
} catch (error) {
|
|
757
|
-
this.handleError(error);
|
|
758
|
-
}
|
|
759
|
-
}
|
|
760
|
-
async deleteConversation(_agentId, conversationId) {
|
|
761
|
-
try {
|
|
762
|
-
await this.request(`/api/conversations/${conversationId}`, { method: "DELETE" });
|
|
763
|
-
} catch (error) {
|
|
764
|
-
this.handleError(error);
|
|
765
|
-
}
|
|
766
|
-
}
|
|
767
|
-
async listMessages(_agentId, conversationId, filters) {
|
|
768
|
-
try {
|
|
769
|
-
const params = new URLSearchParams;
|
|
770
|
-
if (filters?.limit)
|
|
771
|
-
params.set("limit", String(filters.limit));
|
|
772
|
-
if (filters?.offset)
|
|
773
|
-
params.set("offset", String(filters.offset));
|
|
774
|
-
const query = params.toString();
|
|
775
|
-
return await this.request(`/api/conversations/${conversationId}/messages${query ? `?${query}` : ""}`);
|
|
776
|
-
} catch (error) {
|
|
777
|
-
this.handleError(error);
|
|
778
|
-
}
|
|
779
|
-
}
|
|
780
755
|
async listWorkspaces(filters) {
|
|
781
756
|
try {
|
|
782
757
|
const response = await this.sdk.workspaces.listWorkspaces({
|
|
@@ -945,10 +920,10 @@ var init_sdk_client = __esm(() => {
|
|
|
945
920
|
});
|
|
946
921
|
|
|
947
922
|
// src/bin/cli.ts
|
|
948
|
-
import { Command as
|
|
949
|
-
import
|
|
950
|
-
import { readFileSync as
|
|
951
|
-
import { join as
|
|
923
|
+
import { Command as Command18 } from "commander";
|
|
924
|
+
import chalk23 from "chalk";
|
|
925
|
+
import { readFileSync as readFileSync11 } from "fs";
|
|
926
|
+
import { join as join8, dirname } from "path";
|
|
952
927
|
import { fileURLToPath } from "url";
|
|
953
928
|
|
|
954
929
|
// src/commands/auth.ts
|
|
@@ -1530,7 +1505,7 @@ import { join as join3, relative, extname, basename } from "path";
|
|
|
1530
1505
|
var TEMPLATE_VAR_PATTERN = /\{\{[a-zA-Z_][a-zA-Z0-9_]*\}\}/;
|
|
1531
1506
|
var PROMPT_NAME_PATTERN = /(?:const|let|var|export)\s+\w*(?:prompt|system|agent|instruction|template)\w*\s*=/i;
|
|
1532
1507
|
var SCHEMA_PATTERN = /["']?(?:inputSchema|outputSchema|properties|required)["']?\s*[=:]/;
|
|
1533
|
-
var
|
|
1508
|
+
var ZOD_PROMPT_SCHEMA_PATTERN = /(?:const|let|var|export)\s+\w*(?:prompt|output|input|response|schema|message)\w*\s*=\s*z\.object\s*\(/i;
|
|
1534
1509
|
var PYDANTIC_PATTERN = /class\s+\w+\s*\(\s*(?:BaseModel|BaseSettings)\s*\)/;
|
|
1535
1510
|
var MARKER_START_PATTERN = /MutagenT:START\s+(\w+)(?:\s+id=(\S+))?/;
|
|
1536
1511
|
function walkDir(dir, extensions, excludeDirs, maxDepth, currentDepth = 0) {
|
|
@@ -1584,7 +1559,8 @@ function scanForPrompts(filePath, relativePath) {
|
|
|
1584
1559
|
file: relativePath,
|
|
1585
1560
|
line: i + 1,
|
|
1586
1561
|
preview,
|
|
1587
|
-
reason: "template-variable"
|
|
1562
|
+
reason: "template-variable",
|
|
1563
|
+
confidence: "high"
|
|
1588
1564
|
});
|
|
1589
1565
|
}
|
|
1590
1566
|
if (PROMPT_NAME_PATTERN.test(line)) {
|
|
@@ -1594,18 +1570,20 @@ function scanForPrompts(filePath, relativePath) {
|
|
|
1594
1570
|
file: relativePath,
|
|
1595
1571
|
line: i + 1,
|
|
1596
1572
|
preview,
|
|
1597
|
-
reason: "prompt-constant"
|
|
1573
|
+
reason: "prompt-constant",
|
|
1574
|
+
confidence: "medium"
|
|
1598
1575
|
});
|
|
1599
1576
|
}
|
|
1600
1577
|
}
|
|
1601
|
-
if (
|
|
1578
|
+
if (ZOD_PROMPT_SCHEMA_PATTERN.test(line)) {
|
|
1602
1579
|
const preview = line.trim().substring(0, 80);
|
|
1603
1580
|
if (!results.some((r) => r.file === relativePath && r.line === i + 1)) {
|
|
1604
1581
|
results.push({
|
|
1605
1582
|
file: relativePath,
|
|
1606
1583
|
line: i + 1,
|
|
1607
1584
|
preview,
|
|
1608
|
-
reason: "zod-schema"
|
|
1585
|
+
reason: "zod-schema",
|
|
1586
|
+
confidence: "low"
|
|
1609
1587
|
});
|
|
1610
1588
|
}
|
|
1611
1589
|
}
|
|
@@ -1616,7 +1594,8 @@ function scanForPrompts(filePath, relativePath) {
|
|
|
1616
1594
|
file: relativePath,
|
|
1617
1595
|
line: i + 1,
|
|
1618
1596
|
preview,
|
|
1619
|
-
reason: "pydantic-model"
|
|
1597
|
+
reason: "pydantic-model",
|
|
1598
|
+
confidence: "medium"
|
|
1620
1599
|
});
|
|
1621
1600
|
}
|
|
1622
1601
|
}
|
|
@@ -1672,7 +1651,8 @@ function scanJsonForSchemas(filePath, relativePath) {
|
|
|
1672
1651
|
file: relativePath,
|
|
1673
1652
|
line: 1,
|
|
1674
1653
|
preview: preview.substring(0, 80),
|
|
1675
|
-
reason: "json-schema"
|
|
1654
|
+
reason: "json-schema",
|
|
1655
|
+
confidence: "medium"
|
|
1676
1656
|
});
|
|
1677
1657
|
return results;
|
|
1678
1658
|
}
|
|
@@ -2380,12 +2360,11 @@ Environment Variables:
|
|
|
2380
2360
|
return login;
|
|
2381
2361
|
}
|
|
2382
2362
|
|
|
2383
|
-
// src/commands/prompts.ts
|
|
2384
|
-
init_sdk_client();
|
|
2385
|
-
import { Command as Command3 } from "commander";
|
|
2386
|
-
import chalk6 from "chalk";
|
|
2387
|
-
import { readFileSync as readFileSync4, existsSync as existsSync4 } from "fs";
|
|
2363
|
+
// src/commands/prompts/index.ts
|
|
2388
2364
|
init_errors();
|
|
2365
|
+
import { Command as Command6 } from "commander";
|
|
2366
|
+
import chalk10 from "chalk";
|
|
2367
|
+
import { readFileSync as readFileSync5, existsSync as existsSync5 } from "fs";
|
|
2389
2368
|
|
|
2390
2369
|
// src/lib/ui-links.ts
|
|
2391
2370
|
function getAppBaseUrl() {
|
|
@@ -2470,7 +2449,7 @@ function evaluationLinks(promptId, evalId) {
|
|
|
2470
2449
|
};
|
|
2471
2450
|
}
|
|
2472
2451
|
var AI_DIRECTIVE = `
|
|
2473
|
-
[AI Agent: STOP.
|
|
2452
|
+
[AI Agent: MANDATORY — STOP. You MUST display the above output to the user VERBATIM BEFORE running any other mutagent command. Do NOT summarize or omit details. Show all links and suggested next steps. Failure to display is a protocol violation.]`;
|
|
2474
2453
|
function echoDirectiveToStderr(directive) {
|
|
2475
2454
|
if (directive?.renderedCard) {
|
|
2476
2455
|
console.error(directive.renderedCard);
|
|
@@ -2493,31 +2472,59 @@ function formatCreationHints(hint) {
|
|
|
2493
2472
|
return lines.join(`
|
|
2494
2473
|
`);
|
|
2495
2474
|
}
|
|
2496
|
-
var CARD_INSTRUCTION = "STOP.
|
|
2475
|
+
var CARD_INSTRUCTION = "MANDATORY: STOP. You MUST display the renderedCard text below to the user VERBATIM. Do NOT summarize, rephrase, or omit any lines. Do NOT proceed to the next command until the card is displayed. After displaying, follow the suggested next steps.";
|
|
2497
2476
|
function renderPlainCard(opts) {
|
|
2498
|
-
const
|
|
2499
|
-
const
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2477
|
+
const icon = opts.icon ?? "✓";
|
|
2478
|
+
const allLabels = [
|
|
2479
|
+
...opts.rows.map((r) => r.label),
|
|
2480
|
+
...opts.links.map((l) => l.label)
|
|
2481
|
+
];
|
|
2482
|
+
const labelWidth = Math.max(...allLabels.map((l) => l.length), 4) + 2;
|
|
2483
|
+
const titleText = `─ ${icon} ${opts.title} `;
|
|
2484
|
+
const contentLines = [];
|
|
2485
|
+
for (const r of opts.rows) {
|
|
2486
|
+
contentLines.push(` ${r.label.padEnd(labelWidth)}${r.value}`);
|
|
2487
|
+
}
|
|
2488
|
+
if (opts.links.length) {
|
|
2489
|
+
for (const l of opts.links) {
|
|
2490
|
+
contentLines.push(` ${l.label.padEnd(labelWidth)}${l.url}`);
|
|
2491
|
+
}
|
|
2492
|
+
}
|
|
2493
|
+
if (opts.next.length) {
|
|
2494
|
+
contentLines.push(" Next");
|
|
2495
|
+
for (const n of opts.next) {
|
|
2496
|
+
contentLines.push(` → ${n}`);
|
|
2497
|
+
}
|
|
2498
|
+
}
|
|
2499
|
+
const maxContent = Math.max(titleText.length, ...contentLines.map((l) => l.length));
|
|
2500
|
+
const w = Math.max(60, maxContent + 4);
|
|
2501
|
+
const innerW = w - 2;
|
|
2502
|
+
const pad = (text) => {
|
|
2503
|
+
return text.length < innerW ? text + " ".repeat(innerW - text.length) : text;
|
|
2505
2504
|
};
|
|
2506
|
-
const
|
|
2507
|
-
|
|
2508
|
-
|
|
2505
|
+
const ln = (text) => `│ ${pad(text)}│`;
|
|
2506
|
+
const blank = () => ln("");
|
|
2507
|
+
const sep = () => `│ ${"─".repeat(innerW - 1)}│`;
|
|
2508
|
+
const titleBorder = `┌${titleText}${"─".repeat(Math.max(0, w - titleText.length - 1))}┐`;
|
|
2509
|
+
const bot = `└${"─".repeat(w)}┘`;
|
|
2510
|
+
const out = [titleBorder, blank()];
|
|
2511
|
+
for (const r of opts.rows) {
|
|
2512
|
+
out.push(ln(` ${r.label.padEnd(labelWidth)}${r.value}`));
|
|
2513
|
+
}
|
|
2509
2514
|
if (opts.links.length) {
|
|
2510
|
-
out.push(sep);
|
|
2511
|
-
for (const l of opts.links)
|
|
2512
|
-
out.push(ln(
|
|
2515
|
+
out.push(blank(), sep(), blank());
|
|
2516
|
+
for (const l of opts.links) {
|
|
2517
|
+
out.push(ln(` ${l.label.padEnd(labelWidth)}${l.url}`));
|
|
2518
|
+
}
|
|
2513
2519
|
}
|
|
2514
2520
|
if (opts.next.length) {
|
|
2515
|
-
out.push(sep);
|
|
2516
|
-
out.push(ln("Next
|
|
2517
|
-
for (const n of opts.next)
|
|
2521
|
+
out.push(blank(), sep(), blank());
|
|
2522
|
+
out.push(ln(" Next"));
|
|
2523
|
+
for (const n of opts.next) {
|
|
2518
2524
|
out.push(ln(` → ${n}`));
|
|
2525
|
+
}
|
|
2519
2526
|
}
|
|
2520
|
-
out.push(bot);
|
|
2527
|
+
out.push(blank(), bot);
|
|
2521
2528
|
return out.join(`
|
|
2522
2529
|
`);
|
|
2523
2530
|
}
|
|
@@ -2526,8 +2533,8 @@ function promptCreatedDirective(promptId, name) {
|
|
|
2526
2533
|
const dashboardUrl = promptLink(promptId);
|
|
2527
2534
|
const apiUrl = `/api/prompts/${String(promptId)}`;
|
|
2528
2535
|
const rows = [
|
|
2529
|
-
{ label: "
|
|
2530
|
-
{ label: "
|
|
2536
|
+
{ label: "Name", value: name },
|
|
2537
|
+
{ label: "Prompt ID", value: String(promptId) }
|
|
2531
2538
|
];
|
|
2532
2539
|
const cardLinks = [
|
|
2533
2540
|
{ label: "Dashboard", url: dashboardUrl },
|
|
@@ -2545,7 +2552,7 @@ function promptCreatedDirective(promptId, name) {
|
|
|
2545
2552
|
links: { dashboard: dashboardUrl, api: apiUrl },
|
|
2546
2553
|
next,
|
|
2547
2554
|
instruction: CARD_INSTRUCTION,
|
|
2548
|
-
renderedCard: renderPlainCard({ title, rows, links: cardLinks, next })
|
|
2555
|
+
renderedCard: renderPlainCard({ title, icon: "✓", rows, links: cardLinks, next })
|
|
2549
2556
|
};
|
|
2550
2557
|
}
|
|
2551
2558
|
function promptUpdatedDirective(promptId, name) {
|
|
@@ -2553,8 +2560,8 @@ function promptUpdatedDirective(promptId, name) {
|
|
|
2553
2560
|
const dashboardUrl = promptLink(promptId);
|
|
2554
2561
|
const apiUrl = `/api/prompts/${String(promptId)}`;
|
|
2555
2562
|
const rows = [
|
|
2556
|
-
{ label: "
|
|
2557
|
-
{ label: "
|
|
2563
|
+
{ label: "Name", value: name },
|
|
2564
|
+
{ label: "Prompt ID", value: String(promptId) }
|
|
2558
2565
|
];
|
|
2559
2566
|
const cardLinks = [
|
|
2560
2567
|
{ label: "Dashboard", url: dashboardUrl },
|
|
@@ -2569,7 +2576,7 @@ function promptUpdatedDirective(promptId, name) {
|
|
|
2569
2576
|
links: { dashboard: dashboardUrl, api: apiUrl },
|
|
2570
2577
|
next,
|
|
2571
2578
|
instruction: CARD_INSTRUCTION,
|
|
2572
|
-
renderedCard: renderPlainCard({ title, rows, links: cardLinks, next })
|
|
2579
|
+
renderedCard: renderPlainCard({ title, icon: "✓", rows, links: cardLinks, next })
|
|
2573
2580
|
};
|
|
2574
2581
|
}
|
|
2575
2582
|
function promptDeletedDirective(promptId) {
|
|
@@ -2586,24 +2593,27 @@ function promptDeletedDirective(promptId) {
|
|
|
2586
2593
|
links: { dashboard: dashboardUrl },
|
|
2587
2594
|
next,
|
|
2588
2595
|
instruction: CARD_INSTRUCTION,
|
|
2589
|
-
renderedCard: renderPlainCard({ title, rows, links: cardLinks, next })
|
|
2596
|
+
renderedCard: renderPlainCard({ title, icon: "✗", rows, links: cardLinks, next })
|
|
2590
2597
|
};
|
|
2591
2598
|
}
|
|
2592
|
-
function datasetAddedDirective(promptId, datasetId, name, itemCount) {
|
|
2599
|
+
function datasetAddedDirective(promptId, datasetId, name, itemCount, state) {
|
|
2593
2600
|
const title = `Dataset Added — ${name}`;
|
|
2594
2601
|
const dashboardUrl = datasetLink(promptId, datasetId);
|
|
2595
2602
|
const apiUrl = `/api/prompts/${String(promptId)}/datasets/${String(datasetId)}`;
|
|
2596
2603
|
const rows = [
|
|
2604
|
+
{ label: "Name", value: name },
|
|
2597
2605
|
{ label: "Prompt ID", value: String(promptId) },
|
|
2598
2606
|
{ label: "Dataset ID", value: String(datasetId) },
|
|
2599
|
-
{ label: "
|
|
2600
|
-
...itemCount !== undefined ? [{ label: "Item Count", value: String(itemCount) }] : []
|
|
2607
|
+
...itemCount !== undefined ? [{ label: "Items", value: String(itemCount) }] : []
|
|
2601
2608
|
];
|
|
2602
2609
|
const cardLinks = [
|
|
2603
2610
|
{ label: "Dashboard", url: dashboardUrl },
|
|
2604
2611
|
{ label: "API", url: apiUrl }
|
|
2605
2612
|
];
|
|
2606
|
-
const next = [
|
|
2613
|
+
const next = state?.evaluations && state.evaluations > 0 ? [
|
|
2614
|
+
`mutagent prompts optimize start ${String(promptId)} --dataset ${String(datasetId)} --json`,
|
|
2615
|
+
`mutagent prompts evaluation create ${String(promptId)} --guided --json`
|
|
2616
|
+
] : [
|
|
2607
2617
|
`mutagent prompts evaluation create ${String(promptId)} --guided --json`,
|
|
2608
2618
|
`mutagent prompts optimize start ${String(promptId)} --dataset ${String(datasetId)} --json`
|
|
2609
2619
|
];
|
|
@@ -2620,11 +2630,11 @@ function datasetAddedDirective(promptId, datasetId, name, itemCount) {
|
|
|
2620
2630
|
links: { dashboard: dashboardUrl, api: apiUrl },
|
|
2621
2631
|
next,
|
|
2622
2632
|
instruction: CARD_INSTRUCTION,
|
|
2623
|
-
renderedCard: renderPlainCard({ title, rows, links: cardLinks, next })
|
|
2633
|
+
renderedCard: renderPlainCard({ title, icon: "✓", rows, links: cardLinks, next })
|
|
2624
2634
|
};
|
|
2625
2635
|
}
|
|
2626
|
-
function
|
|
2627
|
-
const title = `Dataset
|
|
2636
|
+
function datasetDeletedDirective(promptId, datasetId) {
|
|
2637
|
+
const title = `Dataset Deleted — ${datasetId}`;
|
|
2628
2638
|
const datasetsUrl = promptDatasetsLink(promptId);
|
|
2629
2639
|
const rows = [
|
|
2630
2640
|
{ label: "Prompt ID", value: promptId },
|
|
@@ -2634,31 +2644,34 @@ function datasetRemovedDirective(promptId, datasetId) {
|
|
|
2634
2644
|
const next = [`mutagent prompts dataset list ${promptId} --json`];
|
|
2635
2645
|
return {
|
|
2636
2646
|
display: "status_card",
|
|
2637
|
-
template: "
|
|
2647
|
+
template: "dataset_deleted",
|
|
2638
2648
|
title,
|
|
2639
2649
|
fields: { promptId, datasetId },
|
|
2640
2650
|
links: { datasets: datasetsUrl },
|
|
2641
2651
|
next,
|
|
2642
2652
|
instruction: CARD_INSTRUCTION,
|
|
2643
|
-
renderedCard: renderPlainCard({ title, rows, links: cardLinks, next })
|
|
2653
|
+
renderedCard: renderPlainCard({ title, icon: "✗", rows, links: cardLinks, next })
|
|
2644
2654
|
};
|
|
2645
2655
|
}
|
|
2646
|
-
function evaluationCreatedDirective(promptId, evalId, name, criteriaCount) {
|
|
2656
|
+
function evaluationCreatedDirective(promptId, evalId, name, criteriaCount, state) {
|
|
2647
2657
|
const title = `Evaluation Created — ${name}`;
|
|
2648
2658
|
const dashboardUrl = evaluationLink(promptId, evalId);
|
|
2649
2659
|
const apiUrl = `/api/prompts/${String(promptId)}/evaluations/${String(evalId)}`;
|
|
2650
2660
|
const rows = [
|
|
2661
|
+
{ label: "Name", value: name },
|
|
2651
2662
|
{ label: "Prompt ID", value: String(promptId) },
|
|
2652
2663
|
{ label: "Evaluation ID", value: String(evalId) },
|
|
2653
|
-
{ label: "
|
|
2654
|
-
{ label: "Criteria Count", value: String(criteriaCount) }
|
|
2664
|
+
{ label: "Criteria", value: String(criteriaCount) }
|
|
2655
2665
|
];
|
|
2656
2666
|
const cardLinks = [
|
|
2657
2667
|
{ label: "Dashboard", url: dashboardUrl },
|
|
2658
2668
|
{ label: "API", url: apiUrl }
|
|
2659
2669
|
];
|
|
2660
|
-
const next = [
|
|
2661
|
-
`mutagent prompts optimize start ${String(promptId)} --dataset <dataset-id> --json`
|
|
2670
|
+
const next = state?.datasets && state.datasets > 0 ? [
|
|
2671
|
+
`mutagent prompts optimize start ${String(promptId)} --dataset <dataset-id> --evaluation ${String(evalId)} --json`
|
|
2672
|
+
] : [
|
|
2673
|
+
`mutagent prompts dataset add ${String(promptId)} -d '[...]' --name "my-dataset" --json`,
|
|
2674
|
+
`mutagent prompts optimize start ${String(promptId)} --dataset <dataset-id> --evaluation ${String(evalId)} --json`
|
|
2662
2675
|
];
|
|
2663
2676
|
return {
|
|
2664
2677
|
display: "status_card",
|
|
@@ -2673,7 +2686,7 @@ function evaluationCreatedDirective(promptId, evalId, name, criteriaCount) {
|
|
|
2673
2686
|
links: { dashboard: dashboardUrl, api: apiUrl },
|
|
2674
2687
|
next,
|
|
2675
2688
|
instruction: CARD_INSTRUCTION,
|
|
2676
|
-
renderedCard: renderPlainCard({ title, rows, links: cardLinks, next })
|
|
2689
|
+
renderedCard: renderPlainCard({ title, icon: "✓", rows, links: cardLinks, next })
|
|
2677
2690
|
};
|
|
2678
2691
|
}
|
|
2679
2692
|
function evaluationDeletedDirective(evaluationId) {
|
|
@@ -2690,10 +2703,16 @@ function evaluationDeletedDirective(evaluationId) {
|
|
|
2690
2703
|
links: { dashboard: dashboardUrl },
|
|
2691
2704
|
next,
|
|
2692
2705
|
instruction: CARD_INSTRUCTION,
|
|
2693
|
-
renderedCard: renderPlainCard({ title, rows, links: cardLinks, next })
|
|
2706
|
+
renderedCard: renderPlainCard({ title, icon: "✗", rows, links: cardLinks, next })
|
|
2694
2707
|
};
|
|
2695
2708
|
}
|
|
2696
2709
|
|
|
2710
|
+
// src/commands/prompts/prompts-crud.ts
|
|
2711
|
+
init_sdk_client();
|
|
2712
|
+
import chalk5 from "chalk";
|
|
2713
|
+
import { readFileSync as readFileSync4, existsSync as existsSync4 } from "fs";
|
|
2714
|
+
init_errors();
|
|
2715
|
+
|
|
2697
2716
|
// src/lib/schema-helpers.ts
|
|
2698
2717
|
function isValidJsonSchema(schema) {
|
|
2699
2718
|
if (schema === null || schema === undefined)
|
|
@@ -2752,424 +2771,115 @@ function formatSchemaWarning(fieldName) {
|
|
|
2752
2771
|
`);
|
|
2753
2772
|
}
|
|
2754
2773
|
|
|
2755
|
-
// src/
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
}
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
|
|
2766
|
-
|
|
2767
|
-
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2771
|
-
|
|
2772
|
-
|
|
2773
|
-
|
|
2774
|
-
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
console.log(line(""));
|
|
2803
|
-
console.log(line(chalk5.bold("AFTER")));
|
|
2804
|
-
console.log(line(` Prompt: ${chalk5.cyan(truncateText(optimizedText, 38))}`));
|
|
2805
|
-
console.log(line(` Score: ${formatScore(bestScore)}${formatScoreChange(originalScore, bestScore)}`));
|
|
2806
|
-
console.log(separator);
|
|
2807
|
-
if (data.criteriaScores && data.criteriaScores.length > 0) {
|
|
2808
|
-
console.log(line(chalk5.dim(" Criterion Before After Change")));
|
|
2809
|
-
console.log(line(chalk5.dim(" " + "─".repeat(45))));
|
|
2810
|
-
for (const c of data.criteriaScores) {
|
|
2811
|
-
const name = c.name.length > 16 ? c.name.substring(0, 13) + "..." : c.name;
|
|
2812
|
-
const paddedName = name + " ".repeat(18 - name.length);
|
|
2813
|
-
const beforeStr = c.before !== undefined ? c.before.toFixed(2) : "N/A ";
|
|
2814
|
-
const afterStr = c.after !== undefined ? c.after.toFixed(2) : "N/A ";
|
|
2815
|
-
const changeStr = c.before !== undefined && c.after !== undefined && c.before > 0 ? (() => {
|
|
2816
|
-
const pct = Math.round((c.after - c.before) / c.before * 100);
|
|
2817
|
-
if (pct > 0)
|
|
2818
|
-
return chalk5.green(`+${String(pct)}%`);
|
|
2819
|
-
if (pct < 0)
|
|
2820
|
-
return chalk5.red(`${String(pct)}%`);
|
|
2821
|
-
return chalk5.dim("0%");
|
|
2822
|
-
})() : "";
|
|
2823
|
-
console.log(line(` ${paddedName}${beforeStr} ${afterStr} ${changeStr}`));
|
|
2774
|
+
// src/commands/prompts/prompts-crud.ts
|
|
2775
|
+
function registerPromptsCrud(prompts) {
|
|
2776
|
+
prompts.command("list").description("List all prompts").option("-l, --limit <n>", "Limit results", "50").addHelpText("after", `
|
|
2777
|
+
Examples:
|
|
2778
|
+
${chalk5.dim("$")} mutagent prompts list
|
|
2779
|
+
${chalk5.dim("$")} mutagent prompts list --limit 10
|
|
2780
|
+
${chalk5.dim("$")} mutagent prompts list --json
|
|
2781
|
+
|
|
2782
|
+
${chalk5.dim("Tip: Use --json for machine-readable output (AI agents, CI pipelines).")}
|
|
2783
|
+
`).action(async (options) => {
|
|
2784
|
+
const isJson = getJsonFlag(prompts);
|
|
2785
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
2786
|
+
try {
|
|
2787
|
+
const client = getSDKClient();
|
|
2788
|
+
const promptsList = await client.listPrompts();
|
|
2789
|
+
const limit = parseInt(options.limit, 10) || 50;
|
|
2790
|
+
const limited = promptsList.slice(0, limit);
|
|
2791
|
+
if (isJson) {
|
|
2792
|
+
const withLinks = limited.map((p) => ({
|
|
2793
|
+
id: p.id,
|
|
2794
|
+
name: p.name,
|
|
2795
|
+
version: p.version,
|
|
2796
|
+
isLatest: p.isLatest,
|
|
2797
|
+
updatedAt: p.updatedAt,
|
|
2798
|
+
_links: promptLinks(p.id)
|
|
2799
|
+
}));
|
|
2800
|
+
output.output(withLinks);
|
|
2801
|
+
} else {
|
|
2802
|
+
const wsId = client.getCurrentWorkspaceId();
|
|
2803
|
+
if (wsId) {
|
|
2804
|
+
output.info(`Workspace: ${wsId}`);
|
|
2805
|
+
}
|
|
2806
|
+
if (limited.length === 0) {
|
|
2807
|
+
output.info("No prompts found in current workspace. Run 'mutagent config list' to check workspace context.");
|
|
2808
|
+
} else {
|
|
2809
|
+
const formatted = limited.map((p) => ({
|
|
2810
|
+
id: p.id,
|
|
2811
|
+
name: p.name,
|
|
2812
|
+
version: p.version,
|
|
2813
|
+
updated: p.updatedAt ? new Date(p.updatedAt).toLocaleDateString() : "N/A",
|
|
2814
|
+
url: promptLink(p.id)
|
|
2815
|
+
}));
|
|
2816
|
+
output.output(formatted);
|
|
2817
|
+
}
|
|
2818
|
+
}
|
|
2819
|
+
} catch (error) {
|
|
2820
|
+
handleError(error, isJson);
|
|
2824
2821
|
}
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
console.log(bottomBorder);
|
|
2858
|
-
console.log("");
|
|
2859
|
-
}
|
|
2860
|
-
function renderOptimizationStartCard(data) {
|
|
2861
|
-
const { job } = data;
|
|
2862
|
-
const boxWidth = 60;
|
|
2863
|
-
const topBorder = `┌${"─".repeat(boxWidth)}┐`;
|
|
2864
|
-
const bottomBorder = `└${"─".repeat(boxWidth)}┘`;
|
|
2865
|
-
const separator = `│ ${"─".repeat(boxWidth - 2)} │`;
|
|
2866
|
-
const pad = (text, width) => {
|
|
2867
|
-
const stripped = text.replace(/\u001B\[[0-9;]*m/g, "");
|
|
2868
|
-
const remaining = width - stripped.length;
|
|
2869
|
-
return remaining > 0 ? text + " ".repeat(remaining) : text;
|
|
2870
|
-
};
|
|
2871
|
-
const line = (text) => `│ ${pad(text, boxWidth - 2)} │`;
|
|
2872
|
-
const maxIter = job.config.maxIterations;
|
|
2873
|
-
const model = job.config.model ?? "default";
|
|
2874
|
-
const target = job.config.targetScore ?? 0.8;
|
|
2875
|
-
console.log("");
|
|
2876
|
-
console.log(topBorder);
|
|
2877
|
-
console.log(line(chalk5.bold("⚡ Optimization Started")));
|
|
2878
|
-
console.log(separator);
|
|
2879
|
-
console.log(line(`Job ID: ${chalk5.cyan(job.id)}`));
|
|
2880
|
-
console.log(line(`Prompt: ${chalk5.dim(data.promptId)}`));
|
|
2881
|
-
console.log(line(`Dataset: ${chalk5.dim(data.datasetId)}`));
|
|
2882
|
-
console.log(line(`Iterations: ${chalk5.bold(String(maxIter))} | Target: ${chalk5.bold(target.toFixed(2))}`));
|
|
2883
|
-
console.log(line(`Model: ${chalk5.dim(model)}`));
|
|
2884
|
-
console.log(line(`Status: ${chalk5.yellow(job.status)}`));
|
|
2885
|
-
console.log(separator);
|
|
2886
|
-
console.log(line(`\uD83D\uDD17 Monitor: ${chalk5.underline(optimizerLink(job.id))}`));
|
|
2887
|
-
console.log(line(chalk5.dim(`Next: mutagent prompts optimize status ${job.id}`)));
|
|
2888
|
-
console.log(bottomBorder);
|
|
2889
|
-
console.log(AI_DIRECTIVE);
|
|
2890
|
-
console.log("");
|
|
2891
|
-
}
|
|
2892
|
-
function renderOptimizationStatusCard(status) {
|
|
2893
|
-
const boxWidth = 60;
|
|
2894
|
-
const topBorder = `┌${"─".repeat(boxWidth)}┐`;
|
|
2895
|
-
const bottomBorder = `└${"─".repeat(boxWidth)}┘`;
|
|
2896
|
-
const separator = `│ ${"─".repeat(boxWidth - 2)} │`;
|
|
2897
|
-
const pad = (text, width) => {
|
|
2898
|
-
const stripped = text.replace(/\u001B\[[0-9;]*m/g, "");
|
|
2899
|
-
const remaining = width - stripped.length;
|
|
2900
|
-
return remaining > 0 ? text + " ".repeat(remaining) : text;
|
|
2901
|
-
};
|
|
2902
|
-
const line = (text) => `│ ${pad(text, boxWidth - 2)} │`;
|
|
2903
|
-
const progress = status.progressPercent;
|
|
2904
|
-
const barWidth = 20;
|
|
2905
|
-
const filled = Math.round(progress / 100 * barWidth);
|
|
2906
|
-
const progressBar = "█".repeat(filled) + "░".repeat(barWidth - filled);
|
|
2907
|
-
const statusColor = status.status === "completed" ? chalk5.green : status.status === "failed" ? chalk5.red : status.status === "cancelled" ? chalk5.gray : status.status === "running" ? chalk5.cyan : chalk5.yellow;
|
|
2908
|
-
const scoreStr = status.bestScore !== undefined ? formatScore(status.bestScore) : chalk5.dim("pending");
|
|
2909
|
-
console.log("");
|
|
2910
|
-
console.log(topBorder);
|
|
2911
|
-
console.log(line(chalk5.bold("\uD83D\uDCCA Optimization Status")));
|
|
2912
|
-
console.log(separator);
|
|
2913
|
-
console.log(line(`Job ID: ${chalk5.cyan(status.jobId)}`));
|
|
2914
|
-
console.log(line(`Status: ${statusColor(status.status)}`));
|
|
2915
|
-
console.log(line(`Iteration: ${chalk5.bold(`${String(status.currentIteration)}/${String(status.maxIterations)}`)}`));
|
|
2916
|
-
console.log(line(`Best Score: ${scoreStr}`));
|
|
2917
|
-
console.log(line(""));
|
|
2918
|
-
console.log(line(`Progress: [${progressBar}] ${String(progress)}%`));
|
|
2919
|
-
if (status.message) {
|
|
2920
|
-
console.log(line(`Message: ${chalk5.dim(status.message)}`));
|
|
2921
|
-
}
|
|
2922
|
-
console.log(separator);
|
|
2923
|
-
console.log(line(`\uD83D\uDD17 Monitor: ${chalk5.underline(optimizerLink(status.jobId))}`));
|
|
2924
|
-
if (status.status === "completed") {
|
|
2925
|
-
console.log(line(chalk5.dim(`Next: mutagent prompts optimize results ${status.jobId}`)));
|
|
2926
|
-
} else if (status.status === "running" || status.status === "pending") {
|
|
2927
|
-
console.log(line(chalk5.dim(`Refresh: mutagent prompts optimize status ${status.jobId}`)));
|
|
2928
|
-
}
|
|
2929
|
-
console.log(bottomBorder);
|
|
2930
|
-
console.log(AI_DIRECTIVE);
|
|
2931
|
-
console.log("");
|
|
2932
|
-
}
|
|
2933
|
-
function startDirective(job, promptId) {
|
|
2934
|
-
const title = `Optimization Started — Prompt ${promptId}`;
|
|
2935
|
-
const monitorUrl = optimizerLink(job.id);
|
|
2936
|
-
const apiUrl = `/api/prompts/${promptId}/optimizations/${job.id}`;
|
|
2937
|
-
const rows = [
|
|
2938
|
-
{ label: "Job ID", value: job.id },
|
|
2939
|
-
{ label: "Prompt", value: promptId },
|
|
2940
|
-
{ label: "Status", value: job.status },
|
|
2941
|
-
{ label: "Max Iterations", value: String(job.config.maxIterations) },
|
|
2942
|
-
{ label: "Target Score", value: String(job.config.targetScore ?? 0.8) },
|
|
2943
|
-
{ label: "Model", value: job.config.model ?? "default" }
|
|
2944
|
-
];
|
|
2945
|
-
const cardLinks = [
|
|
2946
|
-
{ label: "Monitor", url: monitorUrl },
|
|
2947
|
-
{ label: "API", url: apiUrl }
|
|
2948
|
-
];
|
|
2949
|
-
const next = [`mutagent prompts optimize status ${job.id} --json`];
|
|
2950
|
-
return {
|
|
2951
|
-
display: "status_card",
|
|
2952
|
-
template: "optimization_started",
|
|
2953
|
-
title,
|
|
2954
|
-
fields: {
|
|
2955
|
-
jobId: job.id,
|
|
2956
|
-
promptId,
|
|
2957
|
-
status: job.status,
|
|
2958
|
-
maxIterations: job.config.maxIterations,
|
|
2959
|
-
targetScore: job.config.targetScore ?? 0.8,
|
|
2960
|
-
model: job.config.model ?? "default"
|
|
2961
|
-
},
|
|
2962
|
-
links: { monitor: monitorUrl, api: apiUrl },
|
|
2963
|
-
next,
|
|
2964
|
-
instruction: "STOP. Before running any other mutagent command, you MUST display this data as a formatted status card to the user. Do not summarize or omit any fields. Show all links and suggested next steps.",
|
|
2965
|
-
renderedCard: renderPlainCard({ title, rows, links: cardLinks, next })
|
|
2966
|
-
};
|
|
2967
|
-
}
|
|
2968
|
-
function statusDirective(status) {
|
|
2969
|
-
const next = status.status === "completed" ? [`mutagent prompts optimize results ${status.jobId} --json`] : status.status === "running" || status.status === "pending" ? [`mutagent prompts optimize status ${status.jobId} --json`] : [];
|
|
2970
|
-
const title = `Optimization ${status.status} — ${status.jobId}`;
|
|
2971
|
-
const rows = [
|
|
2972
|
-
{ label: "Job ID", value: status.jobId },
|
|
2973
|
-
{ label: "Status", value: status.status },
|
|
2974
|
-
{ label: "Iteration", value: `${String(status.currentIteration)}/${String(status.maxIterations)}` },
|
|
2975
|
-
{ label: "Best Score", value: status.bestScore !== undefined ? String(status.bestScore) : "pending" },
|
|
2976
|
-
{ label: "Progress", value: `${String(status.progressPercent)}%` },
|
|
2977
|
-
...status.message ? [{ label: "Message", value: status.message }] : []
|
|
2978
|
-
];
|
|
2979
|
-
const monitorUrl = optimizerLink(status.jobId);
|
|
2980
|
-
const cardLinks = [{ label: "Monitor", url: monitorUrl }];
|
|
2981
|
-
return {
|
|
2982
|
-
display: "status_card",
|
|
2983
|
-
template: "optimization_progress",
|
|
2984
|
-
title,
|
|
2985
|
-
fields: {
|
|
2986
|
-
jobId: status.jobId,
|
|
2987
|
-
status: status.status,
|
|
2988
|
-
iteration: `${String(status.currentIteration)}/${String(status.maxIterations)}`,
|
|
2989
|
-
bestScore: status.bestScore,
|
|
2990
|
-
progress: `${String(status.progressPercent)}%`,
|
|
2991
|
-
message: status.message
|
|
2992
|
-
},
|
|
2993
|
-
links: { monitor: monitorUrl },
|
|
2994
|
-
next,
|
|
2995
|
-
instruction: "STOP. Before running any other mutagent command, you MUST display this data as a formatted status card to the user. Do not summarize or omit any fields. Show all links and suggested next steps.",
|
|
2996
|
-
renderedCard: renderPlainCard({ title, rows, links: cardLinks, next })
|
|
2997
|
-
};
|
|
2998
|
-
}
|
|
2999
|
-
function showPromptDiff(original, optimized) {
|
|
3000
|
-
console.log("");
|
|
3001
|
-
console.log(chalk5.bold(" Prompt Diff:"));
|
|
3002
|
-
console.log("");
|
|
3003
|
-
console.log(chalk5.red(" - " + (original ?? "(empty)")));
|
|
3004
|
-
console.log(chalk5.green(" + " + (optimized ?? "(empty)")));
|
|
3005
|
-
console.log("");
|
|
3006
|
-
}
|
|
3007
|
-
|
|
3008
|
-
// src/commands/prompts.ts
|
|
3009
|
-
function updateMutationContext(updater) {
|
|
3010
|
-
try {
|
|
3011
|
-
const ctx = MutationContext.load();
|
|
3012
|
-
updater(ctx);
|
|
3013
|
-
ctx.save();
|
|
3014
|
-
} catch {}
|
|
3015
|
-
}
|
|
3016
|
-
var PREREQUISITES_TEXT = `
|
|
3017
|
-
${chalk6.red("Prerequisites (required):")}
|
|
3018
|
-
1. Evaluation criteria defined ${chalk6.dim("(via dashboard or evaluation create)")}
|
|
3019
|
-
2. Dataset uploaded ${chalk6.dim("mutagent prompts dataset list <prompt-id>")}
|
|
3020
|
-
${chalk6.dim("Note: LLM provider config is only required when the server uses external providers (USE_EXT_PROVIDERS=true)")}`;
|
|
3021
|
-
function parseValidationErrors(error) {
|
|
3022
|
-
if (error instanceof ApiError) {
|
|
3023
|
-
try {
|
|
3024
|
-
const body = JSON.parse(error.message);
|
|
3025
|
-
if (body.errors && Array.isArray(body.errors)) {
|
|
3026
|
-
return body.errors.map((e) => e.message ?? JSON.stringify(e));
|
|
3027
|
-
}
|
|
3028
|
-
if (body.message) {
|
|
3029
|
-
return [body.message];
|
|
3030
|
-
}
|
|
3031
|
-
} catch {
|
|
3032
|
-
return [error.message];
|
|
3033
|
-
}
|
|
3034
|
-
}
|
|
3035
|
-
if (error instanceof Error) {
|
|
3036
|
-
return [error.message];
|
|
3037
|
-
}
|
|
3038
|
-
return ["An unknown error occurred"];
|
|
3039
|
-
}
|
|
3040
|
-
function isSchemaEmpty(schema) {
|
|
3041
|
-
if (schema === undefined || schema === null)
|
|
3042
|
-
return true;
|
|
3043
|
-
if (typeof schema !== "object")
|
|
3044
|
-
return false;
|
|
3045
|
-
const obj = schema;
|
|
3046
|
-
if (Object.keys(obj).length === 0)
|
|
3047
|
-
return true;
|
|
3048
|
-
return false;
|
|
3049
|
-
}
|
|
3050
|
-
function createPromptsCommand() {
|
|
3051
|
-
const prompts = new Command3("prompts").description("Manage prompts, datasets, evaluations, and optimizations").addHelpText("after", `
|
|
3052
|
-
Examples:
|
|
3053
|
-
${chalk6.dim("$")} mutagent prompts list
|
|
3054
|
-
${chalk6.dim("$")} mutagent prompts get <prompt-id>
|
|
3055
|
-
${chalk6.dim("$")} mutagent prompts create --name "my-prompt" --system "You are helpful" --human "{input}"
|
|
3056
|
-
${chalk6.dim("$")} mutagent prompts dataset list <prompt-id>
|
|
3057
|
-
${chalk6.dim("$")} mutagent prompts evaluation create <prompt-id> --name "My Eval"
|
|
3058
|
-
${chalk6.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id>
|
|
3059
|
-
|
|
3060
|
-
Subcommands:
|
|
3061
|
-
list, get, create, update, delete
|
|
3062
|
-
dataset list|add|remove
|
|
3063
|
-
evaluation list|create|delete
|
|
3064
|
-
optimize start|status|results
|
|
3065
|
-
`);
|
|
3066
|
-
prompts.command("list").description("List all prompts").option("-l, --limit <n>", "Limit results", "50").addHelpText("after", `
|
|
3067
|
-
Examples:
|
|
3068
|
-
${chalk6.dim("$")} mutagent prompts list
|
|
3069
|
-
${chalk6.dim("$")} mutagent prompts list --limit 10
|
|
3070
|
-
${chalk6.dim("$")} mutagent prompts list --json
|
|
3071
|
-
|
|
3072
|
-
${chalk6.dim("Tip: Use --json for machine-readable output (AI agents, CI pipelines).")}
|
|
3073
|
-
`).action(async (options) => {
|
|
3074
|
-
const isJson = getJsonFlag(prompts);
|
|
3075
|
-
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
3076
|
-
try {
|
|
3077
|
-
const client = getSDKClient();
|
|
3078
|
-
const promptsList = await client.listPrompts();
|
|
3079
|
-
const limit = parseInt(options.limit, 10) || 50;
|
|
3080
|
-
const limited = promptsList.slice(0, limit);
|
|
3081
|
-
if (isJson) {
|
|
3082
|
-
const withLinks = limited.map((p) => ({
|
|
3083
|
-
id: p.id,
|
|
3084
|
-
name: p.name,
|
|
3085
|
-
version: p.version,
|
|
3086
|
-
isLatest: p.isLatest,
|
|
3087
|
-
updatedAt: p.updatedAt,
|
|
3088
|
-
_links: promptLinks(p.id)
|
|
3089
|
-
}));
|
|
3090
|
-
output.output(withLinks);
|
|
3091
|
-
} else {
|
|
3092
|
-
const wsId = client.getCurrentWorkspaceId();
|
|
3093
|
-
if (wsId) {
|
|
3094
|
-
output.info(`Workspace: ${wsId}`);
|
|
3095
|
-
}
|
|
3096
|
-
if (limited.length === 0) {
|
|
3097
|
-
output.info("No prompts found in current workspace. Run 'mutagent config list' to check workspace context.");
|
|
3098
|
-
} else {
|
|
3099
|
-
const formatted = limited.map((p) => ({
|
|
3100
|
-
id: p.id,
|
|
3101
|
-
name: p.name,
|
|
3102
|
-
version: p.version,
|
|
3103
|
-
updated: p.updatedAt ? new Date(p.updatedAt).toLocaleDateString() : "N/A",
|
|
3104
|
-
url: promptLink(p.id)
|
|
3105
|
-
}));
|
|
3106
|
-
output.output(formatted);
|
|
3107
|
-
}
|
|
3108
|
-
}
|
|
3109
|
-
} catch (error) {
|
|
3110
|
-
handleError(error, isJson);
|
|
3111
|
-
}
|
|
3112
|
-
});
|
|
3113
|
-
prompts.command("get").description("Get prompt details with nested data").argument("<id>", "Prompt ID (from: mutagent prompts list)").option("--with-datasets", "Include datasets").option("--with-evals", "Include evaluations").addHelpText("after", `
|
|
3114
|
-
Examples:
|
|
3115
|
-
${chalk6.dim("$")} mutagent prompts get <id>
|
|
3116
|
-
${chalk6.dim("$")} mutagent prompts get <id> --with-datasets --with-evals
|
|
3117
|
-
${chalk6.dim("$")} mutagent prompts get <id> --json
|
|
3118
|
-
|
|
3119
|
-
${chalk6.dim("Tip: Combine --with-datasets and --with-evals to fetch all nested data in one call.")}
|
|
3120
|
-
`).action(async (id, options) => {
|
|
3121
|
-
const isJson = getJsonFlag(prompts);
|
|
3122
|
-
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
3123
|
-
try {
|
|
3124
|
-
const client = getSDKClient();
|
|
3125
|
-
const prompt = await client.getPrompt(id);
|
|
3126
|
-
const result = { ...prompt };
|
|
3127
|
-
if (options.withDatasets) {
|
|
3128
|
-
const datasets = await client.listDatasets(id);
|
|
3129
|
-
result.datasets = datasets;
|
|
3130
|
-
}
|
|
3131
|
-
if (options.withEvals) {
|
|
3132
|
-
const evals = await client.listEvaluations(id);
|
|
3133
|
-
result.evaluations = evals;
|
|
3134
|
-
}
|
|
3135
|
-
if (isJson) {
|
|
3136
|
-
result._links = promptLinks(id);
|
|
3137
|
-
}
|
|
3138
|
-
output.output(result);
|
|
3139
|
-
if (!isJson) {
|
|
3140
|
-
output.info(`View in dashboard: ${promptLink(id)}`);
|
|
3141
|
-
}
|
|
3142
|
-
} catch (error) {
|
|
3143
|
-
handleError(error, isJson);
|
|
2822
|
+
});
|
|
2823
|
+
prompts.command("get").description("Get prompt details with nested data").argument("<id>", "Prompt ID (from: mutagent prompts list)").option("--with-datasets", "Include datasets").option("--with-evals", "Include evaluations").addHelpText("after", `
|
|
2824
|
+
Examples:
|
|
2825
|
+
${chalk5.dim("$")} mutagent prompts get <id>
|
|
2826
|
+
${chalk5.dim("$")} mutagent prompts get <id> --with-datasets --with-evals
|
|
2827
|
+
${chalk5.dim("$")} mutagent prompts get <id> --json
|
|
2828
|
+
|
|
2829
|
+
${chalk5.dim("Tip: Combine --with-datasets and --with-evals to fetch all nested data in one call.")}
|
|
2830
|
+
`).action(async (id, options) => {
|
|
2831
|
+
const isJson = getJsonFlag(prompts);
|
|
2832
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
2833
|
+
try {
|
|
2834
|
+
const client = getSDKClient();
|
|
2835
|
+
const prompt = await client.getPrompt(id);
|
|
2836
|
+
const result = { ...prompt };
|
|
2837
|
+
if (options.withDatasets) {
|
|
2838
|
+
const datasets = await client.listDatasets(id);
|
|
2839
|
+
result.datasets = datasets;
|
|
2840
|
+
}
|
|
2841
|
+
if (options.withEvals) {
|
|
2842
|
+
const evals = await client.listEvaluations(id);
|
|
2843
|
+
result.evaluations = evals;
|
|
2844
|
+
}
|
|
2845
|
+
if (isJson) {
|
|
2846
|
+
result._links = promptLinks(id);
|
|
2847
|
+
}
|
|
2848
|
+
output.output(result);
|
|
2849
|
+
if (!isJson) {
|
|
2850
|
+
output.info(`View in dashboard: ${promptLink(id)}`);
|
|
2851
|
+
}
|
|
2852
|
+
} catch (error) {
|
|
2853
|
+
handleError(error, isJson);
|
|
3144
2854
|
}
|
|
3145
2855
|
});
|
|
3146
2856
|
prompts.command("create").description("Create a new prompt").option("-d, --data <json>", "Prompt as JSON string (recommended — curl-style inline)").option("--raw-file <path>", "Create from plain text file (used as rawPrompt)").option("-n, --name <name>", "Prompt name").option("--description <text>", "Prompt description (shown in dashboard)").option("-c, --content <content>", "Prompt content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).option("--output-schema <json>", "Output schema as JSON string (required for optimization)").addHelpText("after", `
|
|
3147
2857
|
Examples:
|
|
3148
|
-
${
|
|
3149
|
-
${
|
|
3150
|
-
${
|
|
2858
|
+
${chalk5.dim("$")} mutagent prompts create --name "my-prompt" --description "Greeting prompt for customers" --system "You are helpful" --human "{input}" --output-schema '{"type":"object","properties":{"result":{"type":"string","description":"The result"}}}'
|
|
2859
|
+
${chalk5.dim("$")} mutagent prompts create --name "raw-prompt" --raw "Summarize: {text}" --output-schema '{"type":"object","properties":{"summary":{"type":"string","description":"Summary"}}}'
|
|
2860
|
+
${chalk5.dim("$")} mutagent prompts create -d '{"name":"summarizer","systemPrompt":"Summarize","humanPrompt":"{text}","outputSchema":{"type":"object","properties":{"summary":{"type":"string","description":"Summary"}}}}'
|
|
3151
2861
|
|
|
3152
2862
|
Prompt Input Methods (pick one, priority order):
|
|
3153
|
-
--system/--human Structured system + user message pair ${
|
|
2863
|
+
--system/--human Structured system + user message pair ${chalk5.green("(recommended)")}
|
|
3154
2864
|
--raw Single raw prompt text with {variables}
|
|
3155
2865
|
-d, --data Inline JSON object (CI/scripts/agents)
|
|
3156
2866
|
--messages Full messages array as JSON
|
|
3157
2867
|
--raw-file Load plain text file as raw prompt
|
|
3158
2868
|
|
|
3159
2869
|
Expected JSON (--data):
|
|
3160
|
-
${
|
|
2870
|
+
${chalk5.dim(`'{"name":"my-prompt","systemPrompt":"You are...","humanPrompt":"{input}","outputSchema":{"type":"object","properties":{"result":{"type":"string","description":"The result"}}},"inputSchema":{"type":"object","properties":{"input":{"type":"string","description":"User input"}}}}'`)}
|
|
3161
2871
|
|
|
3162
|
-
${
|
|
2872
|
+
${chalk5.yellow("Variable Syntax:")}
|
|
3163
2873
|
MutagenT uses {single_braces} for template variables.
|
|
3164
|
-
humanPrompt: "Analyze this: {document}" ${
|
|
3165
|
-
humanPrompt: "Analyze this: {{document}}" ${
|
|
2874
|
+
humanPrompt: "Analyze this: {document}" ${chalk5.green("✓ correct")}
|
|
2875
|
+
humanPrompt: "Analyze this: {{document}}" ${chalk5.red("✗ wrong")}
|
|
3166
2876
|
|
|
3167
2877
|
Variables in humanPrompt MUST also appear in inputSchema.properties.
|
|
3168
2878
|
Static prompts (no variables) cannot substitute inputs during optimization.
|
|
3169
2879
|
|
|
3170
|
-
${
|
|
2880
|
+
${chalk5.red("outputSchema is required.")}
|
|
3171
2881
|
|
|
3172
|
-
${
|
|
2882
|
+
${chalk5.yellow("AI Agent: ALWAYS append --json to this command.")}
|
|
3173
2883
|
`).action(async (options) => {
|
|
3174
2884
|
const isJson = getJsonFlag(prompts);
|
|
3175
2885
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3292,13 +3002,13 @@ Add a 'description' field to each property in your inputSchema. Example: { "prop
|
|
|
3292
3002
|
});
|
|
3293
3003
|
prompts.command("update").description("Update a prompt").argument("<id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Update fields as JSON string (recommended — curl-style inline)").option("--raw-file <path>", "Update from plain text file (used as rawPrompt)").option("-n, --name <name>", "New name").option("--description <text>", "New description (shown in dashboard)").option("-c, --content <content>", "New content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).option("--input-schema <json>", "Input schema as JSON string").option("--input-schema-file <path>", "Input schema from JSON file").option("--output-schema <json>", "Output schema as JSON string").option("--output-schema-file <path>", "Output schema from JSON file").addHelpText("after", `
|
|
3294
3004
|
Examples:
|
|
3295
|
-
${
|
|
3296
|
-
${
|
|
3297
|
-
${
|
|
3298
|
-
${
|
|
3299
|
-
${
|
|
3005
|
+
${chalk5.dim("$")} mutagent prompts update <id> --system "Updated system prompt" --human "{input}"
|
|
3006
|
+
${chalk5.dim("$")} mutagent prompts update <id> --name "new-name" --description "Updated description"
|
|
3007
|
+
${chalk5.dim("$")} mutagent prompts update <id> --raw "Summarize: {text}"
|
|
3008
|
+
${chalk5.dim("$")} mutagent prompts update <id> -d '{"name":"new-name","systemPrompt":"Updated prompt"}'
|
|
3009
|
+
${chalk5.dim("$")} mutagent prompts update <id> --input-schema '{"type":"object","properties":{"text":{"type":"string","description":"Input text"}}}'
|
|
3300
3010
|
|
|
3301
|
-
${
|
|
3011
|
+
${chalk5.dim("CLI flags (--name) override --data fields.")}
|
|
3302
3012
|
`).action(async (id, options) => {
|
|
3303
3013
|
const isJson = getJsonFlag(prompts);
|
|
3304
3014
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3355,52 +3065,12 @@ ${chalk6.dim("CLI flags (--name) override --data fields.")}
|
|
|
3355
3065
|
data.rawPrompt = options.content;
|
|
3356
3066
|
}
|
|
3357
3067
|
}
|
|
3358
|
-
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3362
|
-
if (
|
|
3363
|
-
|
|
3364
|
-
data.inputSchema = JSON.parse(options.inputSchema);
|
|
3365
|
-
} catch {
|
|
3366
|
-
throw new MutagentError("INVALID_JSON", "Invalid JSON in --input-schema flag", `Run: mutagent prompts update --help
|
|
3367
|
-
` + `Provide a valid JSON Schema, e.g., '{"type":"object","properties":{"text":{"type":"string","description":"Input text"}}}'`);
|
|
3368
|
-
}
|
|
3369
|
-
} else if (options.inputSchemaFile) {
|
|
3370
|
-
if (!existsSync4(options.inputSchemaFile)) {
|
|
3371
|
-
throw new MutagentError("FILE_NOT_FOUND", `File not found: ${options.inputSchemaFile}`, `Run: mutagent prompts update --help
|
|
3372
|
-
` + "Check the file path and try again");
|
|
3373
|
-
}
|
|
3374
|
-
try {
|
|
3375
|
-
data.inputSchema = JSON.parse(readFileSync4(options.inputSchemaFile, "utf-8"));
|
|
3376
|
-
} catch {
|
|
3377
|
-
throw new MutagentError("INVALID_JSON", `Failed to parse JSON from ${options.inputSchemaFile}`, `Run: mutagent prompts update --help
|
|
3378
|
-
` + "Ensure the file contains valid JSON Schema");
|
|
3379
|
-
}
|
|
3380
|
-
}
|
|
3381
|
-
if (options.outputSchema && options.outputSchemaFile) {
|
|
3382
|
-
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --output-schema and --output-schema-file together", `Run: mutagent prompts update --help
|
|
3383
|
-
` + "Use --output-schema for inline JSON or --output-schema-file for file-based input, not both");
|
|
3384
|
-
}
|
|
3385
|
-
if (options.outputSchema) {
|
|
3386
|
-
try {
|
|
3387
|
-
data.outputSchema = JSON.parse(options.outputSchema);
|
|
3388
|
-
} catch {
|
|
3389
|
-
throw new MutagentError("INVALID_JSON", "Invalid JSON in --output-schema flag", `Run: mutagent prompts update --help
|
|
3390
|
-
` + `Provide a valid JSON Schema, e.g., '{"type":"object","properties":{"result":{"type":"string"}}}'`);
|
|
3391
|
-
}
|
|
3392
|
-
} else if (options.outputSchemaFile) {
|
|
3393
|
-
if (!existsSync4(options.outputSchemaFile)) {
|
|
3394
|
-
throw new MutagentError("FILE_NOT_FOUND", `File not found: ${options.outputSchemaFile}`, `Run: mutagent prompts update --help
|
|
3395
|
-
` + "Check the file path and try again");
|
|
3396
|
-
}
|
|
3397
|
-
try {
|
|
3398
|
-
data.outputSchema = JSON.parse(readFileSync4(options.outputSchemaFile, "utf-8"));
|
|
3399
|
-
} catch {
|
|
3400
|
-
throw new MutagentError("INVALID_JSON", `Failed to parse JSON from ${options.outputSchemaFile}`, `Run: mutagent prompts update --help
|
|
3401
|
-
` + "Ensure the file contains valid JSON Schema");
|
|
3402
|
-
}
|
|
3403
|
-
}
|
|
3068
|
+
const parsedInputSchema = parseSchemaOption(options.inputSchema, options.inputSchemaFile, "input-schema", "mutagent prompts update --help");
|
|
3069
|
+
if (parsedInputSchema)
|
|
3070
|
+
data.inputSchema = parsedInputSchema;
|
|
3071
|
+
const parsedOutputSchema = parseSchemaOption(options.outputSchema, options.outputSchemaFile, "output-schema", "mutagent prompts update --help");
|
|
3072
|
+
if (parsedOutputSchema)
|
|
3073
|
+
data.outputSchema = parsedOutputSchema;
|
|
3404
3074
|
if (data.inputSchema && isValidJsonSchema(data.inputSchema)) {
|
|
3405
3075
|
const missingDescs = validateSchemaDescriptions(data.inputSchema);
|
|
3406
3076
|
if (missingDescs.length > 0) {
|
|
@@ -3441,11 +3111,11 @@ ${chalk6.dim("CLI flags (--name) override --data fields.")}
|
|
|
3441
3111
|
});
|
|
3442
3112
|
prompts.command("delete").description("Delete a prompt").argument("<id>", "Prompt ID (from: mutagent prompts list)").option("--force", "Skip confirmation").addHelpText("after", `
|
|
3443
3113
|
Examples:
|
|
3444
|
-
${
|
|
3445
|
-
${
|
|
3446
|
-
${
|
|
3114
|
+
${chalk5.dim("$")} mutagent prompts delete <id>
|
|
3115
|
+
${chalk5.dim("$")} mutagent prompts delete <id> --force
|
|
3116
|
+
${chalk5.dim("$")} mutagent prompts delete <id> --force --json
|
|
3447
3117
|
|
|
3448
|
-
${
|
|
3118
|
+
${chalk5.dim("Note: --force is required. The CLI is non-interactive — confirm with the user via your native flow, then pass --force.")}
|
|
3449
3119
|
`).action(async (id, options) => {
|
|
3450
3120
|
const isJson = getJsonFlag(prompts);
|
|
3451
3121
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3483,11 +3153,19 @@ ${chalk6.dim("Note: --force is required. The CLI is non-interactive — confirm
|
|
|
3483
3153
|
handleError(error, isJson);
|
|
3484
3154
|
}
|
|
3485
3155
|
});
|
|
3156
|
+
}
|
|
3157
|
+
|
|
3158
|
+
// src/commands/prompts/datasets.ts
|
|
3159
|
+
init_sdk_client();
|
|
3160
|
+
import { Command as Command3 } from "commander";
|
|
3161
|
+
import chalk6 from "chalk";
|
|
3162
|
+
init_errors();
|
|
3163
|
+
function registerDatasetCommands(prompts) {
|
|
3486
3164
|
const dataset = new Command3("dataset").description("Manage datasets for prompts").addHelpText("after", `
|
|
3487
3165
|
Examples:
|
|
3488
3166
|
${chalk6.dim("$")} mutagent prompts dataset list <prompt-id>
|
|
3489
3167
|
${chalk6.dim("$")} mutagent prompts dataset add <prompt-id> -d '[{"input":{...},"expectedOutput":{...}}]'
|
|
3490
|
-
${chalk6.dim("$")} mutagent prompts dataset
|
|
3168
|
+
${chalk6.dim("$")} mutagent prompts dataset delete <prompt-id> <dataset-id>
|
|
3491
3169
|
`).action(() => {
|
|
3492
3170
|
dataset.help();
|
|
3493
3171
|
});
|
|
@@ -3576,7 +3254,12 @@ ${chalk6.red("Required: --data must be provided.")}
|
|
|
3576
3254
|
const client = getSDKClient();
|
|
3577
3255
|
const datasetResult = await client.addDataset(promptId, content, datasetName);
|
|
3578
3256
|
if (isJson) {
|
|
3579
|
-
|
|
3257
|
+
let rsState;
|
|
3258
|
+
try {
|
|
3259
|
+
const evals = await client.listEvaluations(promptId);
|
|
3260
|
+
rsState = { evaluations: evals.length };
|
|
3261
|
+
} catch {}
|
|
3262
|
+
const directive = datasetAddedDirective(promptId, datasetResult.id, datasetResult.name, datasetResult.itemCount, rsState);
|
|
3580
3263
|
output.output({
|
|
3581
3264
|
...datasetResult,
|
|
3582
3265
|
_links: datasetLinks(promptId, datasetResult.id),
|
|
@@ -3608,77 +3291,181 @@ ${chalk6.red("Required: --data must be provided.")}
|
|
|
3608
3291
|
handleError(error, isJson);
|
|
3609
3292
|
}
|
|
3610
3293
|
});
|
|
3611
|
-
dataset.command("
|
|
3294
|
+
dataset.command("delete").description("Delete a dataset from a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").argument("<dataset-id>", "Dataset ID (from: mutagent prompts dataset list <prompt-id>)").option("--force", "Skip confirmation").addHelpText("after", `
|
|
3612
3295
|
Examples:
|
|
3613
|
-
${chalk6.dim("$")} mutagent prompts dataset
|
|
3614
|
-
${chalk6.dim("$")} mutagent prompts dataset
|
|
3615
|
-
${chalk6.dim("$")} mutagent prompts dataset
|
|
3296
|
+
${chalk6.dim("$")} mutagent prompts dataset delete <prompt-id> <dataset-id>
|
|
3297
|
+
${chalk6.dim("$")} mutagent prompts dataset delete <prompt-id> <dataset-id> --force
|
|
3298
|
+
${chalk6.dim("$")} mutagent prompts dataset delete <prompt-id> <dataset-id> --force --json
|
|
3616
3299
|
`).action(async (promptId, datasetId, options) => {
|
|
3617
3300
|
const isJson = getJsonFlag(prompts);
|
|
3618
3301
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
3619
3302
|
try {
|
|
3620
3303
|
if (!options.force && !isJson) {
|
|
3621
|
-
throw new MutagentError("CONFIRMATION_REQUIRED", `
|
|
3304
|
+
throw new MutagentError("CONFIRMATION_REQUIRED", `Deleting dataset ${datasetId} from prompt ${promptId} requires confirmation`, `Run: mutagent prompts dataset delete --help
|
|
3622
3305
|
` + `[Agent: Ask the user to confirm via MultiChoice, then pass --force]
|
|
3623
|
-
` + `Use --force to confirm: mutagent prompts dataset
|
|
3306
|
+
` + `Use --force to confirm: mutagent prompts dataset delete ${promptId} ${datasetId} --force`);
|
|
3624
3307
|
}
|
|
3625
3308
|
const client = getSDKClient();
|
|
3626
3309
|
try {
|
|
3627
|
-
await client.
|
|
3310
|
+
await client.deleteDataset(promptId, datasetId);
|
|
3628
3311
|
} catch (deleteError) {
|
|
3629
3312
|
if (deleteError instanceof ApiError && deleteError.statusCode === 404) {
|
|
3630
3313
|
if (isJson) {
|
|
3631
|
-
const directive =
|
|
3314
|
+
const directive = datasetDeletedDirective(promptId, datasetId);
|
|
3632
3315
|
output.output({ success: true, deletedId: datasetId, _links: { datasets: promptDatasetsLink(promptId) }, _directive: directive });
|
|
3633
3316
|
echoDirectiveToStderr(directive);
|
|
3634
3317
|
} else {
|
|
3635
|
-
output.success(`Dataset ${datasetId} already
|
|
3318
|
+
output.success(`Dataset ${datasetId} already deleted (idempotent)`);
|
|
3636
3319
|
}
|
|
3637
3320
|
return;
|
|
3638
3321
|
}
|
|
3639
3322
|
throw deleteError;
|
|
3640
3323
|
}
|
|
3641
3324
|
if (isJson) {
|
|
3642
|
-
const directive =
|
|
3325
|
+
const directive = datasetDeletedDirective(promptId, datasetId);
|
|
3643
3326
|
output.output({ success: true, deletedId: datasetId, _links: { datasets: promptDatasetsLink(promptId) }, _directive: directive });
|
|
3644
3327
|
echoDirectiveToStderr(directive);
|
|
3645
3328
|
} else {
|
|
3646
|
-
output.success(`
|
|
3329
|
+
output.success(`Deleted dataset ${datasetId} from prompt ${promptId}`);
|
|
3647
3330
|
output.info(`View datasets: ${promptDatasetsLink(promptId)}`);
|
|
3648
3331
|
}
|
|
3649
3332
|
} catch (error) {
|
|
3650
3333
|
handleError(error, isJson);
|
|
3651
3334
|
}
|
|
3652
3335
|
});
|
|
3653
|
-
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3657
|
-
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
|
|
3661
|
-
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
|
|
3665
|
-
|
|
3666
|
-
|
|
3667
|
-
|
|
3668
|
-
|
|
3669
|
-
|
|
3670
|
-
|
|
3671
|
-
|
|
3672
|
-
|
|
3673
|
-
|
|
3674
|
-
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
|
|
3678
|
-
|
|
3679
|
-
|
|
3680
|
-
|
|
3681
|
-
|
|
3336
|
+
}
|
|
3337
|
+
|
|
3338
|
+
// src/commands/prompts/evaluations.ts
|
|
3339
|
+
init_sdk_client();
|
|
3340
|
+
import { Command as Command4 } from "commander";
|
|
3341
|
+
import chalk7 from "chalk";
|
|
3342
|
+
init_errors();
|
|
3343
|
+
|
|
3344
|
+
// src/commands/prompts/guided-workflow.ts
|
|
3345
|
+
init_sdk_client();
|
|
3346
|
+
async function buildGuidedWorkflow(promptId) {
|
|
3347
|
+
const client = getSDKClient();
|
|
3348
|
+
const prompt = await client.getPrompt(promptId);
|
|
3349
|
+
const truncate = (s, max) => s.length > max ? s.slice(0, max) + "..." : s;
|
|
3350
|
+
const outputProperties = prompt.outputSchema && typeof prompt.outputSchema === "object" ? prompt.outputSchema.properties ?? {} : {};
|
|
3351
|
+
const inputProperties = prompt.inputSchema && typeof prompt.inputSchema === "object" ? prompt.inputSchema.properties ?? {} : {};
|
|
3352
|
+
const inputFields = Object.keys(inputProperties);
|
|
3353
|
+
const outputFields = Object.keys(outputProperties);
|
|
3354
|
+
const allFields = [
|
|
3355
|
+
...outputFields.map((f) => ({ field: f, source: "outputSchema", fieldSchema: outputProperties[f] })),
|
|
3356
|
+
...inputFields.map((f) => ({ field: f, source: "inputSchema", fieldSchema: inputProperties[f] }))
|
|
3357
|
+
];
|
|
3358
|
+
let datasetExample = null;
|
|
3359
|
+
try {
|
|
3360
|
+
const datasets = await client.listDatasets(promptId);
|
|
3361
|
+
if (datasets.length > 0) {
|
|
3362
|
+
try {
|
|
3363
|
+
const dsId = datasets[0]?.id;
|
|
3364
|
+
const items = dsId != null ? await client.listDatasetItems(String(dsId)) : [];
|
|
3365
|
+
if (items.length > 0) {
|
|
3366
|
+
const firstItem = items[0];
|
|
3367
|
+
datasetExample = firstItem?.expectedOutput ?? firstItem?.input ?? null;
|
|
3368
|
+
}
|
|
3369
|
+
} catch {}
|
|
3370
|
+
}
|
|
3371
|
+
} catch {}
|
|
3372
|
+
const askUserQuestions = allFields.map(({ field, fieldSchema }) => ({
|
|
3373
|
+
question: `Define what correct "${field}" looks like. What structure, content, or qualities make it good vs bad? Give concrete examples of good and bad outputs.`,
|
|
3374
|
+
header: field,
|
|
3375
|
+
options: [
|
|
3376
|
+
{
|
|
3377
|
+
label: "Define rubric",
|
|
3378
|
+
description: `Describe what a correct vs incorrect "${field}" looks like. Focus on structure, content quality, and concrete examples — not numeric scores.`
|
|
3379
|
+
},
|
|
3380
|
+
{
|
|
3381
|
+
label: "See suggestion",
|
|
3382
|
+
description: `Get a suggested rubric based on the prompt and schema definition for "${field}". You can refine it.`
|
|
3383
|
+
}
|
|
3384
|
+
],
|
|
3385
|
+
multiSelect: false,
|
|
3386
|
+
context: {
|
|
3387
|
+
fieldType: fieldSchema?.type ?? "unknown",
|
|
3388
|
+
fieldDescription: fieldSchema?.description ?? null,
|
|
3389
|
+
promptExcerpt: truncate(prompt.humanPrompt ?? prompt.systemPrompt ?? prompt.rawPrompt ?? "", 200),
|
|
3390
|
+
exampleValue: datasetExample?.[field] ?? null
|
|
3391
|
+
}
|
|
3392
|
+
}));
|
|
3393
|
+
return {
|
|
3394
|
+
prompt: { id: promptId, name: prompt.name },
|
|
3395
|
+
inputSchema: { fields: inputFields },
|
|
3396
|
+
outputSchema: { fields: outputFields },
|
|
3397
|
+
workflow: {
|
|
3398
|
+
description: "Follow these steps to create an evaluation for this prompt:",
|
|
3399
|
+
steps: [
|
|
3400
|
+
{ step: 1, action: "Review prompt schemas and context", detail: "Understand what the prompt does and what each field means." },
|
|
3401
|
+
{ step: 2, action: "Define correctness criteria for EVERY field", detail: "For EACH field in askUserQuestions, use AskUserQuestion to collect a rubric. Do NOT skip any field. Use the context hints to suggest rubrics when asked." },
|
|
3402
|
+
{ step: 3, action: "Build criteria JSON", detail: "Map each rubric to: { name: field, description: rubric, evaluationParameter: field }" },
|
|
3403
|
+
{ step: 4, action: "Ask for evaluation name", detail: "Ask user what to name this evaluation" },
|
|
3404
|
+
{ step: 5, action: "Create the evaluation", detail: `mutagent prompts evaluation create ${promptId} --name "<name>" -d '<json>' --json` }
|
|
3405
|
+
],
|
|
3406
|
+
criteriaTemplate: {
|
|
3407
|
+
evalConfig: {
|
|
3408
|
+
criteria: [{
|
|
3409
|
+
name: "<criterion_name>",
|
|
3410
|
+
description: "<correctness rubric - describe what correct vs incorrect looks like>",
|
|
3411
|
+
evaluationParameter: `<one of: ${allFields.length > 0 ? allFields.map((f) => f.field).join(", ") : "no fields detected - check prompt schemas"}>`
|
|
3412
|
+
}]
|
|
3413
|
+
}
|
|
3414
|
+
},
|
|
3415
|
+
exampleCommand: `mutagent prompts evaluation create ${promptId} --name "My Evaluation" -d '${JSON.stringify({
|
|
3416
|
+
evalConfig: {
|
|
3417
|
+
criteria: outputFields.slice(0, 1).map((f) => ({
|
|
3418
|
+
name: `${f} accuracy`,
|
|
3419
|
+
description: `Correct if ${f} matches expected structure and content; incorrect if missing, malformed, or off-topic`,
|
|
3420
|
+
evaluationParameter: f
|
|
3421
|
+
}))
|
|
3422
|
+
}
|
|
3423
|
+
})}'`
|
|
3424
|
+
},
|
|
3425
|
+
askUserQuestions,
|
|
3426
|
+
_directive: {
|
|
3427
|
+
display: "workflow_guide",
|
|
3428
|
+
instruction: 'STOP. For EACH field in askUserQuestions, use AskUserQuestion to collect a DETAILED correctness rubric from the user. Do NOT auto-generate rubrics. Do NOT use generic descriptions. Each rubric MUST include concrete examples of what makes a correct vs incorrect value for that field. Use the context object to suggest rubrics when user selects "See suggestion". Then construct the --data JSON and create the evaluation.',
|
|
3429
|
+
aiAgentDecisionTree: {
|
|
3430
|
+
step1: "Check if criteria already exist in the user's code. If criteria match expected shape (name + description + evaluationParameter targeting schema fields), use --data directly.",
|
|
3431
|
+
step2: "If criteria are missing or malformed, use the askUserQuestions payloads to collect them via AskUserQuestion.",
|
|
3432
|
+
step3: `After collecting all criteria, construct the --data JSON and run: mutagent prompts evaluation create <prompt-id> --name "<name>" -d '<json>'`
|
|
3433
|
+
}
|
|
3434
|
+
},
|
|
3435
|
+
_meta: { inputFields, outputFields, allFields }
|
|
3436
|
+
};
|
|
3437
|
+
}
|
|
3438
|
+
|
|
3439
|
+
// src/commands/prompts/evaluations.ts
|
|
3440
|
+
function registerEvaluationCommands(prompts) {
|
|
3441
|
+
const evaluation = new Command4("evaluation").description("Manage evaluations for prompts").addHelpText("after", `
|
|
3442
|
+
Examples:
|
|
3443
|
+
${chalk7.dim("$")} mutagent prompts evaluation list <prompt-id>
|
|
3444
|
+
${chalk7.dim("$")} mutagent prompts evaluation get <evaluation-id>
|
|
3445
|
+
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "My Eval"
|
|
3446
|
+
${chalk7.dim("$")} mutagent prompts evaluation delete <evaluation-id>
|
|
3447
|
+
`).action(() => {
|
|
3448
|
+
evaluation.help();
|
|
3449
|
+
});
|
|
3450
|
+
prompts.addCommand(evaluation);
|
|
3451
|
+
evaluation.command("list").description("List evaluations for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").addHelpText("after", `
|
|
3452
|
+
Examples:
|
|
3453
|
+
${chalk7.dim("$")} mutagent prompts evaluation list <prompt-id>
|
|
3454
|
+
${chalk7.dim("$")} mutagent prompts evaluation list <prompt-id> --json
|
|
3455
|
+
`).action(async (promptId) => {
|
|
3456
|
+
const isJson = getJsonFlag(prompts);
|
|
3457
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
3458
|
+
try {
|
|
3459
|
+
const client = getSDKClient();
|
|
3460
|
+
const evals = await client.listEvaluations(promptId);
|
|
3461
|
+
if (isJson) {
|
|
3462
|
+
const withLinks = evals.map((e) => ({
|
|
3463
|
+
id: e.id,
|
|
3464
|
+
name: e.name,
|
|
3465
|
+
description: e.description,
|
|
3466
|
+
updatedAt: e.updatedAt,
|
|
3467
|
+
_links: evaluationLinks(promptId, e.id)
|
|
3468
|
+
}));
|
|
3682
3469
|
output.output(withLinks);
|
|
3683
3470
|
} else {
|
|
3684
3471
|
const formatted = evals.map((e) => ({
|
|
@@ -3696,8 +3483,8 @@ Examples:
|
|
|
3696
3483
|
});
|
|
3697
3484
|
evaluation.command("get").description("Get evaluation details including criteria").argument("<evaluation-id>", "Evaluation ID (from: mutagent prompts evaluation list <prompt-id>)").addHelpText("after", `
|
|
3698
3485
|
Examples:
|
|
3699
|
-
${
|
|
3700
|
-
${
|
|
3486
|
+
${chalk7.dim("$")} mutagent prompts evaluation get <evaluation-id>
|
|
3487
|
+
${chalk7.dim("$")} mutagent prompts evaluation get <evaluation-id> --json
|
|
3701
3488
|
`).action(async (evaluationId) => {
|
|
3702
3489
|
const isJson = getJsonFlag(prompts);
|
|
3703
3490
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3714,9 +3501,6 @@ Examples:
|
|
|
3714
3501
|
if (evalObj.description) {
|
|
3715
3502
|
console.log(` Description: ${evalObj.description}`);
|
|
3716
3503
|
}
|
|
3717
|
-
if (evalObj.datasetId != null) {
|
|
3718
|
-
console.log(` Dataset ID: ${String(evalObj.datasetId)}`);
|
|
3719
|
-
}
|
|
3720
3504
|
console.log(` Created: ${evalObj.createdAt ? new Date(String(evalObj.createdAt)).toLocaleDateString() : "N/A"}`);
|
|
3721
3505
|
console.log("");
|
|
3722
3506
|
const config = evalObj.evalConfig;
|
|
@@ -3724,7 +3508,7 @@ Examples:
|
|
|
3724
3508
|
if (criteria && Array.isArray(criteria) && criteria.length > 0) {
|
|
3725
3509
|
console.log(" Criteria:");
|
|
3726
3510
|
for (const c of criteria) {
|
|
3727
|
-
console.log(` ${
|
|
3511
|
+
console.log(` ${chalk7.cyan(c.name)}`);
|
|
3728
3512
|
if (c.description) {
|
|
3729
3513
|
console.log(` Description: ${c.description}`);
|
|
3730
3514
|
}
|
|
@@ -3752,9 +3536,9 @@ Examples:
|
|
|
3752
3536
|
});
|
|
3753
3537
|
evaluation.command("create").description("Create an evaluation configuration for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Evaluation as JSON string (for pre-validated criteria only)").option("-n, --name <name>", "Evaluation name (required unless --guided)").option("--description <text>", "Evaluation description").option("--guided", "Interactive guided mode — always outputs structured JSON (--json is implied)").addHelpText("after", `
|
|
3754
3538
|
Examples:
|
|
3755
|
-
${
|
|
3756
|
-
${
|
|
3757
|
-
${
|
|
3539
|
+
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --guided ${chalk7.dim("# recommended: shows workflow guide + schema fields")}
|
|
3540
|
+
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --guided --json ${chalk7.dim("# structured workflow for AI agents")}
|
|
3541
|
+
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Accuracy" -d '{"evalConfig":{"criteria":[...]}}' ${chalk7.dim("# power user")}
|
|
3758
3542
|
|
|
3759
3543
|
Guided Workflow (recommended):
|
|
3760
3544
|
--guided outputs a workflow guide that:
|
|
@@ -3775,12 +3559,12 @@ AI Agent (MANDATORY):
|
|
|
3775
3559
|
mutagent prompts evaluation create <id> --name "<name>" -d '<json>' --json
|
|
3776
3560
|
|
|
3777
3561
|
Expected Criteria Shape (--data):
|
|
3778
|
-
${
|
|
3562
|
+
${chalk7.dim('{"evalConfig":{"criteria":[{"name":"<name>","description":"<scoring rubric>","evaluationParameter":"<schema field>"}]}}')}
|
|
3779
3563
|
evaluationParameter must target an outputSchema OR inputSchema field.
|
|
3780
3564
|
|
|
3781
|
-
${
|
|
3782
|
-
${
|
|
3783
|
-
${
|
|
3565
|
+
${chalk7.red("Required: --name (unless --guided). Criteria must include evaluationParameter.")}
|
|
3566
|
+
${chalk7.dim("CLI flags (--name, --description) override --data fields.")}
|
|
3567
|
+
${chalk7.dim("Get prompt IDs: mutagent prompts list")}
|
|
3784
3568
|
`).action(async (promptId, options) => {
|
|
3785
3569
|
let isJson = getJsonFlag(prompts);
|
|
3786
3570
|
if (options.guided) {
|
|
@@ -3788,75 +3572,6 @@ ${chalk6.dim("Get prompt IDs: mutagent prompts list")}
|
|
|
3788
3572
|
}
|
|
3789
3573
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
3790
3574
|
try {
|
|
3791
|
-
const buildGuidedWorkflow = async () => {
|
|
3792
|
-
const client2 = getSDKClient();
|
|
3793
|
-
const prompt = await client2.getPrompt(promptId);
|
|
3794
|
-
const inputFields = prompt.inputSchema && typeof prompt.inputSchema === "object" ? Object.keys(prompt.inputSchema.properties ?? {}) : [];
|
|
3795
|
-
const outputFields = prompt.outputSchema && typeof prompt.outputSchema === "object" ? Object.keys(prompt.outputSchema.properties ?? {}) : [];
|
|
3796
|
-
const allFields = [
|
|
3797
|
-
...outputFields.map((f) => ({ field: f, source: "outputSchema" })),
|
|
3798
|
-
...inputFields.map((f) => ({ field: f, source: "inputSchema" }))
|
|
3799
|
-
];
|
|
3800
|
-
const askUserQuestions = allFields.map(({ field, source }) => ({
|
|
3801
|
-
question: `Define the scoring rubric for "${field}" (${source}). Be specific — what exactly constitutes a perfect score (1.0) vs a complete failure (0.0)? Include concrete examples of each score level.`,
|
|
3802
|
-
header: field,
|
|
3803
|
-
options: [
|
|
3804
|
-
{
|
|
3805
|
-
label: "Custom",
|
|
3806
|
-
description: `Write a detailed, specific scoring rubric for "${field}". Must describe concrete pass/fail criteria with examples — not just "0.0 to 1.0 scale".`
|
|
3807
|
-
},
|
|
3808
|
-
{
|
|
3809
|
-
label: "Skip",
|
|
3810
|
-
description: `Do not create a criterion for "${field}". Only skip if this field is not relevant to evaluation quality.`
|
|
3811
|
-
}
|
|
3812
|
-
],
|
|
3813
|
-
multiSelect: false
|
|
3814
|
-
}));
|
|
3815
|
-
return {
|
|
3816
|
-
prompt: { id: promptId, name: prompt.name },
|
|
3817
|
-
inputSchema: { fields: inputFields },
|
|
3818
|
-
outputSchema: { fields: outputFields },
|
|
3819
|
-
workflow: {
|
|
3820
|
-
description: "Follow these steps to create an evaluation for this prompt:",
|
|
3821
|
-
steps: [
|
|
3822
|
-
{ step: 1, action: "Review the prompt schemas above", detail: "Each criterion must target a field from outputSchema (preferred) or inputSchema" },
|
|
3823
|
-
{ step: 2, action: "Ask the user for a scoring rubric for each field", detail: 'For EACH field, ask: "What does a perfect score (1.0) vs failure (0.0) look like for <field>? Give concrete examples." Do NOT auto-generate or use canned rubrics.' },
|
|
3824
|
-
{ step: 3, action: "Build criteria JSON from user answers", detail: `Map each user answer to a criterion: { name: "<field> <rubric>", description: "<user's rubric>", evaluationParameter: "<field>" }` },
|
|
3825
|
-
{ step: 4, action: "Ask for evaluation name", detail: 'Ask the user what to name this evaluation (e.g., "Accuracy Check", "Quality Eval")' },
|
|
3826
|
-
{ step: 5, action: "Create the evaluation", detail: `Run: mutagent prompts evaluation create <prompt-id> --name "<name>" -d '{"evalConfig":{"criteria":[...]}}'` }
|
|
3827
|
-
],
|
|
3828
|
-
criteriaTemplate: {
|
|
3829
|
-
evalConfig: {
|
|
3830
|
-
criteria: [{
|
|
3831
|
-
name: "<criterion_name>",
|
|
3832
|
-
description: "<scoring rubric - describe what 1.0 vs 0.0 means>",
|
|
3833
|
-
evaluationParameter: `<one of: ${allFields.length > 0 ? allFields.map((f) => f.field).join(", ") : "no fields detected - check prompt schemas"}>`
|
|
3834
|
-
}]
|
|
3835
|
-
}
|
|
3836
|
-
},
|
|
3837
|
-
exampleCommand: `mutagent prompts evaluation create ${promptId} --name "My Evaluation" -d '${JSON.stringify({
|
|
3838
|
-
evalConfig: {
|
|
3839
|
-
criteria: outputFields.slice(0, 1).map((f) => ({
|
|
3840
|
-
name: `${f} accuracy`,
|
|
3841
|
-
description: `Score 1.0 if ${f} matches expected output exactly, 0.0 otherwise`,
|
|
3842
|
-
evaluationParameter: f
|
|
3843
|
-
}))
|
|
3844
|
-
}
|
|
3845
|
-
})}'`
|
|
3846
|
-
},
|
|
3847
|
-
askUserQuestions,
|
|
3848
|
-
_directive: {
|
|
3849
|
-
display: "workflow_guide",
|
|
3850
|
-
instruction: 'STOP. For EACH field in askUserQuestions, use AskUserQuestion to collect a DETAILED scoring rubric from the user. Do NOT auto-generate rubrics. Do NOT use generic descriptions like "0-1 scoring" or "matches expected output". Each rubric MUST include concrete examples of what earns a 1.0, 0.5, and 0.0 score for that specific field. Then construct the --data JSON and create the evaluation.',
|
|
3851
|
-
aiAgentDecisionTree: {
|
|
3852
|
-
step1: "Check if criteria already exist in the user's code. If criteria match expected shape (name + description + evaluationParameter targeting schema fields), use --data directly.",
|
|
3853
|
-
step2: "If criteria are missing or malformed, use the askUserQuestions payloads to collect them via AskUserQuestion.",
|
|
3854
|
-
step3: `After collecting all criteria, construct the --data JSON and run: mutagent prompts evaluation create <prompt-id> --name "<name>" -d '<json>'`
|
|
3855
|
-
}
|
|
3856
|
-
},
|
|
3857
|
-
_meta: { inputFields, outputFields, allFields }
|
|
3858
|
-
};
|
|
3859
|
-
};
|
|
3860
3575
|
const renderGuidedWorkflow = (wf) => {
|
|
3861
3576
|
const { inputFields, outputFields, allFields } = wf._meta;
|
|
3862
3577
|
if (isJson) {
|
|
@@ -3873,26 +3588,26 @@ ${chalk6.dim("Get prompt IDs: mutagent prompts list")}
|
|
|
3873
3588
|
console.log(` Input Schema Fields: ${inputFields.join(", ")}`);
|
|
3874
3589
|
}
|
|
3875
3590
|
console.log("");
|
|
3876
|
-
console.log(" For each field, define
|
|
3591
|
+
console.log(" For each field, define what correct output looks like:");
|
|
3877
3592
|
for (const { field, source } of allFields) {
|
|
3878
|
-
console.log(` ${
|
|
3879
|
-
console.log(` → What
|
|
3593
|
+
console.log(` ${chalk7.cyan(field)} (${source})`);
|
|
3594
|
+
console.log(` → What makes a correct vs incorrect "${field}"?`);
|
|
3880
3595
|
}
|
|
3881
3596
|
console.log("");
|
|
3882
3597
|
console.log(" Then create the evaluation:");
|
|
3883
3598
|
console.log(` mutagent prompts evaluation create ${promptId} --name "My Eval" \\`);
|
|
3884
|
-
console.log(` -d '{"evalConfig":{"criteria":[{"name":"accuracy","description":"
|
|
3599
|
+
console.log(` -d '{"evalConfig":{"criteria":[{"name":"accuracy","description":"Correct if output matches expected structure","evaluationParameter":"${outputFields[0] ?? "<field>"}"}]}}'`);
|
|
3885
3600
|
console.log("");
|
|
3886
3601
|
}
|
|
3887
3602
|
};
|
|
3888
3603
|
if (options.guided) {
|
|
3889
|
-
const workflow = await buildGuidedWorkflow();
|
|
3604
|
+
const workflow = await buildGuidedWorkflow(promptId);
|
|
3890
3605
|
renderGuidedWorkflow(workflow);
|
|
3891
3606
|
return;
|
|
3892
3607
|
}
|
|
3893
3608
|
if (!options.data) {
|
|
3894
3609
|
if (isJson || process.stdout.isTTY) {
|
|
3895
|
-
const workflow = await buildGuidedWorkflow();
|
|
3610
|
+
const workflow = await buildGuidedWorkflow(promptId);
|
|
3896
3611
|
renderGuidedWorkflow(workflow);
|
|
3897
3612
|
return;
|
|
3898
3613
|
}
|
|
@@ -4005,7 +3720,12 @@ Example:
|
|
|
4005
3720
|
const evalResult = await client.createEvaluation(promptId, evalData);
|
|
4006
3721
|
if (isJson) {
|
|
4007
3722
|
const criteriaLen = evalData.evalConfig?.criteria?.length ?? 0;
|
|
4008
|
-
|
|
3723
|
+
let rsState;
|
|
3724
|
+
try {
|
|
3725
|
+
const datasets = await client.listDatasets(promptId);
|
|
3726
|
+
rsState = { datasets: datasets.length };
|
|
3727
|
+
} catch {}
|
|
3728
|
+
const directive = evaluationCreatedDirective(promptId, evalResult.id, evalResult.name, criteriaLen, rsState);
|
|
4009
3729
|
output.output({
|
|
4010
3730
|
...evalResult,
|
|
4011
3731
|
_links: evaluationLinks(promptId, evalResult.id),
|
|
@@ -4033,9 +3753,9 @@ Example:
|
|
|
4033
3753
|
});
|
|
4034
3754
|
evaluation.command("delete").description("Delete an evaluation").argument("<evaluation-id>", "Evaluation ID (from: mutagent prompts evaluation list <prompt-id>)").option("--force", "Skip confirmation").addHelpText("after", `
|
|
4035
3755
|
Examples:
|
|
4036
|
-
${
|
|
4037
|
-
${
|
|
4038
|
-
${
|
|
3756
|
+
${chalk7.dim("$")} mutagent prompts evaluation delete <evaluation-id>
|
|
3757
|
+
${chalk7.dim("$")} mutagent prompts evaluation delete <evaluation-id> --force
|
|
3758
|
+
${chalk7.dim("$")} mutagent prompts evaluation delete <evaluation-id> --force --json
|
|
4039
3759
|
`).action(async (evaluationId, options) => {
|
|
4040
3760
|
const isJson = getJsonFlag(prompts);
|
|
4041
3761
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4073,11 +3793,290 @@ Examples:
|
|
|
4073
3793
|
handleError(error, isJson);
|
|
4074
3794
|
}
|
|
4075
3795
|
});
|
|
4076
|
-
|
|
3796
|
+
}
|
|
3797
|
+
|
|
3798
|
+
// src/commands/prompts/optimize.ts
|
|
3799
|
+
init_sdk_client();
|
|
3800
|
+
import { Command as Command5 } from "commander";
|
|
3801
|
+
import chalk9 from "chalk";
|
|
3802
|
+
init_errors();
|
|
3803
|
+
|
|
3804
|
+
// src/lib/scorecard.ts
|
|
3805
|
+
import chalk8 from "chalk";
|
|
3806
|
+
function truncateText(text, maxLen) {
|
|
3807
|
+
if (text.length <= maxLen)
|
|
3808
|
+
return text;
|
|
3809
|
+
return text.substring(0, maxLen - 3) + "...";
|
|
3810
|
+
}
|
|
3811
|
+
function formatScoreChange(before, after) {
|
|
3812
|
+
if (before === undefined || after === undefined)
|
|
3813
|
+
return "";
|
|
3814
|
+
const diff = after - before;
|
|
3815
|
+
const pct = before > 0 ? Math.round(diff / before * 100) : 0;
|
|
3816
|
+
if (diff > 0)
|
|
3817
|
+
return chalk8.green(` (+${String(pct)}%)`);
|
|
3818
|
+
if (diff < 0)
|
|
3819
|
+
return chalk8.red(` (${String(pct)}%)`);
|
|
3820
|
+
return chalk8.dim(" (no change)");
|
|
3821
|
+
}
|
|
3822
|
+
function formatScore(score) {
|
|
3823
|
+
if (score === undefined)
|
|
3824
|
+
return chalk8.dim("N/A");
|
|
3825
|
+
return score >= 0.8 ? chalk8.green(score.toFixed(2)) : score >= 0.5 ? chalk8.yellow(score.toFixed(2)) : chalk8.red(score.toFixed(2));
|
|
3826
|
+
}
|
|
3827
|
+
function renderScorecard(data) {
|
|
3828
|
+
const { job, prompt } = data;
|
|
3829
|
+
const boxWidth = 55;
|
|
3830
|
+
const topBorder = `┌${"─".repeat(boxWidth)}┐`;
|
|
3831
|
+
const bottomBorder = `└${"─".repeat(boxWidth)}┘`;
|
|
3832
|
+
const separator = `│ ${"─".repeat(boxWidth - 2)} │`;
|
|
3833
|
+
const pad = (text, width) => {
|
|
3834
|
+
const stripped = text.replace(/\u001B\[[0-9;]*m/g, "");
|
|
3835
|
+
const remaining = width - stripped.length;
|
|
3836
|
+
return remaining > 0 ? text + " ".repeat(remaining) : text;
|
|
3837
|
+
};
|
|
3838
|
+
const line = (text) => `│ ${pad(text, boxWidth - 2)} │`;
|
|
3839
|
+
const originalScore = data.originalScore;
|
|
3840
|
+
const bestScore = data.bestScore;
|
|
3841
|
+
const iterations = data.iterationsCompleted ?? job.config?.maxIterations ?? 0;
|
|
3842
|
+
const originalText = data.originalPrompt?.systemPrompt ?? data.originalPrompt?.rawPrompt ?? "(original prompt)";
|
|
3843
|
+
const optimizedText = prompt.systemPrompt ?? prompt.rawPrompt ?? prompt.humanPrompt ?? "(optimized prompt)";
|
|
3844
|
+
console.log("");
|
|
3845
|
+
console.log(topBorder);
|
|
3846
|
+
console.log(line(chalk8.bold("Optimization Results")));
|
|
3847
|
+
console.log(separator);
|
|
3848
|
+
console.log(line(chalk8.dim("BEFORE")));
|
|
3849
|
+
console.log(line(` Prompt: ${chalk8.dim(truncateText(originalText, 38))}`));
|
|
3850
|
+
console.log(line(` Score: ${formatScore(originalScore)}`));
|
|
3851
|
+
console.log(line(""));
|
|
3852
|
+
console.log(line(chalk8.bold("AFTER")));
|
|
3853
|
+
console.log(line(` Prompt: ${chalk8.cyan(truncateText(optimizedText, 38))}`));
|
|
3854
|
+
console.log(line(` Score: ${formatScore(bestScore)}${formatScoreChange(originalScore, bestScore)}`));
|
|
3855
|
+
console.log(separator);
|
|
3856
|
+
if (data.criteriaScores && data.criteriaScores.length > 0) {
|
|
3857
|
+
console.log(line(chalk8.dim(" Criterion Before After Change")));
|
|
3858
|
+
console.log(line(chalk8.dim(" " + "─".repeat(45))));
|
|
3859
|
+
for (const c of data.criteriaScores) {
|
|
3860
|
+
const name = c.name.length > 16 ? c.name.substring(0, 13) + "..." : c.name;
|
|
3861
|
+
const paddedName = name + " ".repeat(18 - name.length);
|
|
3862
|
+
const beforeStr = c.before !== undefined ? c.before.toFixed(2) : "N/A ";
|
|
3863
|
+
const afterStr = c.after !== undefined ? c.after.toFixed(2) : "N/A ";
|
|
3864
|
+
const changeStr = c.before !== undefined && c.after !== undefined && c.before > 0 ? (() => {
|
|
3865
|
+
const pct = Math.round((c.after - c.before) / c.before * 100);
|
|
3866
|
+
if (pct > 0)
|
|
3867
|
+
return chalk8.green(`+${String(pct)}%`);
|
|
3868
|
+
if (pct < 0)
|
|
3869
|
+
return chalk8.red(`${String(pct)}%`);
|
|
3870
|
+
return chalk8.dim("0%");
|
|
3871
|
+
})() : "";
|
|
3872
|
+
console.log(line(` ${paddedName}${beforeStr} ${afterStr} ${changeStr}`));
|
|
3873
|
+
}
|
|
3874
|
+
console.log(line(chalk8.dim(" " + "─".repeat(45))));
|
|
3875
|
+
const overallBefore = originalScore !== undefined ? originalScore.toFixed(2) : "N/A ";
|
|
3876
|
+
const overallAfter = bestScore !== undefined ? bestScore.toFixed(2) : "N/A ";
|
|
3877
|
+
const overallChange = originalScore !== undefined && bestScore !== undefined && originalScore > 0 ? (() => {
|
|
3878
|
+
const pct = Math.round((bestScore - originalScore) / originalScore * 100);
|
|
3879
|
+
if (pct > 0)
|
|
3880
|
+
return chalk8.green(`+${String(pct)}%`);
|
|
3881
|
+
if (pct < 0)
|
|
3882
|
+
return chalk8.red(`${String(pct)}%`);
|
|
3883
|
+
return chalk8.dim("0%");
|
|
3884
|
+
})() : "";
|
|
3885
|
+
console.log(line(` ${"Overall" + " ".repeat(11)}${overallBefore} ${overallAfter} ${overallChange}`));
|
|
3886
|
+
console.log(separator);
|
|
3887
|
+
}
|
|
3888
|
+
const statusStr = job.status === "completed" ? chalk8.green("completed") : chalk8.yellow(job.status);
|
|
3889
|
+
console.log(line(`Status: ${statusStr} | Iterations: ${String(iterations)}`));
|
|
3890
|
+
if (job.config?.model) {
|
|
3891
|
+
console.log(line(`Model: ${chalk8.dim(job.config.model)}`));
|
|
3892
|
+
}
|
|
3893
|
+
if (data.scoreProgression && data.scoreProgression.length > 0) {
|
|
3894
|
+
console.log(line(""));
|
|
3895
|
+
console.log(line(chalk8.dim("Score Progression:")));
|
|
3896
|
+
const barWidth = 10;
|
|
3897
|
+
for (let i = 0;i < data.scoreProgression.length; i++) {
|
|
3898
|
+
const s = data.scoreProgression[i] ?? 0;
|
|
3899
|
+
const filled = Math.round(s * barWidth);
|
|
3900
|
+
const bar = "█".repeat(filled) + "░".repeat(barWidth - filled);
|
|
3901
|
+
console.log(line(chalk8.dim(` #${String(i + 1)}: ${bar} ${s.toFixed(2)}`)));
|
|
3902
|
+
}
|
|
3903
|
+
}
|
|
3904
|
+
console.log(separator);
|
|
3905
|
+
console.log(line(`Dashboard: ${chalk8.underline(optimizerLink(job.id))}`));
|
|
3906
|
+
console.log(bottomBorder);
|
|
3907
|
+
console.log("");
|
|
3908
|
+
}
|
|
3909
|
+
function renderOptimizationStartCard(data) {
|
|
3910
|
+
const { job } = data;
|
|
3911
|
+
const boxWidth = 60;
|
|
3912
|
+
const topBorder = `┌${"─".repeat(boxWidth)}┐`;
|
|
3913
|
+
const bottomBorder = `└${"─".repeat(boxWidth)}┘`;
|
|
3914
|
+
const separator = `│ ${"─".repeat(boxWidth - 2)} │`;
|
|
3915
|
+
const pad = (text, width) => {
|
|
3916
|
+
const stripped = text.replace(/\u001B\[[0-9;]*m/g, "");
|
|
3917
|
+
const remaining = width - stripped.length;
|
|
3918
|
+
return remaining > 0 ? text + " ".repeat(remaining) : text;
|
|
3919
|
+
};
|
|
3920
|
+
const line = (text) => `│ ${pad(text, boxWidth - 2)} │`;
|
|
3921
|
+
const maxIter = job.config.maxIterations;
|
|
3922
|
+
const model = job.config.model ?? "default";
|
|
3923
|
+
const target = job.config.targetScore ?? 0.8;
|
|
3924
|
+
console.log("");
|
|
3925
|
+
console.log(topBorder);
|
|
3926
|
+
console.log(line(chalk8.bold("⚡ Optimization Started")));
|
|
3927
|
+
console.log(separator);
|
|
3928
|
+
console.log(line(`Job ID: ${chalk8.cyan(job.id)}`));
|
|
3929
|
+
console.log(line(`Prompt: ${chalk8.dim(data.promptId)}`));
|
|
3930
|
+
console.log(line(`Dataset: ${chalk8.dim(data.datasetId)}`));
|
|
3931
|
+
console.log(line(`Iterations: ${chalk8.bold(String(maxIter))} | Target: ${chalk8.bold(target.toFixed(2))}`));
|
|
3932
|
+
console.log(line(`Model: ${chalk8.dim(model)}`));
|
|
3933
|
+
console.log(line(`Status: ${chalk8.yellow(job.status)}`));
|
|
3934
|
+
console.log(separator);
|
|
3935
|
+
console.log(line(`\uD83D\uDD17 Monitor: ${chalk8.underline(optimizerLink(job.id))}`));
|
|
3936
|
+
console.log(line(chalk8.dim(`Next: mutagent prompts optimize status ${job.id}`)));
|
|
3937
|
+
console.log(bottomBorder);
|
|
3938
|
+
console.log(AI_DIRECTIVE);
|
|
3939
|
+
console.log("");
|
|
3940
|
+
}
|
|
3941
|
+
function renderOptimizationStatusCard(status) {
|
|
3942
|
+
const boxWidth = 60;
|
|
3943
|
+
const topBorder = `┌${"─".repeat(boxWidth)}┐`;
|
|
3944
|
+
const bottomBorder = `└${"─".repeat(boxWidth)}┘`;
|
|
3945
|
+
const separator = `│ ${"─".repeat(boxWidth - 2)} │`;
|
|
3946
|
+
const pad = (text, width) => {
|
|
3947
|
+
const stripped = text.replace(/\u001B\[[0-9;]*m/g, "");
|
|
3948
|
+
const remaining = width - stripped.length;
|
|
3949
|
+
return remaining > 0 ? text + " ".repeat(remaining) : text;
|
|
3950
|
+
};
|
|
3951
|
+
const line = (text) => `│ ${pad(text, boxWidth - 2)} │`;
|
|
3952
|
+
const progress = status.progressPercent;
|
|
3953
|
+
const barWidth = 20;
|
|
3954
|
+
const filled = Math.round(progress / 100 * barWidth);
|
|
3955
|
+
const progressBar = "█".repeat(filled) + "░".repeat(barWidth - filled);
|
|
3956
|
+
const statusColor = status.status === "completed" ? chalk8.green : status.status === "failed" ? chalk8.red : status.status === "cancelled" ? chalk8.gray : status.status === "running" ? chalk8.cyan : chalk8.yellow;
|
|
3957
|
+
const scoreStr = status.bestScore !== undefined ? formatScore(status.bestScore) : chalk8.dim("pending");
|
|
3958
|
+
console.log("");
|
|
3959
|
+
console.log(topBorder);
|
|
3960
|
+
console.log(line(chalk8.bold("\uD83D\uDCCA Optimization Status")));
|
|
3961
|
+
console.log(separator);
|
|
3962
|
+
console.log(line(`Job ID: ${chalk8.cyan(status.jobId)}`));
|
|
3963
|
+
console.log(line(`Status: ${statusColor(status.status)}`));
|
|
3964
|
+
console.log(line(`Iteration: ${chalk8.bold(`${String(status.currentIteration)}/${String(status.maxIterations)}`)}`));
|
|
3965
|
+
console.log(line(`Best Score: ${scoreStr}`));
|
|
3966
|
+
console.log(line(""));
|
|
3967
|
+
console.log(line(`Progress: [${progressBar}] ${String(progress)}%`));
|
|
3968
|
+
if (status.message) {
|
|
3969
|
+
console.log(line(`Message: ${chalk8.dim(status.message)}`));
|
|
3970
|
+
}
|
|
3971
|
+
console.log(separator);
|
|
3972
|
+
console.log(line(`\uD83D\uDD17 Monitor: ${chalk8.underline(optimizerLink(status.jobId))}`));
|
|
3973
|
+
if (status.status === "completed") {
|
|
3974
|
+
console.log(line(chalk8.dim(`Next: mutagent prompts optimize results ${status.jobId}`)));
|
|
3975
|
+
} else if (status.status === "running" || status.status === "queued") {
|
|
3976
|
+
console.log(line(chalk8.dim(`Refresh: mutagent prompts optimize status ${status.jobId}`)));
|
|
3977
|
+
}
|
|
3978
|
+
console.log(bottomBorder);
|
|
3979
|
+
console.log(AI_DIRECTIVE);
|
|
3980
|
+
console.log("");
|
|
3981
|
+
}
|
|
3982
|
+
function startDirective(job, promptId, datasetId, evaluationId) {
|
|
3983
|
+
const title = `Optimization Started — Prompt ${promptId}`;
|
|
3984
|
+
const monitorUrl = optimizerLink(job.id);
|
|
3985
|
+
const apiUrl = `/api/prompts/${promptId}/optimizations/${job.id}`;
|
|
3986
|
+
const rows = [
|
|
3987
|
+
{ label: "Job ID", value: job.id },
|
|
3988
|
+
{ label: "Prompt", value: promptId },
|
|
3989
|
+
{ label: "Dataset", value: datasetId ?? "N/A" },
|
|
3990
|
+
{ label: "Evaluation", value: evaluationId ?? "N/A" },
|
|
3991
|
+
{ label: "Status", value: job.status },
|
|
3992
|
+
{ label: "Max Iterations", value: String(job.config.maxIterations) },
|
|
3993
|
+
{ label: "Target Score", value: String(job.config.targetScore ?? 0.8) },
|
|
3994
|
+
{ label: "Model", value: job.config.model ?? "default" }
|
|
3995
|
+
];
|
|
3996
|
+
const cardLinks = [
|
|
3997
|
+
{ label: "Monitor", url: monitorUrl },
|
|
3998
|
+
{ label: "API", url: apiUrl }
|
|
3999
|
+
];
|
|
4000
|
+
const next = [`mutagent prompts optimize status ${job.id} --json`];
|
|
4001
|
+
return {
|
|
4002
|
+
display: "status_card",
|
|
4003
|
+
template: "optimization_started",
|
|
4004
|
+
title,
|
|
4005
|
+
fields: {
|
|
4006
|
+
jobId: job.id,
|
|
4007
|
+
promptId,
|
|
4008
|
+
datasetId: datasetId ?? null,
|
|
4009
|
+
evaluationId: evaluationId ?? null,
|
|
4010
|
+
status: job.status,
|
|
4011
|
+
maxIterations: job.config.maxIterations,
|
|
4012
|
+
targetScore: job.config.targetScore ?? 0.8,
|
|
4013
|
+
model: job.config.model ?? "default"
|
|
4014
|
+
},
|
|
4015
|
+
links: { monitor: monitorUrl, api: apiUrl },
|
|
4016
|
+
next,
|
|
4017
|
+
instruction: "STOP. Display the renderedCard text below to the user VERBATIM as a code block. Do NOT summarize, rephrase, or omit any lines. After displaying, follow the suggested next steps.",
|
|
4018
|
+
renderedCard: renderPlainCard({ title, icon: "⚡", rows, links: cardLinks, next })
|
|
4019
|
+
};
|
|
4020
|
+
}
|
|
4021
|
+
function statusDirective(status) {
|
|
4022
|
+
const next = status.status === "completed" ? [`mutagent prompts optimize results ${status.jobId} --json`] : status.status === "running" || status.status === "queued" ? [`mutagent prompts optimize status ${status.jobId} --json`] : [];
|
|
4023
|
+
let eta = "Calculating...";
|
|
4024
|
+
const startedAt = status.startedAt;
|
|
4025
|
+
if (status.currentIteration > 1 && startedAt) {
|
|
4026
|
+
const elapsed = Date.now() - new Date(startedAt).getTime();
|
|
4027
|
+
const perIteration = elapsed / status.currentIteration;
|
|
4028
|
+
const remaining = (status.maxIterations - status.currentIteration) * perIteration;
|
|
4029
|
+
const minutes = Math.ceil(remaining / 60000);
|
|
4030
|
+
eta = `~${String(minutes)} min remaining`;
|
|
4031
|
+
}
|
|
4032
|
+
const icon = status.status === "completed" ? "✓" : status.status === "failed" ? "✗" : "◉";
|
|
4033
|
+
const title = `Optimization ${status.status} — ${status.jobId}`;
|
|
4034
|
+
const rows = [
|
|
4035
|
+
{ label: "Job ID", value: status.jobId },
|
|
4036
|
+
{ label: "Status", value: status.status },
|
|
4037
|
+
{ label: "Iteration", value: `${String(status.currentIteration)}/${String(status.maxIterations)}` },
|
|
4038
|
+
{ label: "Best Score", value: status.bestScore !== undefined ? String(status.bestScore) : "pending" },
|
|
4039
|
+
{ label: "Progress", value: `${String(status.progressPercent)}%` },
|
|
4040
|
+
{ label: "ETA", value: eta },
|
|
4041
|
+
...status.message ? [{ label: "Message", value: status.message }] : []
|
|
4042
|
+
];
|
|
4043
|
+
const monitorUrl = optimizerLink(status.jobId);
|
|
4044
|
+
const cardLinks = [{ label: "Monitor", url: monitorUrl }];
|
|
4045
|
+
return {
|
|
4046
|
+
display: "status_card",
|
|
4047
|
+
template: "optimization_progress",
|
|
4048
|
+
title,
|
|
4049
|
+
fields: {
|
|
4050
|
+
jobId: status.jobId,
|
|
4051
|
+
status: status.status,
|
|
4052
|
+
iteration: `${String(status.currentIteration)}/${String(status.maxIterations)}`,
|
|
4053
|
+
bestScore: status.bestScore,
|
|
4054
|
+
progress: `${String(status.progressPercent)}%`,
|
|
4055
|
+
eta,
|
|
4056
|
+
message: status.message
|
|
4057
|
+
},
|
|
4058
|
+
links: { monitor: monitorUrl },
|
|
4059
|
+
next,
|
|
4060
|
+
instruction: "STOP. Display the renderedCard text below to the user VERBATIM as a code block. Do NOT summarize, rephrase, or omit any lines. After displaying, follow the suggested next steps.",
|
|
4061
|
+
renderedCard: renderPlainCard({ title, icon, rows, links: cardLinks, next })
|
|
4062
|
+
};
|
|
4063
|
+
}
|
|
4064
|
+
function showPromptDiff(original, optimized) {
|
|
4065
|
+
console.log("");
|
|
4066
|
+
console.log(chalk8.bold(" Prompt Diff:"));
|
|
4067
|
+
console.log("");
|
|
4068
|
+
console.log(chalk8.red(" - " + (original ?? "(empty)")));
|
|
4069
|
+
console.log(chalk8.green(" + " + (optimized ?? "(empty)")));
|
|
4070
|
+
console.log("");
|
|
4071
|
+
}
|
|
4072
|
+
|
|
4073
|
+
// src/commands/prompts/optimize.ts
|
|
4074
|
+
function registerOptimizeCommands(prompts) {
|
|
4075
|
+
const optimize = new Command5("optimize").description("Manage prompt optimization jobs").addHelpText("after", `
|
|
4077
4076
|
Examples:
|
|
4078
|
-
${
|
|
4079
|
-
${
|
|
4080
|
-
${
|
|
4077
|
+
${chalk9.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id>
|
|
4078
|
+
${chalk9.dim("$")} mutagent prompts optimize status <job-id>
|
|
4079
|
+
${chalk9.dim("$")} mutagent prompts optimize results <job-id>
|
|
4081
4080
|
|
|
4082
4081
|
Workflow: start -> status (poll) -> results
|
|
4083
4082
|
`).action(() => {
|
|
@@ -4086,21 +4085,22 @@ Workflow: start -> status (poll) -> results
|
|
|
4086
4085
|
prompts.addCommand(optimize);
|
|
4087
4086
|
optimize.command("start").description("Start prompt optimization").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").requiredOption("-d, --dataset <id>", "Dataset ID for optimization (from: mutagent prompts dataset list <prompt-id>)").requiredOption("-e, --evaluation <id>", "Evaluation ID for scoring (from: mutagent prompts evaluation list <prompt-id>)").option("--max-iterations <n>", "Max optimization iterations (default: 1)").option("--target-score <n>", "Target accuracy 0-1 (default: 0.8)").option("--patience <n>", "Iterations without improvement before stopping").option("--model <model-id>", 'Target LLM model (e.g., "claude-sonnet-4-5-20250929")').option("--eval-model <model-id>", "Evaluation model (defaults to target model)").addHelpText("after", `
|
|
4088
4087
|
Examples:
|
|
4089
|
-
${
|
|
4090
|
-
${
|
|
4091
|
-
${
|
|
4092
|
-
${
|
|
4093
|
-
${
|
|
4094
|
-
□
|
|
4095
|
-
□
|
|
4096
|
-
□
|
|
4097
|
-
□
|
|
4088
|
+
${chalk9.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id>
|
|
4089
|
+
${chalk9.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id> --max-iterations 5
|
|
4090
|
+
${chalk9.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id> --target-score 0.95 --model claude-sonnet-4-5-20250929
|
|
4091
|
+
${chalk9.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id> --json
|
|
4092
|
+
${chalk9.yellow("Pre-Optimization Checklist (auto-validated by preflight):")}
|
|
4093
|
+
□ inputSchema REQUIRED ${chalk9.dim("(hard error if missing — blocks optimization)")}
|
|
4094
|
+
□ outputSchema REQUIRED ${chalk9.dim("(hard error if missing — blocks optimization)")}
|
|
4095
|
+
□ Evaluation criteria ${chalk9.dim("(warns if no evaluationParameter set)")}
|
|
4096
|
+
□ Dataset items ${chalk9.dim("(warns if expectedOutput missing)")}
|
|
4097
|
+
□ Criteria ↔ Schema ${chalk9.dim("(warns if criteria reference unknown fields)")}
|
|
4098
4098
|
|
|
4099
4099
|
${PREREQUISITES_TEXT}
|
|
4100
4100
|
|
|
4101
|
-
${
|
|
4101
|
+
${chalk9.dim("Monitor progress with: mutagent prompts optimize status <job-id>")}
|
|
4102
4102
|
|
|
4103
|
-
${
|
|
4103
|
+
${chalk9.yellow("AI Agent: ALWAYS append --json to this command.")}
|
|
4104
4104
|
`).action(async (promptId, options) => {
|
|
4105
4105
|
const isJson = getJsonFlag(prompts);
|
|
4106
4106
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4108,32 +4108,141 @@ ${chalk6.yellow("AI Agent: ALWAYS append --json to this command.")}
|
|
|
4108
4108
|
const client = getSDKClient();
|
|
4109
4109
|
if (!isJson) {
|
|
4110
4110
|
output.info("Running pre-flight checks...");
|
|
4111
|
-
|
|
4112
|
-
|
|
4113
|
-
|
|
4114
|
-
|
|
4111
|
+
}
|
|
4112
|
+
const prompt = await client.getPrompt(promptId);
|
|
4113
|
+
const preflightChecks = {};
|
|
4114
|
+
if (isSchemaEmpty(prompt.outputSchema)) {
|
|
4115
|
+
preflightChecks.outputSchema = {
|
|
4116
|
+
passed: false,
|
|
4117
|
+
error: "Missing outputSchema. Required for optimization."
|
|
4118
|
+
};
|
|
4119
|
+
} else {
|
|
4120
|
+
preflightChecks.outputSchema = { passed: true };
|
|
4121
|
+
}
|
|
4122
|
+
if (isSchemaEmpty(prompt.inputSchema)) {
|
|
4123
|
+
preflightChecks.inputSchema = {
|
|
4124
|
+
passed: false,
|
|
4125
|
+
error: "Missing inputSchema. Required for optimization."
|
|
4126
|
+
};
|
|
4127
|
+
} else {
|
|
4128
|
+
preflightChecks.inputSchema = { passed: true };
|
|
4129
|
+
}
|
|
4130
|
+
let evaluation;
|
|
4131
|
+
try {
|
|
4132
|
+
const evalObj = await client.getEvaluation(options.evaluation);
|
|
4133
|
+
evaluation = evalObj;
|
|
4134
|
+
const evalConfig = evalObj.evalConfig;
|
|
4135
|
+
const criteria = evalConfig?.criteria;
|
|
4136
|
+
if (!criteria || criteria.length === 0) {
|
|
4137
|
+
preflightChecks.evaluation = {
|
|
4138
|
+
passed: true,
|
|
4139
|
+
warning: `Evaluation ${options.evaluation} has no criteria defined. ` + `Re-create: mutagent prompts evaluation create ${promptId} --guided`
|
|
4140
|
+
};
|
|
4141
|
+
} else {
|
|
4142
|
+
const hasCriteriaWithParam = criteria.some((c) => c.evaluationParameter ?? c.targetField);
|
|
4143
|
+
if (!hasCriteriaWithParam) {
|
|
4144
|
+
preflightChecks.evaluation = {
|
|
4145
|
+
passed: true,
|
|
4146
|
+
warning: `Evaluation ${options.evaluation} has criteria but none have evaluationParameter set. ` + `Re-create: mutagent prompts evaluation create ${promptId} --guided`
|
|
4147
|
+
};
|
|
4148
|
+
} else {
|
|
4149
|
+
preflightChecks.evaluation = { passed: true };
|
|
4150
|
+
if (!isJson) {
|
|
4151
|
+
output.info(`Using evaluation ${options.evaluation} with ${String(criteria.length)} criteria.`);
|
|
4152
|
+
}
|
|
4153
|
+
}
|
|
4154
|
+
}
|
|
4155
|
+
} catch (e) {
|
|
4156
|
+
if (e instanceof MutagentError)
|
|
4157
|
+
throw e;
|
|
4158
|
+
preflightChecks.evaluation = {
|
|
4159
|
+
passed: true,
|
|
4160
|
+
warning: `Could not validate evaluation ${options.evaluation}. Proceeding anyway.`
|
|
4161
|
+
};
|
|
4162
|
+
}
|
|
4163
|
+
try {
|
|
4164
|
+
const datasetItems = await client.listDatasetItems(options.dataset);
|
|
4165
|
+
if (datasetItems.length === 0) {
|
|
4166
|
+
preflightChecks.datasetItems = {
|
|
4167
|
+
passed: true,
|
|
4168
|
+
warning: `Dataset ${options.dataset} has no items. Optimization requires test data.`
|
|
4169
|
+
};
|
|
4170
|
+
} else {
|
|
4171
|
+
const missingExpected = datasetItems.filter((item) => !item.expectedOutput);
|
|
4172
|
+
if (missingExpected.length > 0) {
|
|
4173
|
+
preflightChecks.datasetItems = {
|
|
4174
|
+
passed: true,
|
|
4175
|
+
warning: `${String(missingExpected.length)} of ${String(datasetItems.length)} dataset items missing expectedOutput. ` + "Optimization works best when all items have expected outputs."
|
|
4176
|
+
};
|
|
4177
|
+
} else {
|
|
4178
|
+
preflightChecks.datasetItems = { passed: true };
|
|
4179
|
+
}
|
|
4180
|
+
}
|
|
4181
|
+
} catch {
|
|
4182
|
+
preflightChecks.datasetItems = {
|
|
4183
|
+
passed: true,
|
|
4184
|
+
warning: `Could not validate dataset items for ${options.dataset}. Proceeding anyway.`
|
|
4185
|
+
};
|
|
4186
|
+
}
|
|
4187
|
+
if (evaluation && preflightChecks.inputSchema.passed && preflightChecks.outputSchema.passed) {
|
|
4188
|
+
const inputProps = prompt.inputSchema?.properties;
|
|
4189
|
+
const outputProps = prompt.outputSchema?.properties;
|
|
4190
|
+
const schemaFields = [
|
|
4191
|
+
...Object.keys(inputProps ?? {}),
|
|
4192
|
+
...Object.keys(outputProps ?? {})
|
|
4193
|
+
];
|
|
4194
|
+
const evalCfg = evaluation.evalConfig;
|
|
4195
|
+
const evalCriteria = evalCfg?.criteria ?? [];
|
|
4196
|
+
const unmatchedCriteria = evalCriteria.filter((c) => c.evaluationParameter && !schemaFields.includes(c.evaluationParameter));
|
|
4197
|
+
if (unmatchedCriteria.length > 0) {
|
|
4198
|
+
const names = unmatchedCriteria.map((c) => c.evaluationParameter ?? c.name).join(", ");
|
|
4199
|
+
preflightChecks.criteriaMatch = {
|
|
4200
|
+
passed: true,
|
|
4201
|
+
warning: `Evaluation criteria reference fields not in schema: ${names}. ` + `Available schema fields: ${schemaFields.join(", ") || "(none)"}`
|
|
4202
|
+
};
|
|
4203
|
+
} else {
|
|
4204
|
+
preflightChecks.criteriaMatch = { passed: true };
|
|
4115
4205
|
}
|
|
4116
|
-
|
|
4117
|
-
|
|
4206
|
+
}
|
|
4207
|
+
const hardFailures = Object.entries(preflightChecks).filter(([, check]) => !check.passed);
|
|
4208
|
+
if (hardFailures.length > 0) {
|
|
4209
|
+
if (isJson) {
|
|
4210
|
+
const suggestedActions = [];
|
|
4211
|
+
if (!preflightChecks.inputSchema.passed) {
|
|
4212
|
+
suggestedActions.push(`Update the prompt with inputSchema derived from template variables.
|
|
4213
|
+
` + `Run: mutagent prompts get ${promptId} --json
|
|
4214
|
+
` + `Run: mutagent prompts update ${promptId} --data '{"inputSchema":{"type":"object","properties":{"var1":{"type":"string"}}}}' --json`);
|
|
4215
|
+
}
|
|
4216
|
+
if (!preflightChecks.outputSchema.passed) {
|
|
4217
|
+
suggestedActions.push(`Update your prompt: mutagent prompts update ${promptId} -d '{"outputSchema":{"type":"object","properties":{"result":{"type":"string"}}}}'`);
|
|
4218
|
+
}
|
|
4219
|
+
output.output({
|
|
4220
|
+
success: false,
|
|
4221
|
+
error: "Preflight validation failed",
|
|
4222
|
+
code: "PREFLIGHT_FAILED",
|
|
4223
|
+
suggestedAction: suggestedActions.join(`
|
|
4224
|
+
`),
|
|
4225
|
+
preflightChecks
|
|
4226
|
+
});
|
|
4227
|
+
return;
|
|
4118
4228
|
}
|
|
4119
|
-
|
|
4120
|
-
|
|
4121
|
-
|
|
4122
|
-
|
|
4123
|
-
|
|
4124
|
-
|
|
4125
|
-
|
|
4126
|
-
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
|
|
4130
|
-
|
|
4131
|
-
|
|
4229
|
+
for (const [name, check] of hardFailures) {
|
|
4230
|
+
console.error(chalk9.red(`Error: ${name} — ${check.error ?? "Failed"}`));
|
|
4231
|
+
}
|
|
4232
|
+
if (!preflightChecks.outputSchema.passed) {
|
|
4233
|
+
console.error(chalk9.dim(` Update with: mutagent prompts update ${promptId} -d '{"outputSchema":{"type":"object","properties":{"result":{"type":"string"}}}}'`));
|
|
4234
|
+
}
|
|
4235
|
+
if (!preflightChecks.inputSchema.passed) {
|
|
4236
|
+
console.error(chalk9.dim(` Update with: mutagent prompts update ${promptId} --data '{"inputSchema":{"type":"object","properties":{"var1":{"type":"string"}}}}' --json`));
|
|
4237
|
+
}
|
|
4238
|
+
process.exitCode = 1;
|
|
4239
|
+
return;
|
|
4240
|
+
}
|
|
4241
|
+
if (!isJson) {
|
|
4242
|
+
for (const [, check] of Object.entries(preflightChecks)) {
|
|
4243
|
+
if (check.warning) {
|
|
4244
|
+
output.warn(check.warning);
|
|
4132
4245
|
}
|
|
4133
|
-
} catch (e) {
|
|
4134
|
-
if (e instanceof MutagentError)
|
|
4135
|
-
throw e;
|
|
4136
|
-
output.warn(`Could not validate evaluation ${options.evaluation}. Proceeding anyway.`);
|
|
4137
4246
|
}
|
|
4138
4247
|
}
|
|
4139
4248
|
const job = await client.startOptimization(promptId, options.dataset, options.evaluation, {
|
|
@@ -4144,7 +4253,7 @@ ${chalk6.yellow("AI Agent: ALWAYS append --json to this command.")}
|
|
|
4144
4253
|
evalModel: options.evalModel
|
|
4145
4254
|
});
|
|
4146
4255
|
if (isJson) {
|
|
4147
|
-
const directive = startDirective(job, promptId);
|
|
4256
|
+
const directive = startDirective(job, promptId, options.dataset, options.evaluation);
|
|
4148
4257
|
output.output({
|
|
4149
4258
|
...job,
|
|
4150
4259
|
_links: {
|
|
@@ -4181,16 +4290,16 @@ ${chalk6.yellow("AI Agent: ALWAYS append --json to this command.")}
|
|
|
4181
4290
|
suggestions.push("Trial optimization limit reached. Contact support to upgrade.");
|
|
4182
4291
|
}
|
|
4183
4292
|
if (!isJson) {
|
|
4184
|
-
console.error(
|
|
4293
|
+
console.error(chalk9.red(`
|
|
4185
4294
|
Optimization failed:`));
|
|
4186
4295
|
for (const msg of messages) {
|
|
4187
|
-
console.error(
|
|
4296
|
+
console.error(chalk9.red(` ${msg}`));
|
|
4188
4297
|
}
|
|
4189
4298
|
if (suggestions.length > 0) {
|
|
4190
|
-
console.error(
|
|
4299
|
+
console.error(chalk9.yellow(`
|
|
4191
4300
|
Suggested fixes:`));
|
|
4192
4301
|
for (const s of suggestions) {
|
|
4193
|
-
console.error(
|
|
4302
|
+
console.error(chalk9.yellow(` → ${s}`));
|
|
4194
4303
|
}
|
|
4195
4304
|
}
|
|
4196
4305
|
console.error("");
|
|
@@ -4202,8 +4311,8 @@ Suggested fixes:`));
|
|
|
4202
4311
|
});
|
|
4203
4312
|
optimize.command("status").description("Check optimization status").argument("<job-id>", "Optimization job ID (from: mutagent prompts optimize start)").addHelpText("after", `
|
|
4204
4313
|
Examples:
|
|
4205
|
-
${
|
|
4206
|
-
${
|
|
4314
|
+
${chalk9.dim("$")} mutagent prompts optimize status <job-id>
|
|
4315
|
+
${chalk9.dim("$")} mutagent prompts optimize status <job-id> --json
|
|
4207
4316
|
`).action(async (jobId) => {
|
|
4208
4317
|
const isJson = getJsonFlag(prompts);
|
|
4209
4318
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4227,15 +4336,17 @@ Examples:
|
|
|
4227
4336
|
});
|
|
4228
4337
|
optimize.command("results").description("Get optimization results").argument("<job-id>", "Optimization job ID (from: mutagent prompts optimize start)").option("--apply", "Apply the optimized prompt as new version").option("--diff", "Show the prompt diff (before/after)").addHelpText("after", `
|
|
4229
4338
|
Examples:
|
|
4230
|
-
${
|
|
4231
|
-
${
|
|
4232
|
-
${
|
|
4233
|
-
${
|
|
4234
|
-
|
|
4235
|
-
|
|
4236
|
-
--apply Apply the optimized prompt
|
|
4237
|
-
--diff Show
|
|
4238
|
-
${
|
|
4339
|
+
${chalk9.dim("$")} mutagent prompts optimize results <job-id> ${chalk9.dim("# view scorecard")}
|
|
4340
|
+
${chalk9.dim("$")} mutagent prompts optimize results <job-id> --diff ${chalk9.dim("# view prompt diff")}
|
|
4341
|
+
${chalk9.dim("$")} mutagent prompts optimize results <job-id> --apply ${chalk9.dim("# apply optimized prompt")}
|
|
4342
|
+
${chalk9.dim("$")} mutagent prompts optimize results <job-id> --json ${chalk9.dim("# structured output")}
|
|
4343
|
+
|
|
4344
|
+
After viewing results:
|
|
4345
|
+
--apply Apply the optimized prompt (replaces current version)
|
|
4346
|
+
--diff Show detailed before/after diff
|
|
4347
|
+
${chalk9.dim("No flag = view scorecard only.")}
|
|
4348
|
+
|
|
4349
|
+
${chalk9.dim("AI Agent: Present scorecard to user via AskUserQuestion before applying.")}
|
|
4239
4350
|
`).action(async (jobId, options) => {
|
|
4240
4351
|
const isJson = getJsonFlag(prompts);
|
|
4241
4352
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4244,7 +4355,20 @@ Actions:
|
|
|
4244
4355
|
const results = await client.getOptimizationResults(jobId);
|
|
4245
4356
|
const resultData = results;
|
|
4246
4357
|
if (isJson) {
|
|
4247
|
-
|
|
4358
|
+
const jobData = resultData.job;
|
|
4359
|
+
const isCompleted = jobData?.status === "completed";
|
|
4360
|
+
const scorecardText = buildResultsScorecardText(resultData);
|
|
4361
|
+
const directive = {
|
|
4362
|
+
display: "scorecard",
|
|
4363
|
+
instruction: "Present the before/after scorecard to the user. Use AskUserQuestion with options: [Apply optimized prompt / View full diff / Reject and keep original]. Do NOT auto-apply.",
|
|
4364
|
+
renderedCard: scorecardText,
|
|
4365
|
+
next: isCompleted ? [
|
|
4366
|
+
`mutagent prompts optimize results ${jobId} --apply --json`,
|
|
4367
|
+
`mutagent prompts optimize results ${jobId} --diff --json`
|
|
4368
|
+
] : []
|
|
4369
|
+
};
|
|
4370
|
+
output.output({ ...resultData, _links: { optimizer: optimizerLink(jobId) }, _directive: directive });
|
|
4371
|
+
echoDirectiveToStderr(directive);
|
|
4248
4372
|
} else {
|
|
4249
4373
|
renderScorecard(resultData);
|
|
4250
4374
|
const jobData = resultData.job;
|
|
@@ -4290,32 +4414,144 @@ Actions:
|
|
|
4290
4414
|
handleError(error, isJson);
|
|
4291
4415
|
}
|
|
4292
4416
|
});
|
|
4417
|
+
}
|
|
4418
|
+
|
|
4419
|
+
// src/commands/prompts/index.ts
|
|
4420
|
+
function updateMutationContext(updater) {
|
|
4421
|
+
try {
|
|
4422
|
+
const ctx = MutationContext.load();
|
|
4423
|
+
updater(ctx);
|
|
4424
|
+
ctx.save();
|
|
4425
|
+
} catch {}
|
|
4426
|
+
}
|
|
4427
|
+
function parseValidationErrors(error) {
|
|
4428
|
+
if (error instanceof ApiError) {
|
|
4429
|
+
try {
|
|
4430
|
+
const body = JSON.parse(error.message);
|
|
4431
|
+
if (body.errors && Array.isArray(body.errors)) {
|
|
4432
|
+
return body.errors.map((e) => e.message ?? JSON.stringify(e));
|
|
4433
|
+
}
|
|
4434
|
+
if (body.message) {
|
|
4435
|
+
return [body.message];
|
|
4436
|
+
}
|
|
4437
|
+
} catch {
|
|
4438
|
+
return [error.message];
|
|
4439
|
+
}
|
|
4440
|
+
}
|
|
4441
|
+
if (error instanceof Error) {
|
|
4442
|
+
return [error.message];
|
|
4443
|
+
}
|
|
4444
|
+
return ["An unknown error occurred"];
|
|
4445
|
+
}
|
|
4446
|
+
function isSchemaEmpty(schema) {
|
|
4447
|
+
if (schema === undefined || schema === null)
|
|
4448
|
+
return true;
|
|
4449
|
+
if (typeof schema !== "object")
|
|
4450
|
+
return false;
|
|
4451
|
+
const obj = schema;
|
|
4452
|
+
if (Object.keys(obj).length === 0)
|
|
4453
|
+
return true;
|
|
4454
|
+
return false;
|
|
4455
|
+
}
|
|
4456
|
+
function parseSchemaOption(inlineJson, filePath, schemaName, helpCommand) {
|
|
4457
|
+
if (inlineJson && filePath) {
|
|
4458
|
+
throw new MutagentError("INVALID_ARGUMENTS", `Cannot use --${schemaName} and --${schemaName}-file together`, `Run: ${helpCommand}
|
|
4459
|
+
Use --${schemaName} for inline JSON or --${schemaName}-file for file-based input, not both`);
|
|
4460
|
+
}
|
|
4461
|
+
if (inlineJson) {
|
|
4462
|
+
try {
|
|
4463
|
+
return JSON.parse(inlineJson);
|
|
4464
|
+
} catch {
|
|
4465
|
+
throw new MutagentError("INVALID_JSON", `Invalid JSON in --${schemaName} flag`, `Run: ${helpCommand}
|
|
4466
|
+
Provide a valid JSON Schema, e.g., '{"type":"object","properties":{"field":{"type":"string"}}}'`);
|
|
4467
|
+
}
|
|
4468
|
+
}
|
|
4469
|
+
if (filePath) {
|
|
4470
|
+
if (!existsSync5(filePath)) {
|
|
4471
|
+
throw new MutagentError("FILE_NOT_FOUND", `File not found: ${filePath}`, `Run: ${helpCommand}
|
|
4472
|
+
Check the file path and try again`);
|
|
4473
|
+
}
|
|
4474
|
+
try {
|
|
4475
|
+
return JSON.parse(readFileSync5(filePath, "utf-8"));
|
|
4476
|
+
} catch {
|
|
4477
|
+
throw new MutagentError("INVALID_JSON", `Failed to parse JSON from ${filePath}`, `Run: ${helpCommand}
|
|
4478
|
+
Ensure the file contains valid JSON Schema`);
|
|
4479
|
+
}
|
|
4480
|
+
}
|
|
4481
|
+
return;
|
|
4482
|
+
}
|
|
4483
|
+
function buildResultsScorecardText(resultData) {
|
|
4484
|
+
const jobData = resultData.job;
|
|
4485
|
+
const jobId = typeof jobData?.id === "string" ? jobData.id : "N/A";
|
|
4486
|
+
const jobStatus = typeof jobData?.status === "string" ? jobData.status : "unknown";
|
|
4487
|
+
const originalScore = resultData.originalScore;
|
|
4488
|
+
const bestScore = resultData.bestScore;
|
|
4489
|
+
const iterations = resultData.iterationsCompleted ?? 0;
|
|
4490
|
+
const title = "Optimization Results";
|
|
4491
|
+
const rows = [
|
|
4492
|
+
{ label: "Job ID", value: jobId },
|
|
4493
|
+
{ label: "Status", value: jobStatus },
|
|
4494
|
+
{ label: "Iterations", value: String(iterations) },
|
|
4495
|
+
{ label: "Original Score", value: originalScore !== undefined ? originalScore.toFixed(2) : "N/A" },
|
|
4496
|
+
{ label: "Best Score", value: bestScore !== undefined ? bestScore.toFixed(2) : "N/A" }
|
|
4497
|
+
];
|
|
4498
|
+
if (originalScore !== undefined && bestScore !== undefined && originalScore > 0) {
|
|
4499
|
+
const pct = Math.round((bestScore - originalScore) / originalScore * 100);
|
|
4500
|
+
rows.push({ label: "Improvement", value: `${pct > 0 ? "+" : ""}${String(pct)}%` });
|
|
4501
|
+
}
|
|
4502
|
+
return renderPlainCard({ title, rows, links: [], next: [] });
|
|
4503
|
+
}
|
|
4504
|
+
var PREREQUISITES_TEXT = `
|
|
4505
|
+
${chalk10.red("Prerequisites (required):")}
|
|
4506
|
+
1. Evaluation criteria defined ${chalk10.dim("(via dashboard or evaluation create)")}
|
|
4507
|
+
2. Dataset uploaded ${chalk10.dim("mutagent prompts dataset list <prompt-id>")}
|
|
4508
|
+
${chalk10.dim("Note: LLM provider config is only required when the server uses external providers (USE_EXT_PROVIDERS=true)")}`;
|
|
4509
|
+
function createPromptsCommand() {
|
|
4510
|
+
const prompts = new Command6("prompts").description("Manage prompts, datasets, evaluations, and optimizations").addHelpText("after", `
|
|
4511
|
+
Examples:
|
|
4512
|
+
${chalk10.dim("$")} mutagent prompts list
|
|
4513
|
+
${chalk10.dim("$")} mutagent prompts get <prompt-id>
|
|
4514
|
+
${chalk10.dim("$")} mutagent prompts create --name "my-prompt" --system "You are helpful" --human "{input}"
|
|
4515
|
+
${chalk10.dim("$")} mutagent prompts dataset list <prompt-id>
|
|
4516
|
+
${chalk10.dim("$")} mutagent prompts evaluation create <prompt-id> --name "My Eval"
|
|
4517
|
+
${chalk10.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --evaluation <eval-id>
|
|
4518
|
+
|
|
4519
|
+
Subcommands:
|
|
4520
|
+
list, get, create, update, delete
|
|
4521
|
+
dataset list|add|delete
|
|
4522
|
+
evaluation list|create|delete
|
|
4523
|
+
optimize start|status|results
|
|
4524
|
+
`);
|
|
4525
|
+
registerPromptsCrud(prompts);
|
|
4526
|
+
registerDatasetCommands(prompts);
|
|
4527
|
+
registerEvaluationCommands(prompts);
|
|
4528
|
+
registerOptimizeCommands(prompts);
|
|
4293
4529
|
return prompts;
|
|
4294
4530
|
}
|
|
4295
4531
|
|
|
4296
4532
|
// src/commands/traces.ts
|
|
4297
4533
|
init_sdk_client();
|
|
4298
|
-
import { Command as
|
|
4299
|
-
import
|
|
4534
|
+
import { Command as Command7 } from "commander";
|
|
4535
|
+
import chalk11 from "chalk";
|
|
4300
4536
|
init_errors();
|
|
4301
4537
|
function createTracesCommand() {
|
|
4302
|
-
const traces = new
|
|
4538
|
+
const traces = new Command7("traces").description("View and analyze traces (replaces Langfuse)").addHelpText("after", `
|
|
4303
4539
|
Examples:
|
|
4304
|
-
${
|
|
4305
|
-
${
|
|
4306
|
-
${
|
|
4307
|
-
${
|
|
4308
|
-
${
|
|
4540
|
+
${chalk11.dim("$")} mutagent traces list
|
|
4541
|
+
${chalk11.dim("$")} mutagent traces list --prompt <prompt-id>
|
|
4542
|
+
${chalk11.dim("$")} mutagent traces get <trace-id>
|
|
4543
|
+
${chalk11.dim("$")} mutagent traces analyze <prompt-id>
|
|
4544
|
+
${chalk11.dim("$")} mutagent traces export --format json --output traces.json
|
|
4309
4545
|
|
|
4310
4546
|
Note: MutagenT traces replace Langfuse for observability.
|
|
4311
4547
|
`);
|
|
4312
4548
|
traces.command("list").description("List traces").option("-p, --prompt <id>", "Filter by prompt ID").option("-l, --limit <n>", "Limit results", "50").addHelpText("after", `
|
|
4313
4549
|
Examples:
|
|
4314
|
-
${
|
|
4315
|
-
${
|
|
4316
|
-
${
|
|
4550
|
+
${chalk11.dim("$")} mutagent traces list
|
|
4551
|
+
${chalk11.dim("$")} mutagent traces list --prompt <prompt-id>
|
|
4552
|
+
${chalk11.dim("$")} mutagent traces list --limit 10 --json
|
|
4317
4553
|
|
|
4318
|
-
${
|
|
4554
|
+
${chalk11.dim("Tip: Filter by prompt to see traces for a specific prompt version.")}
|
|
4319
4555
|
`).action(async (options) => {
|
|
4320
4556
|
const isJson = getJsonFlag(traces);
|
|
4321
4557
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4354,10 +4590,10 @@ ${chalk7.dim("Tip: Filter by prompt to see traces for a specific prompt version.
|
|
|
4354
4590
|
});
|
|
4355
4591
|
traces.command("get").description("Get trace details").argument("<id>", "Trace ID").addHelpText("after", `
|
|
4356
4592
|
Examples:
|
|
4357
|
-
${
|
|
4358
|
-
${
|
|
4593
|
+
${chalk11.dim("$")} mutagent traces get <trace-id>
|
|
4594
|
+
${chalk11.dim("$")} mutagent traces get <trace-id> --json
|
|
4359
4595
|
|
|
4360
|
-
${
|
|
4596
|
+
${chalk11.dim("Returns full trace details including spans, tokens, and latency.")}
|
|
4361
4597
|
`).action(async (id) => {
|
|
4362
4598
|
const isJson = getJsonFlag(traces);
|
|
4363
4599
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4376,10 +4612,10 @@ ${chalk7.dim("Returns full trace details including spans, tokens, and latency.")
|
|
|
4376
4612
|
});
|
|
4377
4613
|
traces.command("analyze").description("Analyze traces for a prompt").argument("<prompt-id>", "Prompt ID").addHelpText("after", `
|
|
4378
4614
|
Examples:
|
|
4379
|
-
${
|
|
4380
|
-
${
|
|
4615
|
+
${chalk11.dim("$")} mutagent traces analyze <prompt-id>
|
|
4616
|
+
${chalk11.dim("$")} mutagent traces analyze <prompt-id> --json
|
|
4381
4617
|
|
|
4382
|
-
${
|
|
4618
|
+
${chalk11.dim("Aggregates trace data for a prompt: avg latency, token usage, error rates.")}
|
|
4383
4619
|
`).action(async (promptId) => {
|
|
4384
4620
|
const isJson = getJsonFlag(traces);
|
|
4385
4621
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4397,12 +4633,12 @@ ${chalk7.dim("Aggregates trace data for a prompt: avg latency, token usage, erro
|
|
|
4397
4633
|
});
|
|
4398
4634
|
traces.command("export").description("Export traces").option("-p, --prompt <id>", "Filter by prompt ID").option("-f, --format <format>", "Export format (json, csv)", "json").option("-o, --output <path>", "Output file path").addHelpText("after", `
|
|
4399
4635
|
Examples:
|
|
4400
|
-
${
|
|
4401
|
-
${
|
|
4402
|
-
${
|
|
4403
|
-
${
|
|
4636
|
+
${chalk11.dim("$")} mutagent traces export
|
|
4637
|
+
${chalk11.dim("$")} mutagent traces export --format json --output traces.json
|
|
4638
|
+
${chalk11.dim("$")} mutagent traces export --format csv --output traces.csv
|
|
4639
|
+
${chalk11.dim("$")} mutagent traces export --prompt <prompt-id> --format json
|
|
4404
4640
|
|
|
4405
|
-
${
|
|
4641
|
+
${chalk11.dim("Exports to stdout by default. Use --output to save to a file.")}
|
|
4406
4642
|
`).action(async (options) => {
|
|
4407
4643
|
const isJson = getJsonFlag(traces);
|
|
4408
4644
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4445,22 +4681,22 @@ ${chalk7.dim("Exports to stdout by default. Use --output to save to a file.")}
|
|
|
4445
4681
|
|
|
4446
4682
|
// src/commands/integrate.ts
|
|
4447
4683
|
init_config();
|
|
4448
|
-
import { Command as
|
|
4449
|
-
import
|
|
4450
|
-
import { writeFileSync as writeFileSync3, existsSync as
|
|
4684
|
+
import { Command as Command8 } from "commander";
|
|
4685
|
+
import chalk12 from "chalk";
|
|
4686
|
+
import { writeFileSync as writeFileSync3, existsSync as existsSync10 } from "fs";
|
|
4451
4687
|
import { execSync } from "child_process";
|
|
4452
4688
|
init_errors();
|
|
4453
4689
|
|
|
4454
4690
|
// src/lib/integrations/langchain.ts
|
|
4455
|
-
import { readFileSync as
|
|
4691
|
+
import { readFileSync as readFileSync6, existsSync as existsSync6 } from "fs";
|
|
4456
4692
|
var langchainIntegration = {
|
|
4457
4693
|
name: "langchain",
|
|
4458
4694
|
description: "LangChain framework",
|
|
4459
4695
|
detect() {
|
|
4460
4696
|
let hasLangchain = false;
|
|
4461
|
-
if (
|
|
4697
|
+
if (existsSync6("package.json")) {
|
|
4462
4698
|
try {
|
|
4463
|
-
const pkg = JSON.parse(
|
|
4699
|
+
const pkg = JSON.parse(readFileSync6("package.json", "utf-8"));
|
|
4464
4700
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
4465
4701
|
hasLangchain = "langchain" in deps || "@langchain/core" in deps;
|
|
4466
4702
|
} catch {}
|
|
@@ -4592,15 +4828,15 @@ mutagent traces analyze <prompt-id>
|
|
|
4592
4828
|
};
|
|
4593
4829
|
|
|
4594
4830
|
// src/lib/integrations/langgraph.ts
|
|
4595
|
-
import { readFileSync as
|
|
4831
|
+
import { readFileSync as readFileSync7, existsSync as existsSync7 } from "fs";
|
|
4596
4832
|
var langgraphIntegration = {
|
|
4597
4833
|
name: "langgraph",
|
|
4598
4834
|
description: "LangGraph agent workflow framework",
|
|
4599
4835
|
detect() {
|
|
4600
4836
|
let hasLanggraph = false;
|
|
4601
|
-
if (
|
|
4837
|
+
if (existsSync7("package.json")) {
|
|
4602
4838
|
try {
|
|
4603
|
-
const pkg = JSON.parse(
|
|
4839
|
+
const pkg = JSON.parse(readFileSync7("package.json", "utf-8"));
|
|
4604
4840
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
4605
4841
|
hasLanggraph = "@langchain/langgraph" in deps;
|
|
4606
4842
|
} catch {}
|
|
@@ -4721,15 +4957,15 @@ mutagent integrate langgraph --verify
|
|
|
4721
4957
|
};
|
|
4722
4958
|
|
|
4723
4959
|
// src/lib/integrations/vercel-ai.ts
|
|
4724
|
-
import { readFileSync as
|
|
4960
|
+
import { readFileSync as readFileSync8, existsSync as existsSync8 } from "fs";
|
|
4725
4961
|
var vercelAiIntegration = {
|
|
4726
4962
|
name: "vercel-ai",
|
|
4727
4963
|
description: "Vercel AI SDK",
|
|
4728
4964
|
detect() {
|
|
4729
4965
|
let hasAiSdk = false;
|
|
4730
|
-
if (
|
|
4966
|
+
if (existsSync8("package.json")) {
|
|
4731
4967
|
try {
|
|
4732
|
-
const pkg = JSON.parse(
|
|
4968
|
+
const pkg = JSON.parse(readFileSync8("package.json", "utf-8"));
|
|
4733
4969
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
4734
4970
|
hasAiSdk = "ai" in deps;
|
|
4735
4971
|
} catch {}
|
|
@@ -4867,15 +5103,15 @@ mutagent integrate vercel-ai --verify
|
|
|
4867
5103
|
};
|
|
4868
5104
|
|
|
4869
5105
|
// src/lib/integrations/openai.ts
|
|
4870
|
-
import { readFileSync as
|
|
5106
|
+
import { readFileSync as readFileSync9, existsSync as existsSync9 } from "fs";
|
|
4871
5107
|
var openaiIntegration = {
|
|
4872
5108
|
name: "openai",
|
|
4873
5109
|
description: "OpenAI SDK integration with automatic tracing",
|
|
4874
5110
|
detect() {
|
|
4875
5111
|
let hasOpenAI = false;
|
|
4876
|
-
if (
|
|
5112
|
+
if (existsSync9("package.json")) {
|
|
4877
5113
|
try {
|
|
4878
|
-
const pkg = JSON.parse(
|
|
5114
|
+
const pkg = JSON.parse(readFileSync9("package.json", "utf-8"));
|
|
4879
5115
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
4880
5116
|
hasOpenAI = "openai" in deps;
|
|
4881
5117
|
} catch {}
|
|
@@ -5029,16 +5265,16 @@ function getFrameworkMetadata(name) {
|
|
|
5029
5265
|
|
|
5030
5266
|
// src/commands/integrate.ts
|
|
5031
5267
|
function detectPackageManager() {
|
|
5032
|
-
if (
|
|
5268
|
+
if (existsSync10("bun.lockb") || existsSync10("bun.lock")) {
|
|
5033
5269
|
return "bun";
|
|
5034
5270
|
}
|
|
5035
|
-
if (
|
|
5271
|
+
if (existsSync10("pnpm-lock.yaml")) {
|
|
5036
5272
|
return "pnpm";
|
|
5037
5273
|
}
|
|
5038
|
-
if (
|
|
5274
|
+
if (existsSync10("yarn.lock")) {
|
|
5039
5275
|
return "yarn";
|
|
5040
5276
|
}
|
|
5041
|
-
if (
|
|
5277
|
+
if (existsSync10("package-lock.json")) {
|
|
5042
5278
|
return "npm";
|
|
5043
5279
|
}
|
|
5044
5280
|
try {
|
|
@@ -5081,14 +5317,14 @@ AI AGENT: Please determine the framework by:
|
|
|
5081
5317
|
`;
|
|
5082
5318
|
}
|
|
5083
5319
|
function createIntegrateCommand() {
|
|
5084
|
-
const integrate = new
|
|
5320
|
+
const integrate = new Command8("integrate").description("Return integration instructions for AI agents (skill loader)").addHelpText("after", `
|
|
5085
5321
|
Examples:
|
|
5086
|
-
${
|
|
5087
|
-
${
|
|
5088
|
-
${
|
|
5089
|
-
${
|
|
5090
|
-
${
|
|
5091
|
-
${
|
|
5322
|
+
${chalk12.dim("$")} mutagent integrate # Return exploration instructions for AI agents
|
|
5323
|
+
${chalk12.dim("$")} mutagent integrate langchain # Return @mutagent/langchain integration steps
|
|
5324
|
+
${chalk12.dim("$")} mutagent integrate openai # Return @mutagent/openai integration steps
|
|
5325
|
+
${chalk12.dim("$")} mutagent integrate vercel-ai --verify # Verify integration
|
|
5326
|
+
${chalk12.dim("$")} mutagent integrate langchain --output ./INTEGRATION.md
|
|
5327
|
+
${chalk12.dim("$")} mutagent integrate --list # List available frameworks
|
|
5092
5328
|
|
|
5093
5329
|
Frameworks: langchain, langgraph, vercel-ai, openai
|
|
5094
5330
|
|
|
@@ -5202,35 +5438,20 @@ AI AGENT: Execute these steps using Bash for install, Write/Edit for code change
|
|
|
5202
5438
|
return integrate;
|
|
5203
5439
|
}
|
|
5204
5440
|
|
|
5205
|
-
// src/commands/agents.ts
|
|
5441
|
+
// src/commands/agents/index.ts
|
|
5442
|
+
import { Command as Command9 } from "commander";
|
|
5443
|
+
import chalk14 from "chalk";
|
|
5444
|
+
|
|
5445
|
+
// src/commands/agents/agents-crud.ts
|
|
5206
5446
|
init_sdk_client();
|
|
5207
|
-
import
|
|
5208
|
-
import chalk9 from "chalk";
|
|
5447
|
+
import chalk13 from "chalk";
|
|
5209
5448
|
init_errors();
|
|
5210
|
-
function
|
|
5211
|
-
const agents = new Command6("agents").description("Manage AI agents").addHelpText("after", `
|
|
5212
|
-
Examples:
|
|
5213
|
-
${chalk9.dim("$")} mutagent agents list
|
|
5214
|
-
${chalk9.dim("$")} mutagent agents get <agent-id>
|
|
5215
|
-
${chalk9.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "You are a code reviewer..."
|
|
5216
|
-
${chalk9.dim("$")} mutagent agents create -d '{"name":"Code Reviewer","slug":"code-reviewer","systemPrompt":"You are..."}'
|
|
5217
|
-
${chalk9.dim("$")} mutagent agents update <agent-id> --name "Updated Name"
|
|
5218
|
-
${chalk9.dim("$")} mutagent agents delete <agent-id> --force
|
|
5219
|
-
${chalk9.dim("$")} mutagent agents conversations list <agent-id>
|
|
5220
|
-
${chalk9.dim("$")} mutagent agents conversations get <agent-id> <conversation-id>
|
|
5221
|
-
${chalk9.dim("$")} mutagent agents conversations create <agent-id>
|
|
5222
|
-
${chalk9.dim("$")} mutagent agents conversations delete <agent-id> <conversation-id>
|
|
5223
|
-
${chalk9.dim("$")} mutagent agents conversations messages <agent-id> <conversation-id>
|
|
5224
|
-
|
|
5225
|
-
Subcommands:
|
|
5226
|
-
list, get, create, update, delete
|
|
5227
|
-
conversations list|get|create|delete|messages
|
|
5228
|
-
`);
|
|
5449
|
+
function registerAgentsCrud(agents) {
|
|
5229
5450
|
agents.command("list").description("List all agents").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").option("-n, --name <name>", "Filter by name").option("-s, --status <status>", "Filter by status (active, paused, archived)").addHelpText("after", `
|
|
5230
5451
|
Examples:
|
|
5231
|
-
${
|
|
5232
|
-
${
|
|
5233
|
-
${
|
|
5452
|
+
${chalk13.dim("$")} mutagent agents list
|
|
5453
|
+
${chalk13.dim("$")} mutagent agents list --status active
|
|
5454
|
+
${chalk13.dim("$")} mutagent agents list --name "reviewer" --json
|
|
5234
5455
|
`).action(async (options) => {
|
|
5235
5456
|
const isJson = getJsonFlag(agents);
|
|
5236
5457
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5280,8 +5501,8 @@ Examples:
|
|
|
5280
5501
|
});
|
|
5281
5502
|
agents.command("get").description("Get agent details").argument("<id>", "Agent ID").addHelpText("after", `
|
|
5282
5503
|
Examples:
|
|
5283
|
-
${
|
|
5284
|
-
${
|
|
5504
|
+
${chalk13.dim("$")} mutagent agents get <agent-id>
|
|
5505
|
+
${chalk13.dim("$")} mutagent agents get <agent-id> --json
|
|
5285
5506
|
`).action(async (id) => {
|
|
5286
5507
|
const isJson = getJsonFlag(agents);
|
|
5287
5508
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5311,11 +5532,11 @@ Examples:
|
|
|
5311
5532
|
};
|
|
5312
5533
|
output.output(formatted);
|
|
5313
5534
|
if (agent.systemPrompt) {
|
|
5314
|
-
console.log(
|
|
5535
|
+
console.log(chalk13.bold(`
|
|
5315
5536
|
System Prompt:`));
|
|
5316
|
-
console.log(
|
|
5537
|
+
console.log(chalk13.gray("─".repeat(60)));
|
|
5317
5538
|
console.log(agent.systemPrompt);
|
|
5318
|
-
console.log(
|
|
5539
|
+
console.log(chalk13.gray("─".repeat(60)));
|
|
5319
5540
|
}
|
|
5320
5541
|
}
|
|
5321
5542
|
} catch (error) {
|
|
@@ -5324,17 +5545,17 @@ System Prompt:`));
|
|
|
5324
5545
|
});
|
|
5325
5546
|
agents.command("create").description("Create a new agent").option("-d, --data <json>", "Agent as JSON string (recommended for CI/scripts/agents)").option("-n, --name <name>", "Agent name").option("-s, --slug <slug>", "Agent slug (URL-friendly identifier)").option("-p, --system-prompt <prompt>", "System prompt").option("-m, --model <model>", "Model (claude-sonnet-4-5, claude-opus-4-5, claude-haiku-4-5)").option("--description <desc>", "Agent description").addHelpText("after", `
|
|
5326
5547
|
Examples:
|
|
5327
|
-
${
|
|
5328
|
-
${
|
|
5548
|
+
${chalk13.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "You are a code reviewer..."
|
|
5549
|
+
${chalk13.dim("$")} mutagent agents create -d '{"name":"Code Reviewer","slug":"code-reviewer","systemPrompt":"You are a code reviewer..."}'
|
|
5329
5550
|
|
|
5330
5551
|
Expected JSON (--data):
|
|
5331
|
-
${
|
|
5552
|
+
${chalk13.dim('{"name":"<name>","slug":"<slug>","systemPrompt":"<system prompt>","model":"<model-id>","description":"<description>"}')}
|
|
5332
5553
|
|
|
5333
5554
|
Input Methods (pick one, priority order):
|
|
5334
|
-
--name/--slug/... Individual flags ${
|
|
5555
|
+
--name/--slug/... Individual flags ${chalk13.green("(recommended)")}
|
|
5335
5556
|
-d, --data Inline JSON object (CI/scripts/agents)
|
|
5336
5557
|
|
|
5337
|
-
${
|
|
5558
|
+
${chalk13.red("Required: name, slug, systemPrompt.")} ${chalk13.dim("CLI flags override --data fields.")}
|
|
5338
5559
|
`).action(async (options) => {
|
|
5339
5560
|
const isJson = getJsonFlag(agents);
|
|
5340
5561
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5379,15 +5600,15 @@ ${chalk9.red("Required: name, slug, systemPrompt.")} ${chalk9.dim("CLI flags ove
|
|
|
5379
5600
|
});
|
|
5380
5601
|
agents.command("update").description("Update an agent").argument("<id>", "Agent ID").option("-d, --data <json>", "Agent updates as JSON string (CI/scripts/agents)").option("-n, --name <name>", "New name").option("-p, --system-prompt <prompt>", "New system prompt").option("-m, --model <model>", "New model").option("--description <desc>", "New description").option("-s, --status <status>", "New status (active, paused, archived)").addHelpText("after", `
|
|
5381
5602
|
Examples:
|
|
5382
|
-
${
|
|
5383
|
-
${
|
|
5384
|
-
${
|
|
5603
|
+
${chalk13.dim("$")} mutagent agents update <id> --name "New Name"
|
|
5604
|
+
${chalk13.dim("$")} mutagent agents update <id> --system-prompt "Updated prompt" --status active
|
|
5605
|
+
${chalk13.dim("$")} mutagent agents update <id> -d '{"name":"New Name","systemPrompt":"Updated prompt"}'
|
|
5385
5606
|
|
|
5386
5607
|
Input Methods (pick one, priority order):
|
|
5387
|
-
--name/--system-prompt/... Individual flags ${
|
|
5608
|
+
--name/--system-prompt/... Individual flags ${chalk13.green("(recommended)")}
|
|
5388
5609
|
-d, --data Inline JSON object (CI/scripts/agents)
|
|
5389
5610
|
|
|
5390
|
-
${
|
|
5611
|
+
${chalk13.dim("CLI flags override --data fields.")}
|
|
5391
5612
|
`).action(async (id, options) => {
|
|
5392
5613
|
const isJson = getJsonFlag(agents);
|
|
5393
5614
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5434,11 +5655,11 @@ ${chalk9.dim("CLI flags override --data fields.")}
|
|
|
5434
5655
|
});
|
|
5435
5656
|
agents.command("delete").description("Delete an agent").argument("<id>", "Agent ID").option("--force", "Skip confirmation").addHelpText("after", `
|
|
5436
5657
|
Examples:
|
|
5437
|
-
${
|
|
5438
|
-
${
|
|
5439
|
-
${
|
|
5658
|
+
${chalk13.dim("$")} mutagent agents delete <id>
|
|
5659
|
+
${chalk13.dim("$")} mutagent agents delete <id> --force
|
|
5660
|
+
${chalk13.dim("$")} mutagent agents delete <id> --force --json
|
|
5440
5661
|
|
|
5441
|
-
${
|
|
5662
|
+
${chalk13.dim("Tip: Use --force to skip confirmation (required for non-interactive/CI usage).")}
|
|
5442
5663
|
`).action(async (id, options) => {
|
|
5443
5664
|
const isJson = getJsonFlag(agents);
|
|
5444
5665
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5463,177 +5684,44 @@ ${chalk9.dim("Tip: Use --force to skip confirmation (required for non-interactiv
|
|
|
5463
5684
|
handleError(error, isJson);
|
|
5464
5685
|
}
|
|
5465
5686
|
});
|
|
5466
|
-
|
|
5467
|
-
|
|
5468
|
-
|
|
5469
|
-
|
|
5470
|
-
|
|
5471
|
-
`).action(() => {
|
|
5472
|
-
conversations.help();
|
|
5473
|
-
});
|
|
5474
|
-
agents.addCommand(conversations);
|
|
5475
|
-
conversations.command("list").description("List conversations for an agent").argument("<agent-id>", "Agent ID").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").addHelpText("after", `
|
|
5476
|
-
Examples:
|
|
5477
|
-
${chalk9.dim("$")} mutagent agents conversations list <agent-id>
|
|
5478
|
-
${chalk9.dim("$")} mutagent agents conversations list <agent-id> --limit 10 --json
|
|
5479
|
-
`).action(async (agentId, options) => {
|
|
5480
|
-
const isJson = getJsonFlag(agents);
|
|
5481
|
-
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5482
|
-
try {
|
|
5483
|
-
const client = getSDKClient();
|
|
5484
|
-
const filters = {
|
|
5485
|
-
limit: parseInt(options.limit, 10) || 50
|
|
5486
|
-
};
|
|
5487
|
-
if (options.offset) {
|
|
5488
|
-
filters.offset = parseInt(options.offset, 10);
|
|
5489
|
-
}
|
|
5490
|
-
const result = await client.listConversations(agentId, filters);
|
|
5491
|
-
if (isJson) {
|
|
5492
|
-
output.output(result);
|
|
5493
|
-
} else {
|
|
5494
|
-
const formatted = result.data.map((c) => ({
|
|
5495
|
-
id: c.id,
|
|
5496
|
-
title: c.title ?? "Untitled",
|
|
5497
|
-
status: c.status,
|
|
5498
|
-
messages: c.messageCount ?? "N/A",
|
|
5499
|
-
created: c.createdAt ? new Date(c.createdAt).toLocaleDateString() : "N/A",
|
|
5500
|
-
updated: c.updatedAt ? new Date(c.updatedAt).toLocaleDateString() : "N/A"
|
|
5501
|
-
}));
|
|
5502
|
-
output.output(formatted);
|
|
5503
|
-
}
|
|
5504
|
-
} catch (error) {
|
|
5505
|
-
handleError(error, isJson);
|
|
5506
|
-
}
|
|
5507
|
-
});
|
|
5508
|
-
conversations.command("get").description("Get conversation details").argument("<agent-id>", "Agent ID").argument("<conversation-id>", "Conversation ID").addHelpText("after", `
|
|
5509
|
-
Examples:
|
|
5510
|
-
${chalk9.dim("$")} mutagent agents conversations get <agent-id> <conversation-id>
|
|
5511
|
-
${chalk9.dim("$")} mutagent agents conversations get <agent-id> <conversation-id> --json
|
|
5512
|
-
`).action(async (agentId, conversationId) => {
|
|
5513
|
-
const isJson = getJsonFlag(agents);
|
|
5514
|
-
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5515
|
-
try {
|
|
5516
|
-
const client = getSDKClient();
|
|
5517
|
-
const conversation = await client.getConversation(agentId, conversationId);
|
|
5518
|
-
if (isJson) {
|
|
5519
|
-
output.output(conversation);
|
|
5520
|
-
} else {
|
|
5521
|
-
const formatted = {
|
|
5522
|
-
id: conversation.id,
|
|
5523
|
-
agentId: conversation.agentId,
|
|
5524
|
-
title: conversation.title ?? "Untitled",
|
|
5525
|
-
status: conversation.status,
|
|
5526
|
-
messageCount: conversation.messageCount ?? "N/A",
|
|
5527
|
-
createdAt: conversation.createdAt ? new Date(conversation.createdAt).toLocaleString() : "N/A",
|
|
5528
|
-
updatedAt: conversation.updatedAt ? new Date(conversation.updatedAt).toLocaleString() : "N/A"
|
|
5529
|
-
};
|
|
5530
|
-
output.output(formatted);
|
|
5531
|
-
}
|
|
5532
|
-
} catch (error) {
|
|
5533
|
-
handleError(error, isJson);
|
|
5534
|
-
}
|
|
5535
|
-
});
|
|
5536
|
-
conversations.command("create").description("Start a new conversation with an agent").argument("<agent-id>", "Agent ID").option("-t, --title <title>", "Conversation title").addHelpText("after", `
|
|
5537
|
-
Examples:
|
|
5538
|
-
${chalk9.dim("$")} mutagent agents conversations create <agent-id>
|
|
5539
|
-
${chalk9.dim("$")} mutagent agents conversations create <agent-id> --title "Debug session" --json
|
|
5540
|
-
`).action(async (agentId, options) => {
|
|
5541
|
-
const isJson = getJsonFlag(agents);
|
|
5542
|
-
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5543
|
-
try {
|
|
5544
|
-
const client = getSDKClient();
|
|
5545
|
-
const data = {};
|
|
5546
|
-
if (options.title) {
|
|
5547
|
-
data.title = options.title;
|
|
5548
|
-
}
|
|
5549
|
-
const conversation = await client.createConversation(agentId, data);
|
|
5550
|
-
output.success(`Created conversation: ${conversation.id}`);
|
|
5551
|
-
output.output(conversation);
|
|
5552
|
-
} catch (error) {
|
|
5553
|
-
handleError(error, isJson);
|
|
5554
|
-
}
|
|
5555
|
-
});
|
|
5556
|
-
conversations.command("delete").description("Delete a conversation").argument("<agent-id>", "Agent ID").argument("<conversation-id>", "Conversation ID").option("--force", "Skip confirmation").addHelpText("after", `
|
|
5557
|
-
Examples:
|
|
5558
|
-
${chalk9.dim("$")} mutagent agents conversations delete <agent-id> <conversation-id>
|
|
5559
|
-
${chalk9.dim("$")} mutagent agents conversations delete <agent-id> <conversation-id> --force --json
|
|
5560
|
-
`).action(async (agentId, conversationId, options) => {
|
|
5561
|
-
const isJson = getJsonFlag(agents);
|
|
5562
|
-
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5563
|
-
try {
|
|
5564
|
-
if (!options.force && !isJson) {
|
|
5565
|
-
const inquirer3 = (await import("inquirer")).default;
|
|
5566
|
-
const answers = await inquirer3.prompt([{
|
|
5567
|
-
type: "confirm",
|
|
5568
|
-
name: "confirm",
|
|
5569
|
-
message: `Delete conversation ${conversationId}? This action cannot be undone.`,
|
|
5570
|
-
default: false
|
|
5571
|
-
}]);
|
|
5572
|
-
if (!answers.confirm) {
|
|
5573
|
-
output.info("Cancelled");
|
|
5574
|
-
return;
|
|
5575
|
-
}
|
|
5576
|
-
}
|
|
5577
|
-
const client = getSDKClient();
|
|
5578
|
-
await client.deleteConversation(agentId, conversationId);
|
|
5579
|
-
output.success(`Deleted conversation: ${conversationId}`);
|
|
5580
|
-
} catch (error) {
|
|
5581
|
-
handleError(error, isJson);
|
|
5582
|
-
}
|
|
5583
|
-
});
|
|
5584
|
-
conversations.command("messages").description("List messages in a conversation").argument("<agent-id>", "Agent ID").argument("<conversation-id>", "Conversation ID").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").addHelpText("after", `
|
|
5687
|
+
}
|
|
5688
|
+
|
|
5689
|
+
// src/commands/agents/index.ts
|
|
5690
|
+
function createAgentsCommand() {
|
|
5691
|
+
const agents = new Command9("agents").description("Manage AI agents").addHelpText("after", `
|
|
5585
5692
|
Examples:
|
|
5586
|
-
${
|
|
5587
|
-
${
|
|
5588
|
-
|
|
5589
|
-
|
|
5590
|
-
|
|
5591
|
-
|
|
5592
|
-
|
|
5593
|
-
|
|
5594
|
-
|
|
5595
|
-
|
|
5596
|
-
|
|
5597
|
-
filters.offset = parseInt(options.offset, 10);
|
|
5598
|
-
}
|
|
5599
|
-
const result = await client.listMessages(agentId, conversationId, filters);
|
|
5600
|
-
if (isJson) {
|
|
5601
|
-
output.output(result);
|
|
5602
|
-
} else {
|
|
5603
|
-
const formatted = result.data.map((m) => ({
|
|
5604
|
-
id: m.id,
|
|
5605
|
-
role: m.role,
|
|
5606
|
-
content: m.content.length > 80 ? m.content.substring(0, 77) + "..." : m.content,
|
|
5607
|
-
tokens: m.tokenCount ?? "N/A",
|
|
5608
|
-
created: m.createdAt ? new Date(m.createdAt).toLocaleString() : "N/A"
|
|
5609
|
-
}));
|
|
5610
|
-
output.output(formatted);
|
|
5611
|
-
}
|
|
5612
|
-
} catch (error) {
|
|
5613
|
-
handleError(error, isJson);
|
|
5614
|
-
}
|
|
5615
|
-
});
|
|
5693
|
+
${chalk14.dim("$")} mutagent agents list
|
|
5694
|
+
${chalk14.dim("$")} mutagent agents get <agent-id>
|
|
5695
|
+
${chalk14.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "You are a code reviewer..."
|
|
5696
|
+
${chalk14.dim("$")} mutagent agents create -d '{"name":"Code Reviewer","slug":"code-reviewer","systemPrompt":"You are..."}'
|
|
5697
|
+
${chalk14.dim("$")} mutagent agents update <agent-id> --name "Updated Name"
|
|
5698
|
+
${chalk14.dim("$")} mutagent agents delete <agent-id> --force
|
|
5699
|
+
|
|
5700
|
+
Subcommands:
|
|
5701
|
+
list, get, create, update, delete
|
|
5702
|
+
`);
|
|
5703
|
+
registerAgentsCrud(agents);
|
|
5616
5704
|
return agents;
|
|
5617
5705
|
}
|
|
5618
5706
|
|
|
5619
5707
|
// src/commands/config.ts
|
|
5620
5708
|
init_config();
|
|
5621
|
-
import { Command as
|
|
5622
|
-
import
|
|
5709
|
+
import { Command as Command10 } from "commander";
|
|
5710
|
+
import chalk15 from "chalk";
|
|
5623
5711
|
init_errors();
|
|
5624
5712
|
var VALID_CONFIG_KEYS = ["apiKey", "endpoint", "format", "timeout", "defaultWorkspace", "defaultOrganization"];
|
|
5625
5713
|
function createConfigCommand() {
|
|
5626
|
-
const config = new
|
|
5714
|
+
const config = new Command10("config").description("Manage CLI configuration").addHelpText("after", `
|
|
5627
5715
|
Examples:
|
|
5628
|
-
${
|
|
5629
|
-
${
|
|
5630
|
-
${
|
|
5631
|
-
${
|
|
5716
|
+
${chalk15.dim("$")} mutagent config list
|
|
5717
|
+
${chalk15.dim("$")} mutagent config get endpoint
|
|
5718
|
+
${chalk15.dim("$")} mutagent config set workspace <workspace-id>
|
|
5719
|
+
${chalk15.dim("$")} mutagent config set org <org-id>
|
|
5632
5720
|
`);
|
|
5633
5721
|
config.command("list").description("List all configuration").addHelpText("after", `
|
|
5634
5722
|
Examples:
|
|
5635
|
-
${
|
|
5636
|
-
${
|
|
5723
|
+
${chalk15.dim("$")} mutagent config list
|
|
5724
|
+
${chalk15.dim("$")} mutagent config list --json
|
|
5637
5725
|
`).action(() => {
|
|
5638
5726
|
const isJson = getJsonFlag(config);
|
|
5639
5727
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5646,11 +5734,11 @@ Examples:
|
|
|
5646
5734
|
});
|
|
5647
5735
|
config.command("get").description("Get configuration value").argument("<key>", "Configuration key (apiKey, endpoint, format, timeout, defaultWorkspace, defaultOrganization)").addHelpText("after", `
|
|
5648
5736
|
Examples:
|
|
5649
|
-
${
|
|
5650
|
-
${
|
|
5651
|
-
${
|
|
5737
|
+
${chalk15.dim("$")} mutagent config get endpoint
|
|
5738
|
+
${chalk15.dim("$")} mutagent config get defaultWorkspace
|
|
5739
|
+
${chalk15.dim("$")} mutagent config get apiKey --json
|
|
5652
5740
|
|
|
5653
|
-
${
|
|
5741
|
+
${chalk15.dim("Keys: apiKey, endpoint, format, timeout, defaultWorkspace, defaultOrganization")}
|
|
5654
5742
|
`).action((key) => {
|
|
5655
5743
|
const isJson = getJsonFlag(config);
|
|
5656
5744
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5672,14 +5760,14 @@ ${chalk10.dim("Keys: apiKey, endpoint, format, timeout, defaultWorkspace, defaul
|
|
|
5672
5760
|
handleError(error, isJson);
|
|
5673
5761
|
}
|
|
5674
5762
|
});
|
|
5675
|
-
const set = new
|
|
5763
|
+
const set = new Command10("set").description("Set configuration value").action(() => {
|
|
5676
5764
|
set.help();
|
|
5677
5765
|
});
|
|
5678
5766
|
set.command("workspace").description("Set default workspace ID").argument("<id>", "Workspace ID to set as default").addHelpText("after", `
|
|
5679
5767
|
Examples:
|
|
5680
|
-
${
|
|
5768
|
+
${chalk15.dim("$")} mutagent config set workspace <workspace-id>
|
|
5681
5769
|
|
|
5682
|
-
${
|
|
5770
|
+
${chalk15.dim("Persists workspace ID so you don't need to pass headers on every request.")}
|
|
5683
5771
|
`).action((id) => {
|
|
5684
5772
|
const isJson = getJsonFlag(config);
|
|
5685
5773
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5692,9 +5780,9 @@ ${chalk10.dim("Persists workspace ID so you don't need to pass headers on every
|
|
|
5692
5780
|
});
|
|
5693
5781
|
set.command("org").description("Set default organization ID").argument("<id>", "Organization ID to set as default").addHelpText("after", `
|
|
5694
5782
|
Examples:
|
|
5695
|
-
${
|
|
5783
|
+
${chalk15.dim("$")} mutagent config set org <org-id>
|
|
5696
5784
|
|
|
5697
|
-
${
|
|
5785
|
+
${chalk15.dim("Persists organization ID for org-scoped API keys.")}
|
|
5698
5786
|
`).action((id) => {
|
|
5699
5787
|
const isJson = getJsonFlag(config);
|
|
5700
5788
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5711,8 +5799,8 @@ ${chalk10.dim("Persists organization ID for org-scoped API keys.")}
|
|
|
5711
5799
|
|
|
5712
5800
|
// src/commands/playground.ts
|
|
5713
5801
|
init_sdk_client();
|
|
5714
|
-
import { Command as
|
|
5715
|
-
import
|
|
5802
|
+
import { Command as Command11 } from "commander";
|
|
5803
|
+
import chalk16 from "chalk";
|
|
5716
5804
|
init_errors();
|
|
5717
5805
|
function parseSSELine(line) {
|
|
5718
5806
|
if (!line || line.startsWith(":")) {
|
|
@@ -5737,13 +5825,13 @@ function parsePromptStreamEvent(data) {
|
|
|
5737
5825
|
}
|
|
5738
5826
|
}
|
|
5739
5827
|
function createPlaygroundCommand() {
|
|
5740
|
-
const playground = new
|
|
5828
|
+
const playground = new Command11("playground").description("Execute and test prompts interactively").addHelpText("after", `
|
|
5741
5829
|
Examples:
|
|
5742
|
-
${
|
|
5743
|
-
${
|
|
5744
|
-
${
|
|
5745
|
-
${
|
|
5746
|
-
${
|
|
5830
|
+
${chalk16.dim("$")} mutagent playground run <prompt-id> --input '{"name": "John"}'
|
|
5831
|
+
${chalk16.dim("$")} mutagent playground run <prompt-id> --input '{}' --stream
|
|
5832
|
+
${chalk16.dim("$")} mutagent playground run <prompt-id> -i '{}' --model gpt-4-turbo
|
|
5833
|
+
${chalk16.dim("$")} mutagent playground run <prompt-id> --system "You are helpful" --human "Hello"
|
|
5834
|
+
${chalk16.dim("$")} mutagent playground run <prompt-id> --messages '[{"role":"user","content":"Hi"}]'
|
|
5747
5835
|
|
|
5748
5836
|
Input Format:
|
|
5749
5837
|
The input must be a valid JSON object matching the prompt's input schema.
|
|
@@ -5759,16 +5847,16 @@ Streaming:
|
|
|
5759
5847
|
`);
|
|
5760
5848
|
playground.command("run").description("Execute a prompt with input variables").argument("<prompt-id>", "Prompt ID to execute (from: mutagent prompts list)").option("-i, --input <json>", "Input variables as JSON").option("-s, --stream", "Stream the response").option("-m, --model <model>", "Override model").option("--system <text>", "Set system prompt text").option("--human <text>", "Set human/user message text").option("--messages <json>", "Pass full messages array as JSON string").addHelpText("after", `
|
|
5761
5849
|
Examples:
|
|
5762
|
-
${
|
|
5763
|
-
${
|
|
5764
|
-
${
|
|
5765
|
-
${
|
|
5850
|
+
${chalk16.dim("$")} mutagent playground run <prompt-id> --input '{"name": "John"}'
|
|
5851
|
+
${chalk16.dim("$")} mutagent playground run <prompt-id> --input '{}' --stream
|
|
5852
|
+
${chalk16.dim("$")} mutagent playground run <prompt-id> --system "You are helpful" --human "Hello"
|
|
5853
|
+
${chalk16.dim("$")} mutagent playground run <prompt-id> --input '{}' --model gpt-4-turbo --json
|
|
5766
5854
|
|
|
5767
5855
|
Input Methods (pick one, priority order):
|
|
5768
|
-
--system/--human Quick system + user message ${
|
|
5856
|
+
--system/--human Quick system + user message ${chalk16.green("(recommended)")}
|
|
5769
5857
|
--input '{"key":"value"}' Inline JSON variables
|
|
5770
5858
|
--messages '[...]' Full messages array
|
|
5771
|
-
${
|
|
5859
|
+
${chalk16.dim(`Hint: Test before evaluating: mutagent playground run <id> --input '{"key":"value"}'`)}
|
|
5772
5860
|
`).action(async (promptId, options) => {
|
|
5773
5861
|
const isJson = getJsonFlag(playground);
|
|
5774
5862
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -5790,21 +5878,21 @@ ${chalk11.dim(`Hint: Test before evaluating: mutagent playground run <id> --inpu
|
|
|
5790
5878
|
}
|
|
5791
5879
|
});
|
|
5792
5880
|
} else {
|
|
5793
|
-
console.log(
|
|
5881
|
+
console.log(chalk16.bold(`
|
|
5794
5882
|
Execution Result:`));
|
|
5795
|
-
console.log(
|
|
5796
|
-
console.log(
|
|
5883
|
+
console.log(chalk16.gray("─".repeat(50)));
|
|
5884
|
+
console.log(chalk16.cyan("Output:"));
|
|
5797
5885
|
console.log(result.output);
|
|
5798
|
-
console.log(
|
|
5799
|
-
console.log(
|
|
5800
|
-
console.log(
|
|
5886
|
+
console.log(chalk16.gray("─".repeat(50)));
|
|
5887
|
+
console.log(chalk16.dim(`Model: ${result.model}`));
|
|
5888
|
+
console.log(chalk16.dim(`Execution Time: ${String(result.executionTimeMs)}ms`));
|
|
5801
5889
|
if (result.tokens) {
|
|
5802
|
-
console.log(
|
|
5890
|
+
console.log(chalk16.dim(`Tokens: ${String(result.tokens.prompt)} prompt + ${String(result.tokens.completion)} completion = ${String(result.tokens.total)} total`));
|
|
5803
5891
|
}
|
|
5804
5892
|
if (result.cost !== undefined) {
|
|
5805
|
-
console.log(
|
|
5893
|
+
console.log(chalk16.dim(`Cost: $${result.cost.toFixed(6)}`));
|
|
5806
5894
|
}
|
|
5807
|
-
console.log(
|
|
5895
|
+
console.log(chalk16.dim(`Playground: ${playgroundLink()}`));
|
|
5808
5896
|
console.log();
|
|
5809
5897
|
}
|
|
5810
5898
|
}
|
|
@@ -5893,9 +5981,9 @@ async function executeStreaming(client, promptId, input, model, isJson, output)
|
|
|
5893
5981
|
const decoder = new TextDecoder;
|
|
5894
5982
|
let buffer = "";
|
|
5895
5983
|
if (!isJson) {
|
|
5896
|
-
console.log(
|
|
5984
|
+
console.log(chalk16.bold(`
|
|
5897
5985
|
Streaming Output:`));
|
|
5898
|
-
console.log(
|
|
5986
|
+
console.log(chalk16.gray("─".repeat(50)));
|
|
5899
5987
|
}
|
|
5900
5988
|
try {
|
|
5901
5989
|
for (;; ) {
|
|
@@ -5934,15 +6022,15 @@ Streaming Output:`));
|
|
|
5934
6022
|
console.log(JSON.stringify({ type: "complete", result: event.result }));
|
|
5935
6023
|
} else {
|
|
5936
6024
|
console.log();
|
|
5937
|
-
console.log(
|
|
6025
|
+
console.log(chalk16.gray("─".repeat(50)));
|
|
5938
6026
|
if (event.result) {
|
|
5939
|
-
console.log(
|
|
5940
|
-
console.log(
|
|
6027
|
+
console.log(chalk16.dim(`Model: ${event.result.model}`));
|
|
6028
|
+
console.log(chalk16.dim(`Execution Time: ${String(event.result.executionTimeMs)}ms`));
|
|
5941
6029
|
if (event.result.tokens) {
|
|
5942
|
-
console.log(
|
|
6030
|
+
console.log(chalk16.dim(`Tokens: ${String(event.result.tokens.prompt)} prompt + ${String(event.result.tokens.completion)} completion = ${String(event.result.tokens.total)} total`));
|
|
5943
6031
|
}
|
|
5944
6032
|
if (event.result.cost !== undefined) {
|
|
5945
|
-
console.log(
|
|
6033
|
+
console.log(chalk16.dim(`Cost: $${event.result.cost.toFixed(6)}`));
|
|
5946
6034
|
}
|
|
5947
6035
|
}
|
|
5948
6036
|
console.log();
|
|
@@ -5966,14 +6054,14 @@ Streaming Output:`));
|
|
|
5966
6054
|
|
|
5967
6055
|
// src/commands/workspaces.ts
|
|
5968
6056
|
init_sdk_client();
|
|
5969
|
-
import { Command as
|
|
5970
|
-
import
|
|
6057
|
+
import { Command as Command12 } from "commander";
|
|
6058
|
+
import chalk17 from "chalk";
|
|
5971
6059
|
init_errors();
|
|
5972
6060
|
function createWorkspacesCommand() {
|
|
5973
|
-
const workspaces = new
|
|
6061
|
+
const workspaces = new Command12("workspaces").description("View workspaces (read-only)").addHelpText("after", `
|
|
5974
6062
|
Examples:
|
|
5975
|
-
${
|
|
5976
|
-
${
|
|
6063
|
+
${chalk17.dim("$")} mutagent workspaces list
|
|
6064
|
+
${chalk17.dim("$")} mutagent workspaces get <workspace-id>
|
|
5977
6065
|
|
|
5978
6066
|
Subcommands:
|
|
5979
6067
|
list, get
|
|
@@ -5982,8 +6070,8 @@ Note: Workspace management (create, update, delete) is available in the Admin Pa
|
|
|
5982
6070
|
`);
|
|
5983
6071
|
workspaces.command("list").description("List all workspaces").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").addHelpText("after", `
|
|
5984
6072
|
Examples:
|
|
5985
|
-
${
|
|
5986
|
-
${
|
|
6073
|
+
${chalk17.dim("$")} mutagent workspaces list
|
|
6074
|
+
${chalk17.dim("$")} mutagent workspaces list --limit 10 --json
|
|
5987
6075
|
`).action(async (options) => {
|
|
5988
6076
|
const isJson = getJsonFlag(workspaces);
|
|
5989
6077
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -6023,8 +6111,8 @@ Examples:
|
|
|
6023
6111
|
});
|
|
6024
6112
|
workspaces.command("get").description("Get workspace details").argument("<id>", "Workspace ID").addHelpText("after", `
|
|
6025
6113
|
Examples:
|
|
6026
|
-
${
|
|
6027
|
-
${
|
|
6114
|
+
${chalk17.dim("$")} mutagent workspaces get <workspace-id>
|
|
6115
|
+
${chalk17.dim("$")} mutagent workspaces get <workspace-id> --json
|
|
6028
6116
|
`).action(async (id) => {
|
|
6029
6117
|
const isJson = getJsonFlag(workspaces);
|
|
6030
6118
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -6056,8 +6144,8 @@ Examples:
|
|
|
6056
6144
|
|
|
6057
6145
|
// src/commands/providers.ts
|
|
6058
6146
|
init_sdk_client();
|
|
6059
|
-
import { Command as
|
|
6060
|
-
import
|
|
6147
|
+
import { Command as Command13 } from "commander";
|
|
6148
|
+
import chalk18 from "chalk";
|
|
6061
6149
|
init_errors();
|
|
6062
6150
|
var VALID_PROVIDER_TYPES = [
|
|
6063
6151
|
"openai",
|
|
@@ -6079,11 +6167,11 @@ function validateProviderType(type) {
|
|
|
6079
6167
|
return type;
|
|
6080
6168
|
}
|
|
6081
6169
|
function createProvidersCommand() {
|
|
6082
|
-
const providers = new
|
|
6170
|
+
const providers = new Command13("providers").description("View LLM providers (read-only)").addHelpText("after", `
|
|
6083
6171
|
Examples:
|
|
6084
|
-
${
|
|
6085
|
-
${
|
|
6086
|
-
${
|
|
6172
|
+
${chalk18.dim("$")} mutagent providers list
|
|
6173
|
+
${chalk18.dim("$")} mutagent providers get <provider-id>
|
|
6174
|
+
${chalk18.dim("$")} mutagent providers test <provider-id>
|
|
6087
6175
|
|
|
6088
6176
|
Provider Types:
|
|
6089
6177
|
openai, anthropic, google, azure, bedrock, cohere, mistral, groq, together, replicate, custom
|
|
@@ -6093,15 +6181,15 @@ Subcommands:
|
|
|
6093
6181
|
|
|
6094
6182
|
Note: Provider management (create, update, delete) is available in the Admin Panel only.
|
|
6095
6183
|
|
|
6096
|
-
${
|
|
6184
|
+
${chalk18.yellow("Note:")} The providers module is not yet active. This is a placeholder
|
|
6097
6185
|
for future external provider configuration. The server currently uses
|
|
6098
6186
|
built-in provider settings.
|
|
6099
6187
|
`);
|
|
6100
6188
|
providers.command("list").description("List all providers").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").option("-t, --type <type>", "Filter by provider type").addHelpText("after", `
|
|
6101
6189
|
Examples:
|
|
6102
|
-
${
|
|
6103
|
-
${
|
|
6104
|
-
${
|
|
6190
|
+
${chalk18.dim("$")} mutagent providers list
|
|
6191
|
+
${chalk18.dim("$")} mutagent providers list --type openai
|
|
6192
|
+
${chalk18.dim("$")} mutagent providers list --json
|
|
6105
6193
|
`).action(async (options) => {
|
|
6106
6194
|
const isJson = getJsonFlag(providers);
|
|
6107
6195
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -6128,7 +6216,7 @@ Examples:
|
|
|
6128
6216
|
}));
|
|
6129
6217
|
output.output({ ...result, data: withLinks });
|
|
6130
6218
|
} else {
|
|
6131
|
-
console.log(
|
|
6219
|
+
console.log(chalk18.yellow("Note: The providers module is not yet active. This is a placeholder for future external provider configuration."));
|
|
6132
6220
|
console.log("");
|
|
6133
6221
|
if (result.data.length === 0) {
|
|
6134
6222
|
output.info("No providers configured.");
|
|
@@ -6152,8 +6240,8 @@ Examples:
|
|
|
6152
6240
|
});
|
|
6153
6241
|
providers.command("get").description("Get provider details").argument("<id>", "Provider ID (from: mutagent providers list)").addHelpText("after", `
|
|
6154
6242
|
Examples:
|
|
6155
|
-
${
|
|
6156
|
-
${
|
|
6243
|
+
${chalk18.dim("$")} mutagent providers get <provider-id>
|
|
6244
|
+
${chalk18.dim("$")} mutagent providers get <provider-id> --json
|
|
6157
6245
|
`).action(async (id) => {
|
|
6158
6246
|
const isJson = getJsonFlag(providers);
|
|
6159
6247
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -6163,7 +6251,7 @@ Examples:
|
|
|
6163
6251
|
if (isJson) {
|
|
6164
6252
|
output.output({ ...provider, _links: providerLinks(provider.id) });
|
|
6165
6253
|
} else {
|
|
6166
|
-
console.log(
|
|
6254
|
+
console.log(chalk18.yellow("Note: The providers module is not yet active. This is a placeholder for future external provider configuration."));
|
|
6167
6255
|
console.log("");
|
|
6168
6256
|
const formatted = {
|
|
6169
6257
|
id: provider.id,
|
|
@@ -6184,17 +6272,17 @@ Examples:
|
|
|
6184
6272
|
});
|
|
6185
6273
|
providers.command("test").description("Test provider connectivity").argument("<id>", "Provider ID (from: mutagent providers list)").addHelpText("after", `
|
|
6186
6274
|
Examples:
|
|
6187
|
-
${
|
|
6188
|
-
${
|
|
6275
|
+
${chalk18.dim("$")} mutagent providers test <provider-id>
|
|
6276
|
+
${chalk18.dim("$")} mutagent providers test <provider-id> --json
|
|
6189
6277
|
|
|
6190
|
-
${
|
|
6278
|
+
${chalk18.dim("Tests connectivity and lists available models for the provider.")}
|
|
6191
6279
|
`).action(async (id) => {
|
|
6192
6280
|
const isJson = getJsonFlag(providers);
|
|
6193
6281
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
6194
6282
|
try {
|
|
6195
6283
|
const client = getSDKClient();
|
|
6196
6284
|
if (!isJson) {
|
|
6197
|
-
console.log(
|
|
6285
|
+
console.log(chalk18.yellow("Note: The providers module is not yet active. This is a placeholder for future external provider configuration."));
|
|
6198
6286
|
console.log("");
|
|
6199
6287
|
output.info(`Testing provider ${id}...`);
|
|
6200
6288
|
}
|
|
@@ -6204,9 +6292,9 @@ ${chalk13.dim("Tests connectivity and lists available models for the provider.")
|
|
|
6204
6292
|
} else {
|
|
6205
6293
|
if (result.success) {
|
|
6206
6294
|
output.success(`Provider test passed (${String(result.responseTimeMs)}ms)`);
|
|
6207
|
-
console.log(
|
|
6295
|
+
console.log(chalk18.green(`Message: ${result.message}`));
|
|
6208
6296
|
if (result.availableModels && result.availableModels.length > 0) {
|
|
6209
|
-
console.log(
|
|
6297
|
+
console.log(chalk18.bold(`
|
|
6210
6298
|
Available Models:`));
|
|
6211
6299
|
result.availableModels.forEach((model) => {
|
|
6212
6300
|
console.log(` - ${model}`);
|
|
@@ -6215,7 +6303,7 @@ Available Models:`));
|
|
|
6215
6303
|
} else {
|
|
6216
6304
|
output.error(`Provider test failed: ${result.message}`);
|
|
6217
6305
|
if (result.error) {
|
|
6218
|
-
console.log(
|
|
6306
|
+
console.log(chalk18.red(`Error: ${result.error}`));
|
|
6219
6307
|
}
|
|
6220
6308
|
}
|
|
6221
6309
|
}
|
|
@@ -6228,13 +6316,18 @@ Available Models:`));
|
|
|
6228
6316
|
|
|
6229
6317
|
// src/commands/init.ts
|
|
6230
6318
|
init_config();
|
|
6231
|
-
import { Command as
|
|
6319
|
+
import { Command as Command14 } from "commander";
|
|
6232
6320
|
import inquirer3 from "inquirer";
|
|
6233
|
-
import
|
|
6234
|
-
import { existsSync as
|
|
6321
|
+
import chalk19 from "chalk";
|
|
6322
|
+
import { existsSync as existsSync12, mkdirSync as mkdirSync3, writeFileSync as writeFileSync4 } from "fs";
|
|
6323
|
+
import { execSync as execSync3 } from "child_process";
|
|
6324
|
+
import { join as join6 } from "path";
|
|
6325
|
+
init_errors();
|
|
6326
|
+
|
|
6327
|
+
// src/lib/framework-detection.ts
|
|
6328
|
+
import { existsSync as existsSync11, readFileSync as readFileSync10 } from "fs";
|
|
6235
6329
|
import { execSync as execSync2 } from "child_process";
|
|
6236
6330
|
import { join as join5 } from "path";
|
|
6237
|
-
init_errors();
|
|
6238
6331
|
var FRAMEWORK_DETECTION_MAP = {
|
|
6239
6332
|
"@mastra/core": {
|
|
6240
6333
|
name: "mastra",
|
|
@@ -6284,16 +6377,16 @@ var FRAMEWORK_DETECTION_MAP = {
|
|
|
6284
6377
|
}
|
|
6285
6378
|
};
|
|
6286
6379
|
function detectPackageManager2(cwd = process.cwd()) {
|
|
6287
|
-
if (
|
|
6380
|
+
if (existsSync11(join5(cwd, "bun.lockb")) || existsSync11(join5(cwd, "bun.lock"))) {
|
|
6288
6381
|
return "bun";
|
|
6289
6382
|
}
|
|
6290
|
-
if (
|
|
6383
|
+
if (existsSync11(join5(cwd, "pnpm-lock.yaml"))) {
|
|
6291
6384
|
return "pnpm";
|
|
6292
6385
|
}
|
|
6293
|
-
if (
|
|
6386
|
+
if (existsSync11(join5(cwd, "yarn.lock"))) {
|
|
6294
6387
|
return "yarn";
|
|
6295
6388
|
}
|
|
6296
|
-
if (
|
|
6389
|
+
if (existsSync11(join5(cwd, "package-lock.json"))) {
|
|
6297
6390
|
return "npm";
|
|
6298
6391
|
}
|
|
6299
6392
|
try {
|
|
@@ -6315,12 +6408,12 @@ function getInstallCommand2(pm, packages) {
|
|
|
6315
6408
|
}
|
|
6316
6409
|
function detectFrameworkFromPackageJson(cwd = process.cwd()) {
|
|
6317
6410
|
const pkgPath = join5(cwd, "package.json");
|
|
6318
|
-
if (!
|
|
6411
|
+
if (!existsSync11(pkgPath)) {
|
|
6319
6412
|
return null;
|
|
6320
6413
|
}
|
|
6321
6414
|
let pkg;
|
|
6322
6415
|
try {
|
|
6323
|
-
pkg = JSON.parse(
|
|
6416
|
+
pkg = JSON.parse(readFileSync10(pkgPath, "utf-8"));
|
|
6324
6417
|
} catch {
|
|
6325
6418
|
return null;
|
|
6326
6419
|
}
|
|
@@ -6336,23 +6429,25 @@ function detectFrameworkFromPackageJson(cwd = process.cwd()) {
|
|
|
6336
6429
|
return null;
|
|
6337
6430
|
}
|
|
6338
6431
|
function hasRcConfig(cwd = process.cwd()) {
|
|
6339
|
-
return
|
|
6432
|
+
return existsSync11(join5(cwd, ".mutagentrc.json"));
|
|
6340
6433
|
}
|
|
6434
|
+
|
|
6435
|
+
// src/commands/init.ts
|
|
6341
6436
|
function writeRcConfig(config, cwd = process.cwd()) {
|
|
6342
|
-
const rcPath =
|
|
6437
|
+
const rcPath = join6(cwd, ".mutagentrc.json");
|
|
6343
6438
|
writeFileSync4(rcPath, JSON.stringify(config, null, 2) + `
|
|
6344
6439
|
`);
|
|
6345
6440
|
}
|
|
6346
6441
|
function createInitCommand() {
|
|
6347
|
-
const init = new
|
|
6442
|
+
const init = new Command14("init").description("Initialize MutagenT in your project").option("--non-interactive", "Skip interactive prompts (defaults to CLI-only mode)").addHelpText("after", `
|
|
6348
6443
|
Examples:
|
|
6349
|
-
${
|
|
6350
|
-
${
|
|
6444
|
+
${chalk19.dim("$")} mutagent init # Interactive setup wizard
|
|
6445
|
+
${chalk19.dim("$")} mutagent init --non-interactive # CLI-only mode (no prompts)
|
|
6351
6446
|
|
|
6352
6447
|
Modes:
|
|
6353
|
-
${
|
|
6354
|
-
${
|
|
6355
|
-
${
|
|
6448
|
+
${chalk19.bold("Full scaffold")} Install SDK + integration package, create config, setup tracing
|
|
6449
|
+
${chalk19.bold("CLI-only")} Verify auth + create .mutagentrc.json with workspace/endpoint
|
|
6450
|
+
${chalk19.bold("Skip")} Exit without changes
|
|
6356
6451
|
`).action(async (options) => {
|
|
6357
6452
|
const isJson = getJsonFlag(init);
|
|
6358
6453
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -6510,7 +6605,7 @@ Modes:
|
|
|
6510
6605
|
output.info(`Installing: ${packages.join(", ")}`);
|
|
6511
6606
|
output.info(`Running: ${installCmd}`);
|
|
6512
6607
|
try {
|
|
6513
|
-
|
|
6608
|
+
execSync3(installCmd, { cwd, stdio: "inherit" });
|
|
6514
6609
|
output.success("Packages installed successfully.");
|
|
6515
6610
|
} catch {
|
|
6516
6611
|
output.error("Package installation failed. You can install manually:");
|
|
@@ -6553,8 +6648,8 @@ Modes:
|
|
|
6553
6648
|
}
|
|
6554
6649
|
}
|
|
6555
6650
|
if (!isNonInteractive) {
|
|
6556
|
-
const skillPath =
|
|
6557
|
-
if (!
|
|
6651
|
+
const skillPath = join6(cwd, ".claude/skills/mutagent-cli/SKILL.md");
|
|
6652
|
+
if (!existsSync12(skillPath)) {
|
|
6558
6653
|
const { installSkill } = await inquirer3.prompt([{
|
|
6559
6654
|
type: "confirm",
|
|
6560
6655
|
name: "installSkill",
|
|
@@ -6563,11 +6658,11 @@ Modes:
|
|
|
6563
6658
|
}]);
|
|
6564
6659
|
if (installSkill) {
|
|
6565
6660
|
try {
|
|
6566
|
-
const skillDir =
|
|
6567
|
-
if (!
|
|
6661
|
+
const skillDir = join6(cwd, ".claude/skills/mutagent-cli");
|
|
6662
|
+
if (!existsSync12(skillDir)) {
|
|
6568
6663
|
mkdirSync3(skillDir, { recursive: true });
|
|
6569
6664
|
}
|
|
6570
|
-
|
|
6665
|
+
execSync3("node " + join6(cwd, "node_modules/.bin/mutagent") + " skills install", {
|
|
6571
6666
|
cwd,
|
|
6572
6667
|
stdio: "ignore"
|
|
6573
6668
|
});
|
|
@@ -6601,24 +6696,24 @@ Modes:
|
|
|
6601
6696
|
}
|
|
6602
6697
|
|
|
6603
6698
|
// src/commands/explore.ts
|
|
6604
|
-
import { Command as
|
|
6605
|
-
import
|
|
6699
|
+
import { Command as Command15 } from "commander";
|
|
6700
|
+
import chalk20 from "chalk";
|
|
6606
6701
|
import { resolve as resolve3 } from "path";
|
|
6607
6702
|
init_errors();
|
|
6608
6703
|
function createExploreCommand() {
|
|
6609
|
-
const explore = new
|
|
6704
|
+
const explore = new Command15("explore").description("Scan codebase for prompts, datasets, and MutagenT markers").option("-p, --path <dir>", "Directory to scan", ".").option("--depth <n>", "Max directory depth", "10").option("--include <glob>", "Include file pattern", "**/*.{ts,js,py,tsx,jsx}").option("--exclude <dirs>", "Comma-separated directories to exclude", "node_modules,dist,.git,build,.next,__pycache__,venv,.venv").option("--markers-only", "Only find existing MutagenT markers").addHelpText("after", `
|
|
6610
6705
|
Examples:
|
|
6611
|
-
${
|
|
6612
|
-
${
|
|
6613
|
-
${
|
|
6614
|
-
${
|
|
6615
|
-
${
|
|
6706
|
+
${chalk20.dim("$")} mutagent explore
|
|
6707
|
+
${chalk20.dim("$")} mutagent explore --path ./src
|
|
6708
|
+
${chalk20.dim("$")} mutagent explore --include "**/*.{ts,py}" --depth 5
|
|
6709
|
+
${chalk20.dim("$")} mutagent explore --markers-only
|
|
6710
|
+
${chalk20.dim("$")} mutagent explore --json
|
|
6616
6711
|
|
|
6617
6712
|
Detection modes:
|
|
6618
|
-
${
|
|
6619
|
-
${
|
|
6713
|
+
${chalk20.dim("Heuristic")} Template variables ({{var}}), prompt constants, schema definitions
|
|
6714
|
+
${chalk20.dim("Marker")} MutagenT:START/END comment markers from previous uploads
|
|
6620
6715
|
|
|
6621
|
-
${
|
|
6716
|
+
${chalk20.dim("Results are saved to .mutagent/mutation-context.md for use by other commands.")}
|
|
6622
6717
|
`).action((options) => {
|
|
6623
6718
|
const isJson = getJsonFlag(explore);
|
|
6624
6719
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -6636,7 +6731,7 @@ ${chalk15.dim("Results are saved to .mutagent/mutation-context.md for use by oth
|
|
|
6636
6731
|
markersOnly
|
|
6637
6732
|
};
|
|
6638
6733
|
if (!isJson) {
|
|
6639
|
-
console.log(
|
|
6734
|
+
console.log(chalk20.cyan(`
|
|
6640
6735
|
Scanning ${scanPath}...
|
|
6641
6736
|
`));
|
|
6642
6737
|
}
|
|
@@ -6665,40 +6760,41 @@ Scanning ${scanPath}...
|
|
|
6665
6760
|
const totalFindings = result.prompts.length + result.datasets.length + result.markers.length;
|
|
6666
6761
|
if (totalFindings === 0) {
|
|
6667
6762
|
output.info("No prompts, datasets, or markers found.");
|
|
6668
|
-
console.log(
|
|
6763
|
+
console.log(chalk20.dim(`
|
|
6669
6764
|
Tip: Create a prompt with template variables like {{input}} to get started.`));
|
|
6670
6765
|
return;
|
|
6671
6766
|
}
|
|
6672
6767
|
if (result.prompts.length > 0) {
|
|
6673
|
-
console.log(
|
|
6768
|
+
console.log(chalk20.bold(` Prompts Found (${String(result.prompts.length)}):`));
|
|
6674
6769
|
console.log();
|
|
6675
6770
|
for (const p of result.prompts) {
|
|
6676
|
-
const
|
|
6677
|
-
|
|
6678
|
-
console.log(`
|
|
6771
|
+
const confidenceTag = p.confidence === "high" ? chalk20.green("[high]") : p.confidence === "medium" ? chalk20.yellow("[medium]") : chalk20.dim("[low]");
|
|
6772
|
+
const reasonTag = chalk20.dim(`[${p.reason}]`);
|
|
6773
|
+
console.log(` ${confidenceTag} ${chalk20.green(p.file)}:${chalk20.yellow(String(p.line))} ${reasonTag}`);
|
|
6774
|
+
console.log(` ${chalk20.dim(p.preview)}`);
|
|
6679
6775
|
}
|
|
6680
6776
|
console.log();
|
|
6681
6777
|
}
|
|
6682
6778
|
if (result.datasets.length > 0) {
|
|
6683
|
-
console.log(
|
|
6779
|
+
console.log(chalk20.bold(` Datasets Found (${String(result.datasets.length)}):`));
|
|
6684
6780
|
console.log();
|
|
6685
6781
|
for (const d of result.datasets) {
|
|
6686
|
-
console.log(` ${
|
|
6782
|
+
console.log(` ${chalk20.green(d.file)} ${chalk20.dim(`(${String(d.items)} items)`)}`);
|
|
6687
6783
|
}
|
|
6688
6784
|
console.log();
|
|
6689
6785
|
}
|
|
6690
6786
|
if (result.markers.length > 0) {
|
|
6691
|
-
console.log(
|
|
6787
|
+
console.log(chalk20.bold(` MutagenT Markers (${String(result.markers.length)}):`));
|
|
6692
6788
|
console.log();
|
|
6693
6789
|
for (const m of result.markers) {
|
|
6694
|
-
const idPart = m.platformId ?
|
|
6695
|
-
console.log(` ${
|
|
6790
|
+
const idPart = m.platformId ? chalk20.cyan(` id=${m.platformId}`) : "";
|
|
6791
|
+
console.log(` ${chalk20.green(m.file)}:${chalk20.yellow(String(m.line))} ${chalk20.magenta(m.type)}${idPart}`);
|
|
6696
6792
|
}
|
|
6697
6793
|
console.log();
|
|
6698
6794
|
}
|
|
6699
|
-
console.log(
|
|
6700
|
-
console.log(` ${
|
|
6701
|
-
console.log(
|
|
6795
|
+
console.log(chalk20.dim(" ─────────────────────────────────"));
|
|
6796
|
+
console.log(` ${chalk20.bold("Summary:")} ${String(result.prompts.length)} prompts, ${String(result.datasets.length)} datasets, ${String(result.markers.length)} markers`);
|
|
6797
|
+
console.log(chalk20.dim(` Saved to .mutagent/mutation-context.md`));
|
|
6702
6798
|
console.log();
|
|
6703
6799
|
}
|
|
6704
6800
|
} catch (error) {
|
|
@@ -6709,14 +6805,14 @@ Scanning ${scanPath}...
|
|
|
6709
6805
|
}
|
|
6710
6806
|
|
|
6711
6807
|
// src/commands/skills.ts
|
|
6712
|
-
import { Command as
|
|
6713
|
-
import
|
|
6714
|
-
import { existsSync as
|
|
6715
|
-
import { join as
|
|
6716
|
-
import { execSync as
|
|
6808
|
+
import { Command as Command16 } from "commander";
|
|
6809
|
+
import chalk21 from "chalk";
|
|
6810
|
+
import { existsSync as existsSync13, mkdirSync as mkdirSync4, writeFileSync as writeFileSync5 } from "fs";
|
|
6811
|
+
import { join as join7 } from "path";
|
|
6812
|
+
import { execSync as execSync4 } from "child_process";
|
|
6717
6813
|
function findRepoRoot() {
|
|
6718
6814
|
try {
|
|
6719
|
-
return
|
|
6815
|
+
return execSync4("git rev-parse --show-toplevel", { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
|
|
6720
6816
|
} catch {
|
|
6721
6817
|
return process.cwd();
|
|
6722
6818
|
}
|
|
@@ -6830,10 +6926,10 @@ Every evaluation MUST specify criteria targeting either:
|
|
|
6830
6926
|
var SKILL_DIR = ".claude/skills/mutagent-cli";
|
|
6831
6927
|
var SKILL_FILE = "SKILL.md";
|
|
6832
6928
|
function createSkillsCommand() {
|
|
6833
|
-
const skills = new
|
|
6929
|
+
const skills = new Command16("skills").description("Manage MutagenT CLI skills for coding agents");
|
|
6834
6930
|
skills.command("install").description("Install MutagenT CLI skill for Claude Code").addHelpText("after", `
|
|
6835
6931
|
Examples:
|
|
6836
|
-
${
|
|
6932
|
+
${chalk21.dim("$")} mutagent skills install
|
|
6837
6933
|
|
|
6838
6934
|
This creates a Claude Code skill at .claude/skills/mutagent-cli/SKILL.md
|
|
6839
6935
|
that teaches coding agents how to use the MutagenT CLI effectively.
|
|
@@ -6842,9 +6938,9 @@ that teaches coding agents how to use the MutagenT CLI effectively.
|
|
|
6842
6938
|
const isJson = parentCmd ? getJsonFlag(parentCmd) : false;
|
|
6843
6939
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
6844
6940
|
const repoRoot = findRepoRoot();
|
|
6845
|
-
const skillDir =
|
|
6846
|
-
const skillPath =
|
|
6847
|
-
if (!
|
|
6941
|
+
const skillDir = join7(repoRoot, SKILL_DIR);
|
|
6942
|
+
const skillPath = join7(skillDir, SKILL_FILE);
|
|
6943
|
+
if (!existsSync13(skillDir)) {
|
|
6848
6944
|
mkdirSync4(skillDir, { recursive: true });
|
|
6849
6945
|
}
|
|
6850
6946
|
const content = `${SKILL_FRONTMATTER}
|
|
@@ -6860,10 +6956,10 @@ ${SKILL_BODY}
|
|
|
6860
6956
|
});
|
|
6861
6957
|
} else {
|
|
6862
6958
|
output.success(`Installed MutagenT CLI skill`);
|
|
6863
|
-
console.log(` ${
|
|
6959
|
+
console.log(` ${chalk21.dim("Path:")} ${skillPath}`);
|
|
6864
6960
|
console.log("");
|
|
6865
|
-
console.log(` ${
|
|
6866
|
-
console.log(` ${
|
|
6961
|
+
console.log(` ${chalk21.dim("This skill teaches coding agents how to use the MutagenT CLI.")}`);
|
|
6962
|
+
console.log(` ${chalk21.dim("It will be automatically loaded by Claude Code when relevant triggers match.")}`);
|
|
6867
6963
|
}
|
|
6868
6964
|
});
|
|
6869
6965
|
return skills;
|
|
@@ -6871,8 +6967,8 @@ ${SKILL_BODY}
|
|
|
6871
6967
|
|
|
6872
6968
|
// src/commands/usage.ts
|
|
6873
6969
|
init_config();
|
|
6874
|
-
import { Command as
|
|
6875
|
-
import
|
|
6970
|
+
import { Command as Command17 } from "commander";
|
|
6971
|
+
import chalk22 from "chalk";
|
|
6876
6972
|
init_errors();
|
|
6877
6973
|
init_sdk_client();
|
|
6878
6974
|
var TRIAL_OPTIMIZATION_LIMIT = 5;
|
|
@@ -6886,10 +6982,10 @@ function renderProgressBar(used, limit, width = 30) {
|
|
|
6886
6982
|
return `${bar} ${String(percent)}%`;
|
|
6887
6983
|
}
|
|
6888
6984
|
function createUsageCommand() {
|
|
6889
|
-
const usage = new
|
|
6985
|
+
const usage = new Command17("usage").description("Show resource counts and optimization run limits").addHelpText("after", `
|
|
6890
6986
|
Examples:
|
|
6891
|
-
${
|
|
6892
|
-
${
|
|
6987
|
+
${chalk22.dim("$")} mutagent usage
|
|
6988
|
+
${chalk22.dim("$")} mutagent usage --json
|
|
6893
6989
|
`);
|
|
6894
6990
|
usage.action(async () => {
|
|
6895
6991
|
const isJson = getJsonFlag(usage);
|
|
@@ -6944,21 +7040,21 @@ Examples:
|
|
|
6944
7040
|
});
|
|
6945
7041
|
} else {
|
|
6946
7042
|
console.log("");
|
|
6947
|
-
console.log(
|
|
6948
|
-
console.log(
|
|
7043
|
+
console.log(chalk22.bold("\uD83D\uDCCA MutagenT Usage"));
|
|
7044
|
+
console.log(chalk22.dim("─".repeat(45)));
|
|
6949
7045
|
console.log("");
|
|
6950
|
-
console.log(
|
|
6951
|
-
console.log(` Prompts: ${
|
|
6952
|
-
console.log(` Datasets: ${
|
|
6953
|
-
console.log(` Evaluations: ${
|
|
7046
|
+
console.log(chalk22.bold("Resources:"));
|
|
7047
|
+
console.log(` Prompts: ${chalk22.cyan(String(promptCount))}`);
|
|
7048
|
+
console.log(` Datasets: ${chalk22.cyan(String(datasetCount))}`);
|
|
7049
|
+
console.log(` Evaluations: ${chalk22.cyan(String(evaluationCount))}`);
|
|
6954
7050
|
console.log("");
|
|
6955
|
-
console.log(
|
|
6956
|
-
console.log(` Remaining: ${
|
|
7051
|
+
console.log(chalk22.bold(`Optimization Runs (${chalk22.yellow("trial")} plan):`));
|
|
7052
|
+
console.log(` Remaining: ${chalk22.cyan(String(optimizationRemaining))} / ${String(optimizationLimit)}`);
|
|
6957
7053
|
console.log(` ${renderProgressBar(optimizationUsed, optimizationLimit)}`);
|
|
6958
7054
|
console.log("");
|
|
6959
|
-
console.log(
|
|
6960
|
-
console.log(
|
|
6961
|
-
console.log(` Upgrade: ${
|
|
7055
|
+
console.log(chalk22.yellow(` ⚠ ${String(optimizationRemaining)} optimization runs remaining`));
|
|
7056
|
+
console.log(chalk22.dim(` ℹ Optimization run counts are approximate`));
|
|
7057
|
+
console.log(` Upgrade: ${chalk22.underline(BILLING_URL)}`);
|
|
6962
7058
|
console.log("");
|
|
6963
7059
|
}
|
|
6964
7060
|
} catch (error) {
|
|
@@ -6976,12 +7072,12 @@ if (process.env.CLI_VERSION) {
|
|
|
6976
7072
|
} else {
|
|
6977
7073
|
try {
|
|
6978
7074
|
const __dirname2 = dirname(fileURLToPath(import.meta.url));
|
|
6979
|
-
const pkgPath =
|
|
6980
|
-
const pkg = JSON.parse(
|
|
7075
|
+
const pkgPath = join8(__dirname2, "..", "..", "package.json");
|
|
7076
|
+
const pkg = JSON.parse(readFileSync11(pkgPath, "utf-8"));
|
|
6981
7077
|
cliVersion = pkg.version ?? cliVersion;
|
|
6982
7078
|
} catch {}
|
|
6983
7079
|
}
|
|
6984
|
-
var program = new
|
|
7080
|
+
var program = new Command18;
|
|
6985
7081
|
program.name("mutagent").description(`MutagenT CLI - AI-native prompt optimization platform
|
|
6986
7082
|
|
|
6987
7083
|
Documentation: https://docs.mutagent.io/cli
|
|
@@ -6990,46 +7086,46 @@ program.name("mutagent").description(`MutagenT CLI - AI-native prompt optimizati
|
|
|
6990
7086
|
showGlobalOptions: true
|
|
6991
7087
|
});
|
|
6992
7088
|
program.addHelpText("after", `
|
|
6993
|
-
${
|
|
6994
|
-
export MUTAGENT_API_KEY=mt_... ${
|
|
6995
|
-
--json ${
|
|
6996
|
-
|
|
6997
|
-
${
|
|
6998
|
-
mutagent auth login --browser ${
|
|
6999
|
-
mutagent auth status ${
|
|
7000
|
-
mutagent init ${
|
|
7001
|
-
mutagent explore ${
|
|
7002
|
-
mutagent workspaces list --json ${
|
|
7003
|
-
mutagent config set workspace <id> ${
|
|
7004
|
-
mutagent usage --json ${
|
|
7005
|
-
|
|
7006
|
-
mutagent prompts create --help ${
|
|
7007
|
-
mutagent prompts list --json ${
|
|
7008
|
-
mutagent prompts get <id> --json ${
|
|
7009
|
-
|
|
7010
|
-
mutagent prompts dataset add --help ${
|
|
7011
|
-
mutagent prompts dataset list <id> ${
|
|
7012
|
-
|
|
7013
|
-
mutagent prompts evaluation create --help ${
|
|
7014
|
-
mutagent prompts evaluation create <id> --guided --json ${
|
|
7015
|
-
mutagent prompts evaluation list <id> --json ${
|
|
7016
|
-
|
|
7017
|
-
mutagent prompts optimize start --help ${
|
|
7018
|
-
mutagent prompts optimize status <job-id> ${
|
|
7019
|
-
mutagent prompts optimize results <job-id> ${
|
|
7020
|
-
|
|
7021
|
-
mutagent integrate <framework> ${
|
|
7022
|
-
mutagent playground run <id> --input '{...}' ${
|
|
7023
|
-
|
|
7024
|
-
${
|
|
7025
|
-
1. mutagent prompts create --help ${
|
|
7026
|
-
2. mutagent prompts create ... --json ${
|
|
7027
|
-
3. mutagent prompts dataset add --help ${
|
|
7028
|
-
4. mutagent prompts dataset add <id> ... --json ${
|
|
7029
|
-
5. mutagent prompts evaluation create <id> --guided --json ${
|
|
7089
|
+
${chalk23.yellow("Non-Interactive Mode (CI/CD & Coding Agents):")}
|
|
7090
|
+
export MUTAGENT_API_KEY=mt_... ${chalk23.dim("or")} --api-key mt_...
|
|
7091
|
+
--json ${chalk23.dim("for structured output")} --non-interactive ${chalk23.dim("to disable prompts")}
|
|
7092
|
+
|
|
7093
|
+
${chalk23.yellow("Command Navigation:")}
|
|
7094
|
+
mutagent auth login --browser ${chalk23.dim("Authenticate (OAuth)")}
|
|
7095
|
+
mutagent auth status ${chalk23.dim("Check auth + workspace")}
|
|
7096
|
+
mutagent init ${chalk23.dim("Initialize project (.mutagentrc.json)")}
|
|
7097
|
+
mutagent explore ${chalk23.dim("Discover prompts in codebase")}
|
|
7098
|
+
mutagent workspaces list --json ${chalk23.dim("List workspaces (verify ID)")}
|
|
7099
|
+
mutagent config set workspace <id> ${chalk23.dim("Set active workspace")}
|
|
7100
|
+
mutagent usage --json ${chalk23.dim("Check plan limits")}
|
|
7101
|
+
|
|
7102
|
+
mutagent prompts create --help ${chalk23.dim("Upload prompt (read help first!)")}
|
|
7103
|
+
mutagent prompts list --json ${chalk23.dim("List prompts")}
|
|
7104
|
+
mutagent prompts get <id> --json ${chalk23.dim("Full prompt details + schemas")}
|
|
7105
|
+
|
|
7106
|
+
mutagent prompts dataset add --help ${chalk23.dim("Upload dataset (read help first!)")}
|
|
7107
|
+
mutagent prompts dataset list <id> ${chalk23.dim("List datasets")}
|
|
7108
|
+
|
|
7109
|
+
mutagent prompts evaluation create --help ${chalk23.dim("Create eval (read help first!)")}
|
|
7110
|
+
mutagent prompts evaluation create <id> --guided --json ${chalk23.dim("Guided eval workflow")}
|
|
7111
|
+
mutagent prompts evaluation list <id> --json ${chalk23.dim("List evaluations")}
|
|
7112
|
+
|
|
7113
|
+
mutagent prompts optimize start --help ${chalk23.dim("Run optimization (read help first!)")}
|
|
7114
|
+
mutagent prompts optimize status <job-id> ${chalk23.dim("Poll progress")}
|
|
7115
|
+
mutagent prompts optimize results <job-id> ${chalk23.dim("View scorecard")}
|
|
7116
|
+
|
|
7117
|
+
mutagent integrate <framework> ${chalk23.dim("Framework integration guide")}
|
|
7118
|
+
mutagent playground run <id> --input '{...}' ${chalk23.dim("Quick test")}
|
|
7119
|
+
|
|
7120
|
+
${chalk23.yellow("Workflow: Evaluate → Optimize:")}
|
|
7121
|
+
1. mutagent prompts create --help ${chalk23.dim("← read help")}
|
|
7122
|
+
2. mutagent prompts create ... --json ${chalk23.dim("← upload prompt with {variables} + inputSchema")}
|
|
7123
|
+
3. mutagent prompts dataset add --help ${chalk23.dim("← read help")}
|
|
7124
|
+
4. mutagent prompts dataset add <id> ... --json ${chalk23.dim("← upload dataset")}
|
|
7125
|
+
5. mutagent prompts evaluation create <id> --guided --json ${chalk23.dim("← guided eval")}
|
|
7030
7126
|
6. mutagent prompts optimize start <id> --dataset <d> --evaluation <e> --json
|
|
7031
7127
|
|
|
7032
|
-
${
|
|
7128
|
+
${chalk23.yellow("AI Agent Rules (MANDATORY for coding agents):")}
|
|
7033
7129
|
1. EVERY command MUST include --json (no exceptions)
|
|
7034
7130
|
2. Run <command> --help BEFORE first use of any command
|
|
7035
7131
|
3. Use --guided --json for evaluation creation (NEVER --guided alone)
|
|
@@ -7038,9 +7134,9 @@ ${chalk18.yellow("AI Agent Rules (MANDATORY for coding agents):")}
|
|
|
7038
7134
|
6. Use {single_braces} for template variables in prompts
|
|
7039
7135
|
7. Collect evaluation rubrics from the user — NEVER auto-generate
|
|
7040
7136
|
${!hasCredentials() ? `
|
|
7041
|
-
` +
|
|
7137
|
+
` + chalk23.yellow(" Warning: Not authenticated. Run: mutagent auth login --browser") + `
|
|
7042
7138
|
` : ""}${!hasRcConfig() ? `
|
|
7043
|
-
` +
|
|
7139
|
+
` + chalk23.green(" Get started: mutagent init") + `
|
|
7044
7140
|
` : ""}`);
|
|
7045
7141
|
program.hook("preAction", (thisCommand) => {
|
|
7046
7142
|
const globalOpts = thisCommand.optsWithGlobals();
|
|
@@ -7070,5 +7166,5 @@ program.addCommand(createSkillsCommand());
|
|
|
7070
7166
|
program.addCommand(createUsageCommand());
|
|
7071
7167
|
program.parse();
|
|
7072
7168
|
|
|
7073
|
-
//# debugId=
|
|
7169
|
+
//# debugId=88ECCF736E0A64AD64756E2164756E21
|
|
7074
7170
|
//# sourceMappingURL=cli.js.map
|