@mutagent/cli 0.1.15 → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cli.js +942 -848
- package/dist/bin/cli.js.map +23 -25
- package/dist/index.js +37 -3
- package/dist/index.js.map +3 -3
- package/package.json +1 -1
package/dist/bin/cli.js
CHANGED
|
@@ -464,6 +464,7 @@ class SDKClientWrapper {
|
|
|
464
464
|
systemPrompt: data.systemPrompt ?? undefined,
|
|
465
465
|
humanPrompt: data.humanPrompt ?? undefined,
|
|
466
466
|
rawPrompt: data.rawPrompt ?? undefined,
|
|
467
|
+
inputSchema: data.inputSchema,
|
|
467
468
|
outputSchema: data.outputSchema,
|
|
468
469
|
metadata: data.metadata,
|
|
469
470
|
tags: data.tags ?? undefined
|
|
@@ -529,11 +530,13 @@ class SDKClientWrapper {
|
|
|
529
530
|
labels: item.labels,
|
|
530
531
|
metadata: item.metadata
|
|
531
532
|
}));
|
|
533
|
+
const missingExpectedOutputCount = mappedItems.filter((item) => item.expectedOutput === undefined).length;
|
|
532
534
|
await this.request(`/api/prompts/datasets/${String(dataset.id)}/items/bulk`, {
|
|
533
535
|
method: "POST",
|
|
534
536
|
body: JSON.stringify({ items: mappedItems })
|
|
535
537
|
});
|
|
536
538
|
itemCount = mappedItems.length;
|
|
539
|
+
return { ...dataset, itemCount, missingExpectedOutputCount };
|
|
537
540
|
}
|
|
538
541
|
return { ...dataset, itemCount };
|
|
539
542
|
} catch (error) {
|
|
@@ -574,6 +577,13 @@ class SDKClientWrapper {
|
|
|
574
577
|
this.handleError(error);
|
|
575
578
|
}
|
|
576
579
|
}
|
|
580
|
+
async deleteEvaluation(evaluationId) {
|
|
581
|
+
try {
|
|
582
|
+
await this.request(`/api/prompts/evaluations/${evaluationId}`, { method: "DELETE" });
|
|
583
|
+
} catch (error) {
|
|
584
|
+
this.handleError(error);
|
|
585
|
+
}
|
|
586
|
+
}
|
|
577
587
|
async getEvaluationResults(runId) {
|
|
578
588
|
try {
|
|
579
589
|
return await this.request(`/api/prompts/evaluations/${runId}/result`);
|
|
@@ -588,8 +598,8 @@ class SDKClientWrapper {
|
|
|
588
598
|
body: JSON.stringify({
|
|
589
599
|
datasetId: parseInt(datasetId, 10),
|
|
590
600
|
config: {
|
|
591
|
-
maxIterations: config?.maxIterations ??
|
|
592
|
-
targetScore: config?.targetScore,
|
|
601
|
+
maxIterations: config?.maxIterations ?? 1,
|
|
602
|
+
targetScore: config?.targetScore ?? 0.8,
|
|
593
603
|
patience: config?.patience,
|
|
594
604
|
model: config?.model,
|
|
595
605
|
...config?.evalModel ? { tuningParams: { evaluationModel: config.evalModel } } : {}
|
|
@@ -849,6 +859,12 @@ class SDKClientWrapper {
|
|
|
849
859
|
this.handleError(error);
|
|
850
860
|
}
|
|
851
861
|
}
|
|
862
|
+
getCurrentWorkspaceId() {
|
|
863
|
+
return this.workspaceId;
|
|
864
|
+
}
|
|
865
|
+
getCurrentOrgId() {
|
|
866
|
+
return this.organizationId;
|
|
867
|
+
}
|
|
852
868
|
async testProvider(id) {
|
|
853
869
|
try {
|
|
854
870
|
const response = await this.sdk.providerConfigs.testProvider({
|
|
@@ -922,9 +938,9 @@ var init_sdk_client = __esm(() => {
|
|
|
922
938
|
});
|
|
923
939
|
|
|
924
940
|
// src/bin/cli.ts
|
|
925
|
-
import { Command as
|
|
926
|
-
import
|
|
927
|
-
import { readFileSync as
|
|
941
|
+
import { Command as Command15 } from "commander";
|
|
942
|
+
import chalk19 from "chalk";
|
|
943
|
+
import { readFileSync as readFileSync12 } from "fs";
|
|
928
944
|
import { join as join7, dirname } from "path";
|
|
929
945
|
import { fileURLToPath } from "url";
|
|
930
946
|
|
|
@@ -1058,6 +1074,24 @@ ${String(data.length)} result(s)`));
|
|
|
1058
1074
|
}
|
|
1059
1075
|
}
|
|
1060
1076
|
}
|
|
1077
|
+
function createSpinner(text, isJson) {
|
|
1078
|
+
if (isJson) {
|
|
1079
|
+
const noop = () => {
|
|
1080
|
+
return;
|
|
1081
|
+
};
|
|
1082
|
+
return {
|
|
1083
|
+
start: noop,
|
|
1084
|
+
stop: noop,
|
|
1085
|
+
succeed: noop,
|
|
1086
|
+
fail: noop
|
|
1087
|
+
};
|
|
1088
|
+
}
|
|
1089
|
+
const start = async () => {
|
|
1090
|
+
const { default: ora } = await import("ora");
|
|
1091
|
+
return ora(text).start();
|
|
1092
|
+
};
|
|
1093
|
+
return { start };
|
|
1094
|
+
}
|
|
1061
1095
|
|
|
1062
1096
|
// src/commands/auth.ts
|
|
1063
1097
|
init_errors();
|
|
@@ -1779,12 +1813,10 @@ async function runIntegrationPath() {
|
|
|
1779
1813
|
name: "framework",
|
|
1780
1814
|
message: "Which AI framework are you using?",
|
|
1781
1815
|
choices: [
|
|
1782
|
-
{ name: "Mastra", value: "mastra" },
|
|
1783
1816
|
{ name: "LangChain", value: "langchain" },
|
|
1784
1817
|
{ name: "LangGraph", value: "langgraph" },
|
|
1785
1818
|
{ name: "Vercel AI SDK", value: "vercel-ai" },
|
|
1786
|
-
{ name: "
|
|
1787
|
-
{ name: "Other / Generic", value: "generic" }
|
|
1819
|
+
{ name: "OpenAI SDK", value: "openai" }
|
|
1788
1820
|
]
|
|
1789
1821
|
}]);
|
|
1790
1822
|
console.log("");
|
|
@@ -2419,17 +2451,39 @@ function isValidJsonSchema(schema) {
|
|
|
2419
2451
|
function buildSchemaFromVariables(variables) {
|
|
2420
2452
|
const properties = {};
|
|
2421
2453
|
for (const variable of variables) {
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
}
|
|
2426
|
-
properties[variable.name] = prop;
|
|
2454
|
+
properties[variable.name] = {
|
|
2455
|
+
type: variable.type,
|
|
2456
|
+
description: variable.description
|
|
2457
|
+
};
|
|
2427
2458
|
}
|
|
2428
2459
|
return {
|
|
2429
2460
|
type: "object",
|
|
2430
2461
|
properties
|
|
2431
2462
|
};
|
|
2432
2463
|
}
|
|
2464
|
+
function validateSchemaDescriptions(schema) {
|
|
2465
|
+
if (schema === null || schema === undefined)
|
|
2466
|
+
return [];
|
|
2467
|
+
if (typeof schema !== "object" || Array.isArray(schema))
|
|
2468
|
+
return [];
|
|
2469
|
+
const obj = schema;
|
|
2470
|
+
const properties = obj.properties;
|
|
2471
|
+
if (!properties || typeof properties !== "object" || Array.isArray(properties))
|
|
2472
|
+
return [];
|
|
2473
|
+
const missing = [];
|
|
2474
|
+
const props = properties;
|
|
2475
|
+
for (const [key, value] of Object.entries(props)) {
|
|
2476
|
+
if (value === null || value === undefined || typeof value !== "object") {
|
|
2477
|
+
missing.push(key);
|
|
2478
|
+
continue;
|
|
2479
|
+
}
|
|
2480
|
+
const prop = value;
|
|
2481
|
+
if (typeof prop.description !== "string" || prop.description.trim() === "") {
|
|
2482
|
+
missing.push(key);
|
|
2483
|
+
}
|
|
2484
|
+
}
|
|
2485
|
+
return missing;
|
|
2486
|
+
}
|
|
2433
2487
|
function formatSchemaWarning(fieldName) {
|
|
2434
2488
|
return [
|
|
2435
2489
|
`${fieldName} doesn't appear to be a valid JSON Schema.`,
|
|
@@ -2534,9 +2588,17 @@ async function runGuidedEvalCreator(promptId) {
|
|
|
2534
2588
|
}
|
|
2535
2589
|
}]);
|
|
2536
2590
|
let targetField;
|
|
2537
|
-
|
|
2591
|
+
const usedParams = new Set(criteria.map((c) => c.evaluationParameter));
|
|
2592
|
+
const availableFields = allFields.filter((f) => {
|
|
2593
|
+
const param = f.startsWith("output.") ? f.slice("output.".length) : f.startsWith("input.") ? f.slice("input.".length) : f;
|
|
2594
|
+
return !usedParams.has(param);
|
|
2595
|
+
});
|
|
2596
|
+
if (availableFields.length === 0 && allFields.length > 0) {
|
|
2597
|
+
console.log(chalk5.yellow(" All detected schema fields are already used by existing criteria."));
|
|
2598
|
+
}
|
|
2599
|
+
if (availableFields.length > 0) {
|
|
2538
2600
|
const fieldChoices = [
|
|
2539
|
-
...
|
|
2601
|
+
...availableFields.map((f) => ({ name: f, value: f })),
|
|
2540
2602
|
{ name: "(custom field name)", value: "__custom__" }
|
|
2541
2603
|
];
|
|
2542
2604
|
const { field } = await inquirer3.prompt([{
|
|
@@ -2593,10 +2655,15 @@ async function runGuidedEvalCreator(promptId) {
|
|
|
2593
2655
|
} else {
|
|
2594
2656
|
scoringRubric = rubric;
|
|
2595
2657
|
}
|
|
2658
|
+
const evaluationParameter = targetField.startsWith("output.") ? targetField.slice("output.".length) : targetField.startsWith("input.") ? targetField.slice("input.".length) : targetField;
|
|
2659
|
+
if (criteria.some((c) => c.evaluationParameter === evaluationParameter)) {
|
|
2660
|
+
console.log(chalk5.red(` Error: "${evaluationParameter}" is already used by another criterion. Each criterion must target a unique output field.`));
|
|
2661
|
+
continue;
|
|
2662
|
+
}
|
|
2596
2663
|
criteria.push({
|
|
2597
2664
|
name: criterionName.trim(),
|
|
2598
|
-
|
|
2599
|
-
|
|
2665
|
+
description: scoringRubric,
|
|
2666
|
+
evaluationParameter,
|
|
2600
2667
|
weight: 1
|
|
2601
2668
|
});
|
|
2602
2669
|
const { continueAdding } = await inquirer3.prompt([{
|
|
@@ -2618,7 +2685,7 @@ async function runGuidedEvalCreator(promptId) {
|
|
|
2618
2685
|
console.log(` Type: ${evalType}`);
|
|
2619
2686
|
console.log(` Criteria: ${String(criteria.length)}`);
|
|
2620
2687
|
for (const c of criteria) {
|
|
2621
|
-
console.log(` - ${chalk5.cyan(c.name)} → ${c.
|
|
2688
|
+
console.log(` - ${chalk5.cyan(c.name)} → ${c.evaluationParameter}`);
|
|
2622
2689
|
}
|
|
2623
2690
|
console.log("");
|
|
2624
2691
|
const { confirmed } = await inquirer3.prompt([{
|
|
@@ -2691,6 +2758,38 @@ function renderScorecard(data) {
|
|
|
2691
2758
|
console.log(line(` Prompt: ${chalk6.cyan(truncateText(optimizedText, 38))}`));
|
|
2692
2759
|
console.log(line(` Score: ${formatScore(bestScore)}${formatScoreChange(originalScore, bestScore)}`));
|
|
2693
2760
|
console.log(separator);
|
|
2761
|
+
if (data.criteriaScores && data.criteriaScores.length > 0) {
|
|
2762
|
+
console.log(line(chalk6.dim(" Criterion Before After Change")));
|
|
2763
|
+
console.log(line(chalk6.dim(" " + "─".repeat(45))));
|
|
2764
|
+
for (const c of data.criteriaScores) {
|
|
2765
|
+
const name = c.name.length > 16 ? c.name.substring(0, 13) + "..." : c.name;
|
|
2766
|
+
const paddedName = name + " ".repeat(18 - name.length);
|
|
2767
|
+
const beforeStr = c.before !== undefined ? c.before.toFixed(2) : "N/A ";
|
|
2768
|
+
const afterStr = c.after !== undefined ? c.after.toFixed(2) : "N/A ";
|
|
2769
|
+
const changeStr = c.before !== undefined && c.after !== undefined && c.before > 0 ? (() => {
|
|
2770
|
+
const pct = Math.round((c.after - c.before) / c.before * 100);
|
|
2771
|
+
if (pct > 0)
|
|
2772
|
+
return chalk6.green(`+${String(pct)}%`);
|
|
2773
|
+
if (pct < 0)
|
|
2774
|
+
return chalk6.red(`${String(pct)}%`);
|
|
2775
|
+
return chalk6.dim("0%");
|
|
2776
|
+
})() : "";
|
|
2777
|
+
console.log(line(` ${paddedName}${beforeStr} ${afterStr} ${changeStr}`));
|
|
2778
|
+
}
|
|
2779
|
+
console.log(line(chalk6.dim(" " + "─".repeat(45))));
|
|
2780
|
+
const overallBefore = originalScore !== undefined ? originalScore.toFixed(2) : "N/A ";
|
|
2781
|
+
const overallAfter = bestScore !== undefined ? bestScore.toFixed(2) : "N/A ";
|
|
2782
|
+
const overallChange = originalScore !== undefined && bestScore !== undefined && originalScore > 0 ? (() => {
|
|
2783
|
+
const pct = Math.round((bestScore - originalScore) / originalScore * 100);
|
|
2784
|
+
if (pct > 0)
|
|
2785
|
+
return chalk6.green(`+${String(pct)}%`);
|
|
2786
|
+
if (pct < 0)
|
|
2787
|
+
return chalk6.red(`${String(pct)}%`);
|
|
2788
|
+
return chalk6.dim("0%");
|
|
2789
|
+
})() : "";
|
|
2790
|
+
console.log(line(` ${"Overall" + " ".repeat(11)}${overallBefore} ${overallAfter} ${overallChange}`));
|
|
2791
|
+
console.log(separator);
|
|
2792
|
+
}
|
|
2694
2793
|
const statusStr = job.status === "completed" ? chalk6.green("completed") : chalk6.yellow(job.status);
|
|
2695
2794
|
console.log(line(`Status: ${statusStr} | Iterations: ${String(iterations)}`));
|
|
2696
2795
|
if (job.config?.model) {
|
|
@@ -2699,19 +2798,16 @@ function renderScorecard(data) {
|
|
|
2699
2798
|
if (data.scoreProgression && data.scoreProgression.length > 0) {
|
|
2700
2799
|
console.log(line(""));
|
|
2701
2800
|
console.log(line(chalk6.dim("Score Progression:")));
|
|
2702
|
-
const
|
|
2703
|
-
|
|
2704
|
-
const
|
|
2705
|
-
const
|
|
2706
|
-
const
|
|
2707
|
-
console.log(line(chalk6.dim(
|
|
2708
|
-
console.log(line(chalk6.dim(line2)));
|
|
2709
|
-
} else {
|
|
2710
|
-
console.log(line(chalk6.dim(progression)));
|
|
2801
|
+
const barWidth = 10;
|
|
2802
|
+
for (let i = 0;i < data.scoreProgression.length; i++) {
|
|
2803
|
+
const s = data.scoreProgression[i] ?? 0;
|
|
2804
|
+
const filled = Math.round(s * barWidth);
|
|
2805
|
+
const bar = "█".repeat(filled) + "░".repeat(barWidth - filled);
|
|
2806
|
+
console.log(line(chalk6.dim(` #${String(i + 1)}: ${bar} ${s.toFixed(2)}`)));
|
|
2711
2807
|
}
|
|
2712
2808
|
}
|
|
2713
2809
|
console.log(separator);
|
|
2714
|
-
console.log(line(`Dashboard: ${chalk6.underline(optimizerLink(job.
|
|
2810
|
+
console.log(line(`Dashboard: ${chalk6.underline(optimizerLink(job.id))}`));
|
|
2715
2811
|
console.log(bottomBorder);
|
|
2716
2812
|
console.log("");
|
|
2717
2813
|
}
|
|
@@ -2747,7 +2843,7 @@ function updateMutationContext(updater) {
|
|
|
2747
2843
|
} catch {}
|
|
2748
2844
|
}
|
|
2749
2845
|
var PREREQUISITES_TEXT = `
|
|
2750
|
-
${chalk7.
|
|
2846
|
+
${chalk7.red("Prerequisites (required):")}
|
|
2751
2847
|
1. Evaluation criteria defined ${chalk7.dim("(via dashboard or evaluation create)")}
|
|
2752
2848
|
2. Dataset uploaded ${chalk7.dim("mutagent prompts dataset list <prompt-id>")}
|
|
2753
2849
|
${chalk7.dim("Note: LLM provider config is only required when the server uses external providers (USE_EXT_PROVIDERS=true)")}`;
|
|
@@ -2780,18 +2876,6 @@ function isSchemaEmpty(schema) {
|
|
|
2780
2876
|
return true;
|
|
2781
2877
|
return false;
|
|
2782
2878
|
}
|
|
2783
|
-
function warnMissingSchemas(data, output) {
|
|
2784
|
-
if (isSchemaEmpty(data.inputSchema)) {
|
|
2785
|
-
output.warn("No inputSchema provided. Optimization requires inputSchema with defined variables.");
|
|
2786
|
-
} else if (!isValidJsonSchema(data.inputSchema)) {
|
|
2787
|
-
output.warn(formatSchemaWarning("inputSchema"));
|
|
2788
|
-
}
|
|
2789
|
-
if (isSchemaEmpty(data.outputSchema)) {
|
|
2790
|
-
output.warn("No outputSchema provided. This may limit optimization effectiveness.");
|
|
2791
|
-
} else if (!isValidJsonSchema(data.outputSchema)) {
|
|
2792
|
-
output.warn(formatSchemaWarning("outputSchema"));
|
|
2793
|
-
}
|
|
2794
|
-
}
|
|
2795
2879
|
async function collectSchemaInteractively() {
|
|
2796
2880
|
const inquirer3 = (await import("inquirer")).default;
|
|
2797
2881
|
const variables = [];
|
|
@@ -2820,13 +2904,18 @@ async function collectSchemaInteractively() {
|
|
|
2820
2904
|
{
|
|
2821
2905
|
type: "input",
|
|
2822
2906
|
name: "description",
|
|
2823
|
-
message: "Description (
|
|
2907
|
+
message: "Description (required):",
|
|
2908
|
+
validate: (input) => {
|
|
2909
|
+
if (!input.trim())
|
|
2910
|
+
return "Variable description is required";
|
|
2911
|
+
return true;
|
|
2912
|
+
}
|
|
2824
2913
|
}
|
|
2825
2914
|
]);
|
|
2826
2915
|
variables.push({
|
|
2827
2916
|
name: answers.name.trim(),
|
|
2828
2917
|
type: answers.type,
|
|
2829
|
-
description: answers.description.trim()
|
|
2918
|
+
description: answers.description.trim()
|
|
2830
2919
|
});
|
|
2831
2920
|
const continueAnswer = await inquirer3.prompt([{
|
|
2832
2921
|
type: "confirm",
|
|
@@ -2938,7 +3027,7 @@ Examples:
|
|
|
2938
3027
|
Subcommands:
|
|
2939
3028
|
list, get, create, update, delete
|
|
2940
3029
|
dataset list|add|remove
|
|
2941
|
-
evaluation list|create|results
|
|
3030
|
+
evaluation list|create|delete|results
|
|
2942
3031
|
optimize start|status|results
|
|
2943
3032
|
`);
|
|
2944
3033
|
prompts.command("list").description("List all prompts").option("-l, --limit <n>", "Limit results", "50").addHelpText("after", `
|
|
@@ -2963,14 +3052,22 @@ ${chalk7.dim("Tip: Use --json for machine-readable output (AI agents, CI pipelin
|
|
|
2963
3052
|
}));
|
|
2964
3053
|
output.output(withLinks);
|
|
2965
3054
|
} else {
|
|
2966
|
-
const
|
|
2967
|
-
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
}
|
|
2973
|
-
|
|
3055
|
+
const wsId = client.getCurrentWorkspaceId();
|
|
3056
|
+
if (wsId) {
|
|
3057
|
+
output.info(`Workspace: ${wsId}`);
|
|
3058
|
+
}
|
|
3059
|
+
if (limited.length === 0) {
|
|
3060
|
+
output.info("No prompts found in current workspace. Run 'mutagent config list' to check workspace context.");
|
|
3061
|
+
} else {
|
|
3062
|
+
const formatted = limited.map((p) => ({
|
|
3063
|
+
id: p.id,
|
|
3064
|
+
name: p.name,
|
|
3065
|
+
version: p.version,
|
|
3066
|
+
updated: p.updatedAt ? new Date(p.updatedAt).toLocaleDateString() : "N/A",
|
|
3067
|
+
url: promptLink(p.id)
|
|
3068
|
+
}));
|
|
3069
|
+
output.output(formatted);
|
|
3070
|
+
}
|
|
2974
3071
|
}
|
|
2975
3072
|
} catch (error) {
|
|
2976
3073
|
handleError(error, isJson);
|
|
@@ -3008,20 +3105,24 @@ ${chalk7.dim("Tip: Combine --with-datasets and --with-evals to fetch all nested
|
|
|
3008
3105
|
});
|
|
3009
3106
|
prompts.command("create").description("Create a new prompt").option("-d, --data <json>", "Prompt as JSON string (recommended — curl-style inline)").option("-f, --file <path>", "Create from JSON file").option("--raw-file <path>", "Create from plain text file (used as rawPrompt)").option("-n, --name <name>", "Prompt name").option("-c, --content <content>", "Prompt content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).option("--output-schema <json>", "Output schema as JSON string (required for optimization)").addHelpText("after", `
|
|
3010
3107
|
Examples:
|
|
3011
|
-
${chalk7.dim("$")} mutagent prompts create
|
|
3012
|
-
${chalk7.dim("$")} mutagent prompts create --name "
|
|
3013
|
-
${chalk7.dim("$")} mutagent prompts create
|
|
3014
|
-
${chalk7.dim("$")} mutagent prompts create --
|
|
3108
|
+
${chalk7.dim("$")} mutagent prompts create --name "my-prompt" --system "You are helpful" --human "{{input}}" --output-schema '{"type":"object","properties":{"result":{"type":"string","description":"The result"}}}'
|
|
3109
|
+
${chalk7.dim("$")} mutagent prompts create --name "raw-prompt" --raw "Summarize: {{text}}" --output-schema '{"type":"object","properties":{"summary":{"type":"string","description":"Summary"}}}'
|
|
3110
|
+
${chalk7.dim("$")} mutagent prompts create -d '{"name":"summarizer","systemPrompt":"Summarize","humanPrompt":"{{text}}","outputSchema":{"type":"object","properties":{"summary":{"type":"string","description":"Summary"}}}}'
|
|
3111
|
+
${chalk7.dim("$")} mutagent prompts create --file prompt.json ${chalk7.dim("# full prompt object as JSON file")}
|
|
3015
3112
|
|
|
3016
3113
|
Prompt Input Methods (pick one, priority order):
|
|
3017
|
-
|
|
3018
|
-
--file Load from JSON file (full prompt object)
|
|
3019
|
-
--raw-file Load plain text file as raw prompt
|
|
3020
|
-
--system/--human Structured system + user message pair
|
|
3114
|
+
--system/--human Structured system + user message pair ${chalk7.green("(recommended)")}
|
|
3021
3115
|
--raw Single raw prompt text with {{variables}}
|
|
3116
|
+
-d, --data Inline JSON object (CI/scripts/agents)
|
|
3022
3117
|
--messages Full messages array as JSON
|
|
3118
|
+
--raw-file Load plain text file as raw prompt
|
|
3119
|
+
--file Load from JSON file (full prompt object)
|
|
3120
|
+
|
|
3121
|
+
Expected JSON (--data):
|
|
3122
|
+
${chalk7.dim(`'{"name":"my-prompt","systemPrompt":"You are...","humanPrompt":"{{input}}","outputSchema":{"type":"object","properties":{"result":{"type":"string","description":"The result"}}},"inputSchema":{"type":"object","properties":{"input":{"type":"string","description":"User input"}}}}'`)}
|
|
3023
3123
|
|
|
3024
|
-
${chalk7.
|
|
3124
|
+
${chalk7.yellow("Note: Prefer --system/--human or --data over --file to avoid stale files living in your repo.")}
|
|
3125
|
+
${chalk7.red("outputSchema is required.")} ${chalk7.dim("--data and --file are mutually exclusive.")}
|
|
3025
3126
|
`).action(async (options) => {
|
|
3026
3127
|
const isJson = getJsonFlag(prompts);
|
|
3027
3128
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3120,17 +3221,32 @@ ${chalk7.dim("Note: --data and --file are mutually exclusive. outputSchema is re
|
|
|
3120
3221
|
if (schema) {
|
|
3121
3222
|
data.inputSchema = schema;
|
|
3122
3223
|
}
|
|
3123
|
-
} else {
|
|
3124
|
-
output.warn("No inputSchema provided. Optimization requires inputSchema with defined variables.");
|
|
3125
3224
|
}
|
|
3126
3225
|
}
|
|
3127
|
-
|
|
3226
|
+
if (isSchemaEmpty(data.inputSchema)) {
|
|
3227
|
+
throw new MutagentError("VALIDATION_ERROR", "inputSchema is required. Optimization cannot run without defined input variables.", `Provide inputSchema via --data, --file, or interactive mode. Example:
|
|
3228
|
+
--data '{"inputSchema":{"type":"object","properties":{"query":{"type":"string","description":"User query"}}}}'
|
|
3229
|
+
Or use interactive mode (TTY) to define variables step by step.`);
|
|
3230
|
+
} else if (!isValidJsonSchema(data.inputSchema)) {
|
|
3231
|
+
throw new MutagentError("VALIDATION_ERROR", "inputSchema is not a valid JSON Schema object.", formatSchemaWarning("inputSchema"));
|
|
3232
|
+
} else {
|
|
3233
|
+
const missingDescs = validateSchemaDescriptions(data.inputSchema);
|
|
3234
|
+
if (missingDescs.length > 0) {
|
|
3235
|
+
throw new MutagentError("VALIDATION_ERROR", `inputSchema properties missing descriptions: ${missingDescs.join(", ")}`, `Add a 'description' field to each property in your inputSchema. Example: { "properties": { "field": { "type": "string", "description": "What this field contains" } } }`);
|
|
3236
|
+
}
|
|
3237
|
+
}
|
|
3238
|
+
if (isSchemaEmpty(data.outputSchema)) {
|
|
3239
|
+
output.warn("No outputSchema provided. This may limit optimization effectiveness.");
|
|
3240
|
+
} else if (!isValidJsonSchema(data.outputSchema)) {
|
|
3241
|
+
output.warn(formatSchemaWarning("outputSchema"));
|
|
3242
|
+
}
|
|
3128
3243
|
const client = getSDKClient();
|
|
3129
3244
|
const prompt = await client.createPrompt(data);
|
|
3130
3245
|
if (isJson) {
|
|
3131
3246
|
output.output({ ...prompt, _links: promptLinks(prompt.id) });
|
|
3132
3247
|
} else {
|
|
3133
3248
|
output.success(`Created prompt: ${prompt.name} (id: ${String(prompt.id)})`);
|
|
3249
|
+
output.info(`Workspace: ${client.getCurrentWorkspaceId() ?? "auto-inferred from API key"}`);
|
|
3134
3250
|
const hints = formatCreationHints({
|
|
3135
3251
|
resourceType: "Prompt",
|
|
3136
3252
|
id: prompt.id,
|
|
@@ -3152,15 +3268,17 @@ ${chalk7.dim("Note: --data and --file are mutually exclusive. outputSchema is re
|
|
|
3152
3268
|
handleError(error, isJson);
|
|
3153
3269
|
}
|
|
3154
3270
|
});
|
|
3155
|
-
prompts.command("update").description("Update a prompt").argument("<id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Update fields as JSON string (recommended — curl-style inline)").option("-f, --file <path>", "Update from JSON file").option("--raw-file <path>", "Update from plain text file (used as rawPrompt)").option("-n, --name <name>", "New name").option("-c, --content <content>", "New content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).addHelpText("after", `
|
|
3271
|
+
prompts.command("update").description("Update a prompt").argument("<id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Update fields as JSON string (recommended — curl-style inline)").option("-f, --file <path>", "Update from JSON file").option("--raw-file <path>", "Update from plain text file (used as rawPrompt)").option("-n, --name <name>", "New name").option("-c, --content <content>", "New content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).option("--input-schema <json>", "Input schema as JSON string").option("--input-schema-file <path>", "Input schema from JSON file").option("--output-schema <json>", "Output schema as JSON string").option("--output-schema-file <path>", "Output schema from JSON file").addHelpText("after", `
|
|
3156
3272
|
Examples:
|
|
3157
|
-
${chalk7.dim("$")} mutagent prompts update <id>
|
|
3273
|
+
${chalk7.dim("$")} mutagent prompts update <id> --system "Updated system prompt" --human "{{input}}"
|
|
3158
3274
|
${chalk7.dim("$")} mutagent prompts update <id> --name "new-name"
|
|
3159
|
-
${chalk7.dim("$")} mutagent prompts update <id> --
|
|
3160
|
-
${chalk7.dim("$")} mutagent prompts update <id>
|
|
3161
|
-
${chalk7.dim("$")} mutagent prompts update <id> --
|
|
3275
|
+
${chalk7.dim("$")} mutagent prompts update <id> --raw "Summarize: {{text}}"
|
|
3276
|
+
${chalk7.dim("$")} mutagent prompts update <id> -d '{"name":"new-name","systemPrompt":"Updated prompt"}'
|
|
3277
|
+
${chalk7.dim("$")} mutagent prompts update <id> --input-schema '{"type":"object","properties":{"text":{"type":"string","description":"Input text"}}}'
|
|
3278
|
+
${chalk7.dim("$")} mutagent prompts update <id> --file updated-prompt.json ${chalk7.dim("# full prompt object")}
|
|
3162
3279
|
|
|
3163
|
-
${chalk7.
|
|
3280
|
+
${chalk7.yellow("Note: Prefer --system/--human or --data over --file to avoid stale files living in your repo.")}
|
|
3281
|
+
${chalk7.dim("--data and --file are mutually exclusive. CLI flags (--name) override --data fields.")}
|
|
3164
3282
|
`).action(async (id, options) => {
|
|
3165
3283
|
const isJson = getJsonFlag(prompts);
|
|
3166
3284
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3214,8 +3332,52 @@ ${chalk7.dim("Note: --data and --file are mutually exclusive. CLI flags (--name)
|
|
|
3214
3332
|
data.rawPrompt = options.content;
|
|
3215
3333
|
}
|
|
3216
3334
|
}
|
|
3335
|
+
if (options.inputSchema && options.inputSchemaFile) {
|
|
3336
|
+
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --input-schema and --input-schema-file together", "Use --input-schema for inline JSON or --input-schema-file for file-based input, not both");
|
|
3337
|
+
}
|
|
3338
|
+
if (options.inputSchema) {
|
|
3339
|
+
try {
|
|
3340
|
+
data.inputSchema = JSON.parse(options.inputSchema);
|
|
3341
|
+
} catch {
|
|
3342
|
+
throw new MutagentError("INVALID_JSON", "Invalid JSON in --input-schema flag", `Provide a valid JSON Schema, e.g., '{"type":"object","properties":{"text":{"type":"string","description":"Input text"}}}'`);
|
|
3343
|
+
}
|
|
3344
|
+
} else if (options.inputSchemaFile) {
|
|
3345
|
+
if (!existsSync4(options.inputSchemaFile)) {
|
|
3346
|
+
throw new MutagentError("FILE_NOT_FOUND", `File not found: ${options.inputSchemaFile}`, "Check the file path and try again");
|
|
3347
|
+
}
|
|
3348
|
+
try {
|
|
3349
|
+
data.inputSchema = JSON.parse(readFileSync4(options.inputSchemaFile, "utf-8"));
|
|
3350
|
+
} catch {
|
|
3351
|
+
throw new MutagentError("INVALID_JSON", `Failed to parse JSON from ${options.inputSchemaFile}`, "Ensure the file contains valid JSON Schema");
|
|
3352
|
+
}
|
|
3353
|
+
}
|
|
3354
|
+
if (options.outputSchema && options.outputSchemaFile) {
|
|
3355
|
+
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --output-schema and --output-schema-file together", "Use --output-schema for inline JSON or --output-schema-file for file-based input, not both");
|
|
3356
|
+
}
|
|
3357
|
+
if (options.outputSchema) {
|
|
3358
|
+
try {
|
|
3359
|
+
data.outputSchema = JSON.parse(options.outputSchema);
|
|
3360
|
+
} catch {
|
|
3361
|
+
throw new MutagentError("INVALID_JSON", "Invalid JSON in --output-schema flag", `Provide a valid JSON Schema, e.g., '{"type":"object","properties":{"result":{"type":"string"}}}'`);
|
|
3362
|
+
}
|
|
3363
|
+
} else if (options.outputSchemaFile) {
|
|
3364
|
+
if (!existsSync4(options.outputSchemaFile)) {
|
|
3365
|
+
throw new MutagentError("FILE_NOT_FOUND", `File not found: ${options.outputSchemaFile}`, "Check the file path and try again");
|
|
3366
|
+
}
|
|
3367
|
+
try {
|
|
3368
|
+
data.outputSchema = JSON.parse(readFileSync4(options.outputSchemaFile, "utf-8"));
|
|
3369
|
+
} catch {
|
|
3370
|
+
throw new MutagentError("INVALID_JSON", `Failed to parse JSON from ${options.outputSchemaFile}`, "Ensure the file contains valid JSON Schema");
|
|
3371
|
+
}
|
|
3372
|
+
}
|
|
3373
|
+
if (data.inputSchema && isValidJsonSchema(data.inputSchema)) {
|
|
3374
|
+
const missingDescs = validateSchemaDescriptions(data.inputSchema);
|
|
3375
|
+
if (missingDescs.length > 0) {
|
|
3376
|
+
throw new MutagentError("VALIDATION_ERROR", `inputSchema properties missing descriptions: ${missingDescs.join(", ")}`, `Add a 'description' field to each property in your inputSchema. Example: { "properties": { "field": { "type": "string", "description": "What this field contains" } } }`);
|
|
3377
|
+
}
|
|
3378
|
+
}
|
|
3217
3379
|
if (Object.keys(data).length === 0) {
|
|
3218
|
-
throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --data, --file, --raw-file, --name, --raw, --system/--human, or --
|
|
3380
|
+
throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --data, --file, --raw-file, --name, --raw, --system/--human, --messages, --input-schema, or --output-schema");
|
|
3219
3381
|
}
|
|
3220
3382
|
const promptContent = data.rawPrompt ?? data.systemPrompt ?? data.humanPrompt ?? "";
|
|
3221
3383
|
if (promptContent && !isJson) {
|
|
@@ -3277,12 +3439,15 @@ ${chalk7.dim("Tip: Use --force to skip confirmation (required for non-interactiv
|
|
|
3277
3439
|
handleError(error, isJson);
|
|
3278
3440
|
}
|
|
3279
3441
|
});
|
|
3280
|
-
const dataset =
|
|
3442
|
+
const dataset = new Command3("dataset").description("Manage datasets for prompts").addHelpText("after", `
|
|
3281
3443
|
Examples:
|
|
3282
3444
|
${chalk7.dim("$")} mutagent prompts dataset list <prompt-id>
|
|
3283
3445
|
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.jsonl
|
|
3284
3446
|
${chalk7.dim("$")} mutagent prompts dataset remove <prompt-id> <dataset-id>
|
|
3285
|
-
`)
|
|
3447
|
+
`).action(() => {
|
|
3448
|
+
dataset.help();
|
|
3449
|
+
});
|
|
3450
|
+
prompts.addCommand(dataset);
|
|
3286
3451
|
dataset.command("list").description("List datasets for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").addHelpText("after", `
|
|
3287
3452
|
Examples:
|
|
3288
3453
|
${chalk7.dim("$")} mutagent prompts dataset list <prompt-id>
|
|
@@ -3312,10 +3477,9 @@ Examples:
|
|
|
3312
3477
|
});
|
|
3313
3478
|
dataset.command("add").description("Add dataset to a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").option("-f, --file <path>", "Dataset file (JSON array, JSONL, or CSV)").option("-d, --data <json>", "Inline JSON array of dataset items").option("-n, --name <name>", "Dataset name (default: timestamped)").addHelpText("after", `
|
|
3314
3479
|
Examples:
|
|
3315
|
-
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.json
|
|
3316
|
-
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.jsonl --name "My Dataset"
|
|
3317
|
-
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.csv
|
|
3318
3480
|
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> -d '[{"input":{"text":"hello"},"expectedOutput":{"result":"world"}}]'
|
|
3481
|
+
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> -d '[{"input":{"text":"hello"},"expectedOutput":{"result":"world"}}]' --name "My Dataset"
|
|
3482
|
+
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.jsonl ${chalk7.dim("# also supports .json, .csv")}
|
|
3319
3483
|
|
|
3320
3484
|
Supported file formats:
|
|
3321
3485
|
${chalk7.dim(".json")} JSON array of objects: [{"input": {...}, "expectedOutput": {...}}, ...]
|
|
@@ -3326,7 +3490,12 @@ Inline data format (-d):
|
|
|
3326
3490
|
JSON array of objects, e.g.:
|
|
3327
3491
|
${chalk7.dim('[{"input": {"text": "hello"}, "expectedOutput": {"result": "world"}}]')}
|
|
3328
3492
|
|
|
3329
|
-
|
|
3493
|
+
Expected item format:
|
|
3494
|
+
${chalk7.dim('{"input": {"<field>": "<value>"}, "expectedOutput": {"<field>": "<value>"}}')}
|
|
3495
|
+
|
|
3496
|
+
${chalk7.red("Required: --data or --file must be provided.")}
|
|
3497
|
+
${chalk7.yellow("Note: Prefer -d/--data for inline JSON over --file to avoid stale files living in your repo.")}
|
|
3498
|
+
${chalk7.dim("--file and -d are mutually exclusive.")}
|
|
3330
3499
|
`).action(async (promptId, options) => {
|
|
3331
3500
|
const isJson = getJsonFlag(prompts);
|
|
3332
3501
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3386,6 +3555,9 @@ ${chalk7.dim("Note: --file and -d are mutually exclusive. Provide one or the oth
|
|
|
3386
3555
|
if (datasetResult.itemCount !== undefined && datasetResult.itemCount > 0) {
|
|
3387
3556
|
output.info(`Items uploaded: ${String(datasetResult.itemCount)}`);
|
|
3388
3557
|
}
|
|
3558
|
+
if (datasetResult.missingExpectedOutputCount !== undefined && datasetResult.missingExpectedOutputCount > 0) {
|
|
3559
|
+
output.warn(`${String(datasetResult.missingExpectedOutputCount)} of ${String(datasetResult.itemCount ?? 0)} items have no expectedOutput. Optimization may fail without expected outputs.`);
|
|
3560
|
+
}
|
|
3389
3561
|
const hints = formatCreationHints({
|
|
3390
3562
|
resourceType: "Dataset",
|
|
3391
3563
|
id: datasetResult.id,
|
|
@@ -3419,12 +3591,16 @@ Examples:
|
|
|
3419
3591
|
handleError(error, isJson);
|
|
3420
3592
|
}
|
|
3421
3593
|
});
|
|
3422
|
-
const evaluation =
|
|
3594
|
+
const evaluation = new Command3("evaluation").description("Manage evaluations for prompts").addHelpText("after", `
|
|
3423
3595
|
Examples:
|
|
3424
3596
|
${chalk7.dim("$")} mutagent prompts evaluation list <prompt-id>
|
|
3425
3597
|
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "My Eval"
|
|
3598
|
+
${chalk7.dim("$")} mutagent prompts evaluation delete <evaluation-id>
|
|
3426
3599
|
${chalk7.dim("$")} mutagent prompts evaluation results <run-id>
|
|
3427
|
-
`)
|
|
3600
|
+
`).action(() => {
|
|
3601
|
+
evaluation.help();
|
|
3602
|
+
});
|
|
3603
|
+
prompts.addCommand(evaluation);
|
|
3428
3604
|
evaluation.command("list").description("List evaluations for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").addHelpText("after", `
|
|
3429
3605
|
Examples:
|
|
3430
3606
|
${chalk7.dim("$")} mutagent prompts evaluation list <prompt-id>
|
|
@@ -3454,17 +3630,19 @@ Examples:
|
|
|
3454
3630
|
});
|
|
3455
3631
|
evaluation.command("create").description("Create an evaluation configuration for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Evaluation as JSON string (recommended — curl-style inline)").option("-n, --name <name>", "Evaluation name (required unless --guided)").option("-f, --file <path>", "Load evaluation criteria from JSON file").option("--description <text>", "Evaluation description").option("--dataset <id>", "Dataset ID to associate (from: mutagent prompts dataset list)").option("--guided", "Interactive guided mode — build criteria step by step").addHelpText("after", `
|
|
3456
3632
|
Examples:
|
|
3457
|
-
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --guided
|
|
3458
|
-
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --
|
|
3459
|
-
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Accuracy Check"
|
|
3633
|
+
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --guided ${chalk7.dim("# recommended: interactive walkthrough")}
|
|
3634
|
+
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Accuracy" -d '{"evalConfig":{"criteria":[{"name":"Accuracy","description":"Score 1.0 if output matches expected","evaluationParameter":"result"}]}}'
|
|
3460
3635
|
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Full Eval" --file criteria.json --dataset <dataset-id>
|
|
3461
|
-
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "QA" --description "Quality assurance eval" --json
|
|
3462
3636
|
|
|
3463
|
-
|
|
3464
|
-
${chalk7.dim('{
|
|
3637
|
+
Expected JSON (--data):
|
|
3638
|
+
${chalk7.dim('{"evalConfig":{"criteria":[{"name":"<name>","description":"<scoring rubric>","evaluationParameter":"<output field to score>"}]}}')}
|
|
3639
|
+
|
|
3640
|
+
${chalk7.red("Each criterion MUST have: name, description, evaluationParameter")}
|
|
3641
|
+
evaluationParameter maps to output schema fields (e.g., "result", "classification")
|
|
3465
3642
|
|
|
3466
|
-
${chalk7.
|
|
3467
|
-
${chalk7.
|
|
3643
|
+
${chalk7.yellow("Note: Prefer --guided or --data over --file to avoid stale files living in your repo.")}
|
|
3644
|
+
${chalk7.red("Required: --name (unless --guided). Criteria must include evaluationParameter.")}
|
|
3645
|
+
${chalk7.dim("--data and --file are mutually exclusive. CLI flags (--name, --description) override --data fields.")}
|
|
3468
3646
|
${chalk7.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent prompts dataset list <prompt-id>")}
|
|
3469
3647
|
`).action(async (promptId, options) => {
|
|
3470
3648
|
const isJson = getJsonFlag(prompts);
|
|
@@ -3549,6 +3727,79 @@ ${chalk7.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent
|
|
|
3549
3727
|
evalData.name = options.name;
|
|
3550
3728
|
if (options.description)
|
|
3551
3729
|
evalData.description = options.description;
|
|
3730
|
+
const criteria = evalData.evalConfig?.criteria;
|
|
3731
|
+
if (!criteria || !Array.isArray(criteria) || criteria.length === 0) {
|
|
3732
|
+
let fieldsHint = "";
|
|
3733
|
+
try {
|
|
3734
|
+
const client2 = getSDKClient();
|
|
3735
|
+
const prompt = await client2.getPrompt(promptId);
|
|
3736
|
+
if (prompt.outputSchema && typeof prompt.outputSchema === "object") {
|
|
3737
|
+
const props = prompt.outputSchema.properties;
|
|
3738
|
+
if (props && typeof props === "object") {
|
|
3739
|
+
const fields = Object.keys(props);
|
|
3740
|
+
if (fields.length > 0) {
|
|
3741
|
+
fieldsHint = `
|
|
3742
|
+
Detected output fields from prompt schema: ${fields.join(", ")}
|
|
3743
|
+
`;
|
|
3744
|
+
}
|
|
3745
|
+
}
|
|
3746
|
+
}
|
|
3747
|
+
} catch {}
|
|
3748
|
+
throw new MutagentError("VALIDATION_ERROR", "Evaluation criteria are required. Provide criteria via --data, --file, or use --guided mode.", `Each criterion needs: name, description, evaluationParameter (the output field to score).
|
|
3749
|
+
` + fieldsHint + `
|
|
3750
|
+
Example JSON (--data flag):
|
|
3751
|
+
--data '{"evalConfig":{"criteria":[{"name":"Accuracy","description":"Score 1.0 if output matches expected, 0.0 otherwise","evaluationParameter":"classification"}]}}'
|
|
3752
|
+
|
|
3753
|
+
Or use --guided for interactive creation: mutagent prompts evaluation create <id> --guided`);
|
|
3754
|
+
}
|
|
3755
|
+
for (const c of criteria) {
|
|
3756
|
+
const hasDescription = c.description ?? c.scoringRubric;
|
|
3757
|
+
const hasEvalParam = c.evaluationParameter ?? c.targetField;
|
|
3758
|
+
if (!c.name || !hasDescription || !hasEvalParam) {
|
|
3759
|
+
const cName = typeof c.name === "string" ? c.name : "unnamed";
|
|
3760
|
+
throw new MutagentError("VALIDATION_ERROR", `Criterion "${cName}" is missing required fields. Need: name, description (or scoringRubric), evaluationParameter (or targetField).`, 'evaluationParameter maps to the output field to evaluate (e.g., "classification", "riskLevel"). Use --guided for interactive creation.');
|
|
3761
|
+
}
|
|
3762
|
+
if (c.scoringRubric && !c.description) {
|
|
3763
|
+
c.description = c.scoringRubric;
|
|
3764
|
+
delete c.scoringRubric;
|
|
3765
|
+
}
|
|
3766
|
+
if (c.targetField && !c.evaluationParameter) {
|
|
3767
|
+
const tf = c.targetField;
|
|
3768
|
+
c.evaluationParameter = tf.startsWith("output.") ? tf.slice("output.".length) : tf.startsWith("input.") ? tf.slice("input.".length) : tf;
|
|
3769
|
+
delete c.targetField;
|
|
3770
|
+
}
|
|
3771
|
+
}
|
|
3772
|
+
const seenParams = new Set;
|
|
3773
|
+
const duplicateParams = [];
|
|
3774
|
+
for (const c of criteria) {
|
|
3775
|
+
const param = c.evaluationParameter;
|
|
3776
|
+
if (param) {
|
|
3777
|
+
if (seenParams.has(param)) {
|
|
3778
|
+
duplicateParams.push(param);
|
|
3779
|
+
}
|
|
3780
|
+
seenParams.add(param);
|
|
3781
|
+
}
|
|
3782
|
+
}
|
|
3783
|
+
if (duplicateParams.length > 0) {
|
|
3784
|
+
let availableFields = [];
|
|
3785
|
+
try {
|
|
3786
|
+
const client2 = getSDKClient();
|
|
3787
|
+
const prompt = await client2.getPrompt(promptId);
|
|
3788
|
+
if (prompt.outputSchema && typeof prompt.outputSchema === "object") {
|
|
3789
|
+
const props = prompt.outputSchema.properties;
|
|
3790
|
+
if (props && typeof props === "object") {
|
|
3791
|
+
availableFields = Object.keys(props);
|
|
3792
|
+
}
|
|
3793
|
+
}
|
|
3794
|
+
} catch {}
|
|
3795
|
+
const uniqueDupes = [...new Set(duplicateParams)];
|
|
3796
|
+
const fieldsHint = availableFields.length > 0 ? `
|
|
3797
|
+
Available output fields: ${availableFields.join(", ")}` : "";
|
|
3798
|
+
throw new MutagentError("VALIDATION_ERROR", `Duplicate evaluationParameter: "${uniqueDupes.join('", "')}". Each criterion must target a unique output field.`, "Each criterion scores a different output field. Fix by changing the evaluationParameter to a unique field." + fieldsHint + `
|
|
3799
|
+
|
|
3800
|
+
Example:
|
|
3801
|
+
--data '{"evalConfig":{"criteria":[{"name":"Accuracy","description":"...","evaluationParameter":"classification"},{"name":"Confidence","description":"...","evaluationParameter":"confidence"}]}}'`);
|
|
3802
|
+
}
|
|
3552
3803
|
if (options.dataset) {
|
|
3553
3804
|
evalData.datasetId = parseInt(options.dataset, 10);
|
|
3554
3805
|
if (isNaN(evalData.datasetId)) {
|
|
@@ -3596,15 +3847,37 @@ Examples:
|
|
|
3596
3847
|
handleError(error, isJson);
|
|
3597
3848
|
}
|
|
3598
3849
|
});
|
|
3599
|
-
|
|
3850
|
+
evaluation.command("delete").description("Delete an evaluation").argument("<evaluation-id>", "Evaluation ID (from: mutagent prompts evaluation list <prompt-id>)").addHelpText("after", `
|
|
3851
|
+
Examples:
|
|
3852
|
+
${chalk7.dim("$")} mutagent prompts evaluation delete <evaluation-id>
|
|
3853
|
+
${chalk7.dim("$")} mutagent prompts evaluation delete <evaluation-id> --json
|
|
3854
|
+
`).action(async (evaluationId) => {
|
|
3855
|
+
const isJson = getJsonFlag(prompts);
|
|
3856
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
3857
|
+
try {
|
|
3858
|
+
const client = getSDKClient();
|
|
3859
|
+
await client.deleteEvaluation(evaluationId);
|
|
3860
|
+
if (isJson) {
|
|
3861
|
+
output.output({ success: true, message: "Evaluation deleted successfully", evaluationId });
|
|
3862
|
+
} else {
|
|
3863
|
+
output.success(`Evaluation ${evaluationId} deleted successfully`);
|
|
3864
|
+
}
|
|
3865
|
+
} catch (error) {
|
|
3866
|
+
handleError(error, isJson);
|
|
3867
|
+
}
|
|
3868
|
+
});
|
|
3869
|
+
const optimize = new Command3("optimize").description("Manage prompt optimization jobs").addHelpText("after", `
|
|
3600
3870
|
Examples:
|
|
3601
3871
|
${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id>
|
|
3602
3872
|
${chalk7.dim("$")} mutagent prompts optimize status <job-id>
|
|
3603
3873
|
${chalk7.dim("$")} mutagent prompts optimize results <job-id>
|
|
3604
3874
|
|
|
3605
3875
|
Workflow: start -> status (poll) -> results
|
|
3606
|
-
`)
|
|
3607
|
-
|
|
3876
|
+
`).action(() => {
|
|
3877
|
+
optimize.help();
|
|
3878
|
+
});
|
|
3879
|
+
prompts.addCommand(optimize);
|
|
3880
|
+
optimize.command("start").description("Start prompt optimization").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").requiredOption("-d, --dataset <id>", "Dataset ID for optimization (from: mutagent prompts dataset list <prompt-id>)").option("--max-iterations <n>", "Max optimization iterations (default: 1)").option("--target-score <n>", "Target accuracy 0-1 (default: 0.8)").option("--patience <n>", "Iterations without improvement before stopping").option("--model <model-id>", 'Target LLM model (e.g., "claude-sonnet-4-5-20250929")').option("--eval-model <model-id>", "Evaluation model (defaults to target model)").addHelpText("after", `
|
|
3608
3881
|
Examples:
|
|
3609
3882
|
${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id>
|
|
3610
3883
|
${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --max-iterations 5
|
|
@@ -3613,11 +3886,45 @@ Examples:
|
|
|
3613
3886
|
${PREREQUISITES_TEXT}
|
|
3614
3887
|
|
|
3615
3888
|
${chalk7.dim("Monitor progress with: mutagent prompts optimize status <job-id>")}
|
|
3889
|
+
|
|
3890
|
+
${chalk7.dim("AI Agent Note: After running commands, present CLI output to the user as a status report. Use --json for structured parsing.")}
|
|
3616
3891
|
`).action(async (promptId, options) => {
|
|
3617
3892
|
const isJson = getJsonFlag(prompts);
|
|
3618
3893
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
3619
3894
|
try {
|
|
3620
3895
|
const client = getSDKClient();
|
|
3896
|
+
if (!isJson) {
|
|
3897
|
+
output.info("Running pre-flight checks...");
|
|
3898
|
+
const prompt = await client.getPrompt(promptId);
|
|
3899
|
+
if (isSchemaEmpty(prompt.outputSchema)) {
|
|
3900
|
+
throw new MutagentError("MISSING_OUTPUT_SCHEMA", "Prompt is missing outputSchema — required for optimization", `Update your prompt: mutagent prompts update ${promptId} -d '{"outputSchema":{"type":"object","properties":{"result":{"type":"string"}}}}'`);
|
|
3901
|
+
}
|
|
3902
|
+
if (isSchemaEmpty(prompt.inputSchema)) {
|
|
3903
|
+
output.warn("Prompt has no inputSchema. Optimization works best with defined input variables.");
|
|
3904
|
+
}
|
|
3905
|
+
try {
|
|
3906
|
+
const evals = await client.listEvaluations(promptId);
|
|
3907
|
+
if (evals.length === 0) {
|
|
3908
|
+
throw new MutagentError("MISSING_EVALUATION", "No evaluations found for this prompt — required for optimization", `Create an evaluation: mutagent prompts evaluation create ${promptId} --guided`);
|
|
3909
|
+
}
|
|
3910
|
+
const validEvals = evals.filter((e) => {
|
|
3911
|
+
const config = e.evalConfig;
|
|
3912
|
+
const criteria = config?.criteria;
|
|
3913
|
+
if (!criteria || criteria.length === 0)
|
|
3914
|
+
return false;
|
|
3915
|
+
return criteria.some((c) => c.evaluationParameter ?? c.targetField);
|
|
3916
|
+
});
|
|
3917
|
+
if (validEvals.length === 0) {
|
|
3918
|
+
output.warn(`Evaluations exist but none have evaluationParameter set. Optimization works best with criteria targeting specific output fields. Re-create: mutagent prompts evaluation create ${promptId} --guided`);
|
|
3919
|
+
} else {
|
|
3920
|
+
output.info(`Found ${String(validEvals.length)} evaluation(s) with proper criteria.`);
|
|
3921
|
+
}
|
|
3922
|
+
} catch (e) {
|
|
3923
|
+
if (e instanceof MutagentError)
|
|
3924
|
+
throw e;
|
|
3925
|
+
output.warn("Could not validate evaluations. Optimization will use default evaluation.");
|
|
3926
|
+
}
|
|
3927
|
+
}
|
|
3621
3928
|
const job = await client.startOptimization(promptId, options.dataset, {
|
|
3622
3929
|
maxIterations: options.maxIterations ? parseInt(options.maxIterations, 10) : undefined,
|
|
3623
3930
|
targetScore: options.targetScore ? parseFloat(options.targetScore) : undefined,
|
|
@@ -3629,27 +3936,55 @@ ${chalk7.dim("Monitor progress with: mutagent prompts optimize status <job-id>")
|
|
|
3629
3936
|
output.output({
|
|
3630
3937
|
...job,
|
|
3631
3938
|
_links: {
|
|
3632
|
-
dashboard: optimizerLink(job.
|
|
3633
|
-
api: `/api/prompts/${promptId}/optimizations/${job.
|
|
3939
|
+
dashboard: optimizerLink(job.id),
|
|
3940
|
+
api: `/api/prompts/${promptId}/optimizations/${job.id}`
|
|
3634
3941
|
}
|
|
3635
3942
|
});
|
|
3636
3943
|
} else {
|
|
3637
|
-
output.success(`Started optimization job: ${job.
|
|
3944
|
+
output.success(`Started optimization job: ${job.id}`);
|
|
3638
3945
|
const hints = formatCreationHints({
|
|
3639
3946
|
resourceType: "Optimization",
|
|
3640
|
-
id: job.
|
|
3641
|
-
dashboardUrl: optimizerLink(job.
|
|
3642
|
-
apiPath: `/api/prompts/${promptId}/optimizations/${job.
|
|
3947
|
+
id: job.id,
|
|
3948
|
+
dashboardUrl: optimizerLink(job.id),
|
|
3949
|
+
apiPath: `/api/prompts/${promptId}/optimizations/${job.id}`
|
|
3643
3950
|
});
|
|
3644
3951
|
console.log(hints);
|
|
3645
3952
|
}
|
|
3646
3953
|
} catch (error) {
|
|
3647
3954
|
if (error instanceof ApiError) {
|
|
3648
3955
|
const messages = parseValidationErrors(error);
|
|
3956
|
+
const errorText = messages.join(" ");
|
|
3957
|
+
const suggestions = [];
|
|
3958
|
+
if (errorText.includes("Output Schema is not present") || errorText.includes("outputSchema")) {
|
|
3959
|
+
suggestions.push(`Update prompt: mutagent prompts update ${promptId} -d '{"outputSchema":{"type":"object","properties":{...}}}'`);
|
|
3960
|
+
}
|
|
3961
|
+
if (errorText.includes("Outputs are not valid") || errorText.includes("Parameters Mismatch")) {
|
|
3962
|
+
suggestions.push("Dataset items may not match the prompt schema. Check expectedOutput fields in your dataset.");
|
|
3963
|
+
}
|
|
3964
|
+
if (errorText.includes("Inputs are not valid")) {
|
|
3965
|
+
suggestions.push("Dataset item inputs don't match the prompt's inputSchema. Verify field names and types.");
|
|
3966
|
+
}
|
|
3967
|
+
if (errorText.includes("has no items") || errorText.includes("Dataset") && errorText.includes("not found")) {
|
|
3968
|
+
suggestions.push(`Upload data: mutagent prompts dataset add ${promptId} --file data.json`);
|
|
3969
|
+
}
|
|
3970
|
+
if (errorText.includes("Evaluations are not present")) {
|
|
3971
|
+
suggestions.push(`Create evaluation: mutagent prompts evaluation create ${promptId} --guided`);
|
|
3972
|
+
}
|
|
3973
|
+
if (errorText.includes("Trial") && errorText.includes("limit")) {
|
|
3974
|
+
suggestions.push("Trial optimization limit reached. Contact support to upgrade.");
|
|
3975
|
+
}
|
|
3649
3976
|
if (!isJson) {
|
|
3650
|
-
console.error(chalk7.red(
|
|
3977
|
+
console.error(chalk7.red(`
|
|
3978
|
+
Optimization failed:`));
|
|
3651
3979
|
for (const msg of messages) {
|
|
3652
|
-
console.error(chalk7.red(`
|
|
3980
|
+
console.error(chalk7.red(` ${msg}`));
|
|
3981
|
+
}
|
|
3982
|
+
if (suggestions.length > 0) {
|
|
3983
|
+
console.error(chalk7.yellow(`
|
|
3984
|
+
Suggested fixes:`));
|
|
3985
|
+
for (const s of suggestions) {
|
|
3986
|
+
console.error(chalk7.yellow(` → ${s}`));
|
|
3987
|
+
}
|
|
3653
3988
|
}
|
|
3654
3989
|
console.error("");
|
|
3655
3990
|
console.error(PREREQUISITES_TEXT);
|
|
@@ -3894,235 +4229,179 @@ ${chalk8.dim("Exports to stdout by default. Use --output to save to a file.")}
|
|
|
3894
4229
|
init_config();
|
|
3895
4230
|
import { Command as Command5 } from "commander";
|
|
3896
4231
|
import chalk9 from "chalk";
|
|
3897
|
-
import { writeFileSync as writeFileSync3, existsSync as
|
|
4232
|
+
import { writeFileSync as writeFileSync3, existsSync as existsSync9 } from "fs";
|
|
3898
4233
|
import { execSync } from "child_process";
|
|
3899
4234
|
init_errors();
|
|
3900
4235
|
|
|
3901
|
-
// src/lib/integrations/
|
|
4236
|
+
// src/lib/integrations/langchain.ts
|
|
3902
4237
|
import { readFileSync as readFileSync5, existsSync as existsSync5 } from "fs";
|
|
3903
|
-
|
|
3904
|
-
|
|
3905
|
-
|
|
3906
|
-
✓ MUTAGENT_ENDPOINT: ${config.endpoint}
|
|
3907
|
-
✓ API Connection: Verified`;
|
|
3908
|
-
}
|
|
3909
|
-
var mastraIntegration = {
|
|
3910
|
-
name: "mastra",
|
|
3911
|
-
description: "Mastra AI framework",
|
|
4238
|
+
var langchainIntegration = {
|
|
4239
|
+
name: "langchain",
|
|
4240
|
+
description: "LangChain framework",
|
|
3912
4241
|
async detect() {
|
|
3913
|
-
|
|
3914
|
-
const found = [];
|
|
3915
|
-
for (const file of files) {
|
|
3916
|
-
if (existsSync5(file)) {
|
|
3917
|
-
found.push(file);
|
|
3918
|
-
}
|
|
3919
|
-
}
|
|
3920
|
-
let hasMastraDep = false;
|
|
4242
|
+
let hasLangchain = false;
|
|
3921
4243
|
if (existsSync5("package.json")) {
|
|
3922
4244
|
try {
|
|
3923
4245
|
const pkg = JSON.parse(readFileSync5("package.json", "utf-8"));
|
|
3924
4246
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
3925
|
-
|
|
4247
|
+
hasLangchain = "langchain" in deps || "@langchain/core" in deps;
|
|
3926
4248
|
} catch {}
|
|
3927
4249
|
}
|
|
3928
4250
|
return {
|
|
3929
|
-
detected:
|
|
3930
|
-
confidence:
|
|
3931
|
-
files:
|
|
4251
|
+
detected: hasLangchain,
|
|
4252
|
+
confidence: hasLangchain ? 0.9 : 0,
|
|
4253
|
+
files: hasLangchain ? ["package.json"] : []
|
|
3932
4254
|
};
|
|
3933
4255
|
},
|
|
3934
4256
|
async generate(config) {
|
|
3935
4257
|
return `---
|
|
3936
|
-
name: mutagent-
|
|
3937
|
-
description: Integrate MutagenT with
|
|
3938
|
-
framework:
|
|
4258
|
+
name: mutagent-langchain-integration
|
|
4259
|
+
description: Integrate MutagenT with LangChain for prompt optimization and trace observability
|
|
4260
|
+
framework: langchain
|
|
3939
4261
|
version: 1.0.0
|
|
3940
4262
|
---
|
|
3941
4263
|
|
|
3942
|
-
# MutagenT +
|
|
4264
|
+
# MutagenT + LangChain Integration
|
|
3943
4265
|
|
|
3944
4266
|
## Prerequisites Verification
|
|
3945
4267
|
|
|
3946
|
-
${
|
|
4268
|
+
- MUTAGENT_API_KEY: ${config.apiKey.slice(0, 8)}...${config.apiKey.slice(-4)}
|
|
4269
|
+
- MUTAGENT_ENDPOINT: ${config.endpoint}
|
|
4270
|
+
- API Connection: Verified
|
|
3947
4271
|
|
|
3948
4272
|
## Installation
|
|
3949
4273
|
|
|
3950
4274
|
\`\`\`bash
|
|
3951
|
-
|
|
4275
|
+
# The @mutagent/langchain package was installed automatically
|
|
4276
|
+
# If you need to install manually:
|
|
4277
|
+
bun add @mutagent/langchain
|
|
3952
4278
|
# or
|
|
3953
|
-
npm install @mutagent/
|
|
4279
|
+
npm install @mutagent/langchain
|
|
3954
4280
|
\`\`\`
|
|
3955
4281
|
|
|
3956
4282
|
## Configuration
|
|
3957
4283
|
|
|
3958
4284
|
### 1. Environment Variables
|
|
3959
4285
|
|
|
3960
|
-
Add to your \`.env\` file:
|
|
3961
|
-
|
|
3962
4286
|
\`\`\`env
|
|
3963
4287
|
MUTAGENT_API_KEY=${config.apiKey}
|
|
3964
4288
|
MUTAGENT_ENDPOINT=${config.endpoint}
|
|
3965
4289
|
\`\`\`
|
|
3966
4290
|
|
|
3967
|
-
### 2.
|
|
3968
|
-
|
|
3969
|
-
Update \`mastra.config.ts\`:
|
|
4291
|
+
### 2. LangChain Callback Handler
|
|
3970
4292
|
|
|
3971
4293
|
\`\`\`typescript
|
|
3972
|
-
import {
|
|
3973
|
-
import {
|
|
3974
|
-
|
|
3975
|
-
export default defineConfig({
|
|
3976
|
-
observers: [
|
|
3977
|
-
new MutagentObserver({
|
|
3978
|
-
apiKey: process.env.MUTAGENT_API_KEY!,
|
|
3979
|
-
endpoint: process.env.MUTAGENT_ENDPOINT,
|
|
3980
|
-
// Optional: auto-capture all agent traces
|
|
3981
|
-
captureTraces: true,
|
|
3982
|
-
})
|
|
3983
|
-
],
|
|
3984
|
-
});
|
|
3985
|
-
\`\`\`
|
|
3986
|
-
|
|
3987
|
-
### 3. Per-Agent Configuration
|
|
4294
|
+
import { ChatOpenAI } from '@langchain/openai';
|
|
4295
|
+
import { MutagentCallbackHandler } from '@mutagent/langchain';
|
|
4296
|
+
import { initTracing } from '@mutagent/sdk/tracing';
|
|
3988
4297
|
|
|
3989
|
-
|
|
4298
|
+
// Initialize tracing (or set MUTAGENT_API_KEY env var for auto-init)
|
|
4299
|
+
initTracing({
|
|
4300
|
+
apiKey: process.env.MUTAGENT_API_KEY!,
|
|
4301
|
+
endpoint: process.env.MUTAGENT_ENDPOINT,
|
|
4302
|
+
});
|
|
3990
4303
|
|
|
3991
|
-
|
|
3992
|
-
|
|
3993
|
-
import { withMutagent } from '@mutagent/sdk/mastra';
|
|
4304
|
+
// Create callback handler (no args needed — uses SDK tracing)
|
|
4305
|
+
const handler = new MutagentCallbackHandler();
|
|
3994
4306
|
|
|
3995
|
-
const
|
|
3996
|
-
|
|
3997
|
-
instructions: '...',
|
|
4307
|
+
const llm = new ChatOpenAI({
|
|
4308
|
+
callbacks: [handler],
|
|
3998
4309
|
});
|
|
3999
4310
|
|
|
4000
|
-
|
|
4001
|
-
|
|
4002
|
-
promptId: 'support-template',
|
|
4003
|
-
autoOptimize: true,
|
|
4004
|
-
});
|
|
4311
|
+
const result = await llm.invoke('Hello world');
|
|
4312
|
+
// Traces automatically captured via LangChain callbacks
|
|
4005
4313
|
\`\`\`
|
|
4006
4314
|
|
|
4007
|
-
|
|
4008
|
-
|
|
4009
|
-
### Trace Collection (Replaces Langfuse)
|
|
4010
|
-
|
|
4011
|
-
All agent calls are automatically traced to MutagenT:
|
|
4315
|
+
### 3. Chain Integration
|
|
4012
4316
|
|
|
4013
4317
|
\`\`\`typescript
|
|
4014
|
-
|
|
4015
|
-
|
|
4016
|
-
|
|
4017
|
-
|
|
4018
|
-
### Prompt Optimization
|
|
4318
|
+
import { ChatOpenAI } from '@langchain/openai';
|
|
4319
|
+
import { PromptTemplate } from '@langchain/core/prompts';
|
|
4320
|
+
import { MutagentCallbackHandler } from '@mutagent/langchain';
|
|
4019
4321
|
|
|
4020
|
-
|
|
4021
|
-
import { optimizePrompt } from '@mutagent/sdk';
|
|
4322
|
+
const handler = new MutagentCallbackHandler();
|
|
4022
4323
|
|
|
4023
|
-
|
|
4024
|
-
|
|
4025
|
-
promptId: 'support-template',
|
|
4026
|
-
datasetId: 'support-tickets',
|
|
4027
|
-
metric: 'response_quality',
|
|
4324
|
+
const llm = new ChatOpenAI({
|
|
4325
|
+
callbacks: [handler],
|
|
4028
4326
|
});
|
|
4029
4327
|
|
|
4030
|
-
|
|
4031
|
-
|
|
4032
|
-
\`\`\`
|
|
4033
|
-
|
|
4034
|
-
### Manual Tracing
|
|
4035
|
-
|
|
4036
|
-
For custom traces:
|
|
4037
|
-
|
|
4038
|
-
\`\`\`typescript
|
|
4039
|
-
import { trace } from '@mutagent/sdk';
|
|
4040
|
-
|
|
4041
|
-
const span = trace.start({
|
|
4042
|
-
name: 'custom-operation',
|
|
4043
|
-
promptId: 'my-prompt',
|
|
4044
|
-
});
|
|
4328
|
+
const template = PromptTemplate.fromTemplate('Answer the question: {question}');
|
|
4329
|
+
const chain = template.pipe(llm);
|
|
4045
4330
|
|
|
4046
|
-
|
|
4047
|
-
|
|
4048
|
-
|
|
4049
|
-
|
|
4050
|
-
span.error(error);
|
|
4051
|
-
}
|
|
4331
|
+
const result = await chain.invoke(
|
|
4332
|
+
{ question: 'What is TypeScript?' },
|
|
4333
|
+
{ callbacks: [handler] },
|
|
4334
|
+
);
|
|
4052
4335
|
\`\`\`
|
|
4053
4336
|
|
|
4054
4337
|
## Migration from Langfuse
|
|
4055
4338
|
|
|
4056
|
-
Replace Langfuse
|
|
4339
|
+
Replace Langfuse callback:
|
|
4057
4340
|
|
|
4058
4341
|
\`\`\`typescript
|
|
4059
4342
|
// Before
|
|
4060
|
-
import {
|
|
4343
|
+
import { CallbackHandler } from 'langfuse-langchain';
|
|
4344
|
+
const callbacks = [new CallbackHandler()];
|
|
4061
4345
|
|
|
4062
4346
|
// After
|
|
4063
|
-
import {
|
|
4347
|
+
import { MutagentCallbackHandler } from '@mutagent/langchain';
|
|
4348
|
+
const callbacks = [new MutagentCallbackHandler()];
|
|
4064
4349
|
\`\`\`
|
|
4065
4350
|
|
|
4066
4351
|
## Verification
|
|
4067
4352
|
|
|
4068
|
-
Run this to verify your integration:
|
|
4069
|
-
|
|
4070
4353
|
\`\`\`bash
|
|
4071
|
-
mutagent integrate
|
|
4354
|
+
mutagent integrate langchain --verify
|
|
4072
4355
|
\`\`\`
|
|
4073
4356
|
|
|
4074
|
-
##
|
|
4357
|
+
## Documentation
|
|
4075
4358
|
|
|
4076
|
-
|
|
4077
|
-
|
|
4078
|
-
|
|
4079
|
-
4. View traces: \`mutagent traces list --prompt <prompt-id>\`
|
|
4359
|
+
- Full guide: https://docs.mutagent.io/integrations/langchain
|
|
4360
|
+
- API Reference: https://docs.mutagent.io/sdk/tracing
|
|
4361
|
+
- Dashboard: https://app.mutagent.io
|
|
4080
4362
|
|
|
4081
|
-
## CLI
|
|
4363
|
+
## CLI Reference
|
|
4082
4364
|
|
|
4083
4365
|
\`\`\`bash
|
|
4084
|
-
#
|
|
4085
|
-
mutagent
|
|
4086
|
-
|
|
4087
|
-
# Get prompt with traces
|
|
4088
|
-
mutagent prompts get <id> --with-datasets
|
|
4366
|
+
# View traces
|
|
4367
|
+
mutagent traces list --prompt <prompt-id>
|
|
4089
4368
|
|
|
4090
|
-
# Analyze
|
|
4369
|
+
# Analyze performance
|
|
4091
4370
|
mutagent traces analyze <prompt-id>
|
|
4092
4371
|
\`\`\`
|
|
4093
4372
|
`;
|
|
4094
4373
|
}
|
|
4095
4374
|
};
|
|
4096
4375
|
|
|
4097
|
-
// src/lib/integrations/
|
|
4376
|
+
// src/lib/integrations/langgraph.ts
|
|
4098
4377
|
import { readFileSync as readFileSync6, existsSync as existsSync6 } from "fs";
|
|
4099
|
-
var
|
|
4100
|
-
name: "
|
|
4101
|
-
description: "
|
|
4378
|
+
var langgraphIntegration = {
|
|
4379
|
+
name: "langgraph",
|
|
4380
|
+
description: "LangGraph agent workflow framework",
|
|
4102
4381
|
async detect() {
|
|
4103
|
-
let
|
|
4382
|
+
let hasLanggraph = false;
|
|
4104
4383
|
if (existsSync6("package.json")) {
|
|
4105
4384
|
try {
|
|
4106
4385
|
const pkg = JSON.parse(readFileSync6("package.json", "utf-8"));
|
|
4107
4386
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
4108
|
-
|
|
4387
|
+
hasLanggraph = "@langchain/langgraph" in deps;
|
|
4109
4388
|
} catch {}
|
|
4110
4389
|
}
|
|
4111
4390
|
return {
|
|
4112
|
-
detected:
|
|
4113
|
-
confidence:
|
|
4114
|
-
files:
|
|
4391
|
+
detected: hasLanggraph,
|
|
4392
|
+
confidence: hasLanggraph ? 0.95 : 0,
|
|
4393
|
+
files: hasLanggraph ? ["package.json"] : []
|
|
4115
4394
|
};
|
|
4116
4395
|
},
|
|
4117
4396
|
async generate(config) {
|
|
4118
4397
|
return `---
|
|
4119
|
-
name: mutagent-
|
|
4120
|
-
description: Integrate MutagenT with
|
|
4121
|
-
framework:
|
|
4122
|
-
version:
|
|
4398
|
+
name: mutagent-langgraph-integration
|
|
4399
|
+
description: Integrate MutagenT with LangGraph for agent workflow observability
|
|
4400
|
+
framework: langgraph
|
|
4401
|
+
version: 2.0.0
|
|
4123
4402
|
---
|
|
4124
4403
|
|
|
4125
|
-
# MutagenT +
|
|
4404
|
+
# MutagenT + LangGraph Integration
|
|
4126
4405
|
|
|
4127
4406
|
## Prerequisites Verification
|
|
4128
4407
|
|
|
@@ -4130,173 +4409,51 @@ version: 1.0.0
|
|
|
4130
4409
|
- MUTAGENT_ENDPOINT: ${config.endpoint}
|
|
4131
4410
|
- API Connection: Verified
|
|
4132
4411
|
|
|
4412
|
+
> **Note**: LangGraph is built on LangChain. The same \`MutagentCallbackHandler\` from
|
|
4413
|
+
> \`@mutagent/langchain\` works for both LangChain and LangGraph — no separate package needed.
|
|
4414
|
+
|
|
4133
4415
|
## Installation
|
|
4134
4416
|
|
|
4135
4417
|
\`\`\`bash
|
|
4136
|
-
|
|
4137
|
-
# If you need to install manually:
|
|
4138
|
-
bun add @mutagent/langchain
|
|
4418
|
+
bun add @mutagent/langchain @mutagent/sdk
|
|
4139
4419
|
# or
|
|
4140
|
-
npm install @mutagent/langchain
|
|
4420
|
+
npm install @mutagent/langchain @mutagent/sdk
|
|
4141
4421
|
\`\`\`
|
|
4142
4422
|
|
|
4143
|
-
|
|
4144
|
-
|
|
4145
|
-
### 1. Environment Variables
|
|
4146
|
-
|
|
4147
|
-
\`\`\`env
|
|
4148
|
-
MUTAGENT_API_KEY=${config.apiKey}
|
|
4149
|
-
MUTAGENT_ENDPOINT=${config.endpoint}
|
|
4150
|
-
\`\`\`
|
|
4423
|
+
> **Deprecation Notice**: The \`@mutagent/langgraph\` package is deprecated.
|
|
4424
|
+
> Use \`@mutagent/langchain\` instead — it supports both LangChain and LangGraph.
|
|
4151
4425
|
|
|
4152
|
-
|
|
4426
|
+
## Integration
|
|
4153
4427
|
|
|
4154
4428
|
\`\`\`typescript
|
|
4155
|
-
import { ChatOpenAI } from '@langchain/openai';
|
|
4156
4429
|
import { MutagentCallbackHandler } from '@mutagent/langchain';
|
|
4157
4430
|
import { initTracing } from '@mutagent/sdk/tracing';
|
|
4158
4431
|
|
|
4159
|
-
// Initialize tracing (
|
|
4160
|
-
initTracing({
|
|
4161
|
-
apiKey: process.env.MUTAGENT_API_KEY!,
|
|
4162
|
-
endpoint: process.env.MUTAGENT_ENDPOINT,
|
|
4163
|
-
});
|
|
4164
|
-
|
|
4165
|
-
// Create callback handler (no args needed — uses SDK tracing)
|
|
4166
|
-
const handler = new MutagentCallbackHandler();
|
|
4432
|
+
// Initialize tracing (once at app startup)
|
|
4433
|
+
initTracing({ apiKey: process.env.MUTAGENT_API_KEY! });
|
|
4167
4434
|
|
|
4168
|
-
|
|
4169
|
-
|
|
4435
|
+
// Create the handler
|
|
4436
|
+
const handler = new MutagentCallbackHandler({
|
|
4437
|
+
sessionId: 'my-session', // optional
|
|
4438
|
+
userId: 'user-123', // optional
|
|
4170
4439
|
});
|
|
4171
4440
|
|
|
4172
|
-
|
|
4173
|
-
|
|
4441
|
+
// Pass to any LangGraph invoke/stream call
|
|
4442
|
+
const result = await graph.invoke(input, { callbacks: [handler] });
|
|
4174
4443
|
\`\`\`
|
|
4175
4444
|
|
|
4176
|
-
|
|
4445
|
+
## Full Graph Example
|
|
4177
4446
|
|
|
4178
4447
|
\`\`\`typescript
|
|
4448
|
+
import { StateGraph, Annotation } from '@langchain/langgraph';
|
|
4179
4449
|
import { ChatOpenAI } from '@langchain/openai';
|
|
4180
|
-
import { PromptTemplate } from '@langchain/core/prompts';
|
|
4181
4450
|
import { MutagentCallbackHandler } from '@mutagent/langchain';
|
|
4182
|
-
|
|
4183
|
-
const handler = new MutagentCallbackHandler();
|
|
4184
|
-
|
|
4185
|
-
const llm = new ChatOpenAI({
|
|
4186
|
-
callbacks: [handler],
|
|
4187
|
-
});
|
|
4188
|
-
|
|
4189
|
-
const template = PromptTemplate.fromTemplate('Answer the question: {question}');
|
|
4190
|
-
const chain = template.pipe(llm);
|
|
4191
|
-
|
|
4192
|
-
const result = await chain.invoke(
|
|
4193
|
-
{ question: 'What is TypeScript?' },
|
|
4194
|
-
{ callbacks: [handler] },
|
|
4195
|
-
);
|
|
4196
|
-
\`\`\`
|
|
4197
|
-
|
|
4198
|
-
## Migration from Langfuse
|
|
4199
|
-
|
|
4200
|
-
Replace Langfuse callback:
|
|
4201
|
-
|
|
4202
|
-
\`\`\`typescript
|
|
4203
|
-
// Before
|
|
4204
|
-
import { CallbackHandler } from 'langfuse-langchain';
|
|
4205
|
-
const callbacks = [new CallbackHandler()];
|
|
4206
|
-
|
|
4207
|
-
// After
|
|
4208
|
-
import { MutagentCallbackHandler } from '@mutagent/langchain';
|
|
4209
|
-
const callbacks = [new MutagentCallbackHandler()];
|
|
4210
|
-
\`\`\`
|
|
4211
|
-
|
|
4212
|
-
## Verification
|
|
4213
|
-
|
|
4214
|
-
\`\`\`bash
|
|
4215
|
-
mutagent integrate langchain --verify
|
|
4216
|
-
\`\`\`
|
|
4217
|
-
|
|
4218
|
-
## Documentation
|
|
4219
|
-
|
|
4220
|
-
- Full guide: https://docs.mutagent.io/integrations/langchain
|
|
4221
|
-
- API Reference: https://docs.mutagent.io/sdk/tracing
|
|
4222
|
-
- Dashboard: https://app.mutagent.io
|
|
4223
|
-
|
|
4224
|
-
## CLI Reference
|
|
4225
|
-
|
|
4226
|
-
\`\`\`bash
|
|
4227
|
-
# View traces
|
|
4228
|
-
mutagent traces list --prompt <prompt-id>
|
|
4229
|
-
|
|
4230
|
-
# Analyze performance
|
|
4231
|
-
mutagent traces analyze <prompt-id>
|
|
4232
|
-
\`\`\`
|
|
4233
|
-
`;
|
|
4234
|
-
}
|
|
4235
|
-
};
|
|
4236
|
-
|
|
4237
|
-
// src/lib/integrations/langgraph.ts
|
|
4238
|
-
import { readFileSync as readFileSync7, existsSync as existsSync7 } from "fs";
|
|
4239
|
-
var langgraphIntegration = {
|
|
4240
|
-
name: "langgraph",
|
|
4241
|
-
description: "LangGraph agent workflow framework",
|
|
4242
|
-
async detect() {
|
|
4243
|
-
let hasLanggraph = false;
|
|
4244
|
-
if (existsSync7("package.json")) {
|
|
4245
|
-
try {
|
|
4246
|
-
const pkg = JSON.parse(readFileSync7("package.json", "utf-8"));
|
|
4247
|
-
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
4248
|
-
hasLanggraph = "@langchain/langgraph" in deps;
|
|
4249
|
-
} catch {}
|
|
4250
|
-
}
|
|
4251
|
-
return {
|
|
4252
|
-
detected: hasLanggraph,
|
|
4253
|
-
confidence: hasLanggraph ? 0.95 : 0,
|
|
4254
|
-
files: hasLanggraph ? ["package.json"] : []
|
|
4255
|
-
};
|
|
4256
|
-
},
|
|
4257
|
-
async generate(config) {
|
|
4258
|
-
return `---
|
|
4259
|
-
name: mutagent-langgraph-integration
|
|
4260
|
-
description: Integrate MutagenT with LangGraph for agent workflow observability
|
|
4261
|
-
framework: langgraph
|
|
4262
|
-
version: 1.0.0
|
|
4263
|
-
---
|
|
4264
|
-
|
|
4265
|
-
# MutagenT + LangGraph Integration
|
|
4266
|
-
|
|
4267
|
-
## Prerequisites Verification
|
|
4268
|
-
|
|
4269
|
-
- MUTAGENT_API_KEY: ${config.apiKey.slice(0, 8)}...${config.apiKey.slice(-4)}
|
|
4270
|
-
- MUTAGENT_ENDPOINT: ${config.endpoint}
|
|
4271
|
-
- API Connection: Verified
|
|
4272
|
-
|
|
4273
|
-
## Installation
|
|
4274
|
-
|
|
4275
|
-
\`\`\`bash
|
|
4276
|
-
# The @mutagent/langgraph package was installed automatically
|
|
4277
|
-
# If you need to install manually:
|
|
4278
|
-
bun add @mutagent/langgraph
|
|
4279
|
-
# or
|
|
4280
|
-
npm install @mutagent/langgraph
|
|
4281
|
-
\`\`\`
|
|
4282
|
-
|
|
4283
|
-
## Configuration
|
|
4284
|
-
|
|
4285
|
-
### Graph with MutagenT Tracer
|
|
4286
|
-
|
|
4287
|
-
\`\`\`typescript
|
|
4288
|
-
import { StateGraph, Annotation } from '@langchain/langgraph';
|
|
4289
|
-
import { MutagentGraphTracer } from '@mutagent/langgraph';
|
|
4290
4451
|
import { initTracing } from '@mutagent/sdk/tracing';
|
|
4291
4452
|
|
|
4292
|
-
// Initialize tracing (
|
|
4293
|
-
initTracing({
|
|
4294
|
-
apiKey: process.env.MUTAGENT_API_KEY!,
|
|
4295
|
-
endpoint: process.env.MUTAGENT_ENDPOINT,
|
|
4296
|
-
});
|
|
4453
|
+
// Initialize tracing (once at app startup)
|
|
4454
|
+
initTracing({ apiKey: process.env.MUTAGENT_API_KEY! });
|
|
4297
4455
|
|
|
4298
|
-
|
|
4299
|
-
const tracer = new MutagentGraphTracer();
|
|
4456
|
+
const handler = new MutagentCallbackHandler();
|
|
4300
4457
|
|
|
4301
4458
|
// Define your graph as usual
|
|
4302
4459
|
const StateAnnotation = Annotation.Root({
|
|
@@ -4310,29 +4467,24 @@ const graph = new StateGraph(StateAnnotation)
|
|
|
4310
4467
|
.addEdge('__start__', 'agent')
|
|
4311
4468
|
.compile();
|
|
4312
4469
|
|
|
4313
|
-
//
|
|
4314
|
-
|
|
4315
|
-
|
|
4316
|
-
|
|
4470
|
+
// All nodes, edges, and LLM calls are automatically traced
|
|
4471
|
+
const result = await graph.invoke(
|
|
4472
|
+
{ input: 'Hello' },
|
|
4473
|
+
{ callbacks: [handler] },
|
|
4474
|
+
);
|
|
4317
4475
|
\`\`\`
|
|
4318
4476
|
|
|
4319
|
-
|
|
4477
|
+
## Streaming
|
|
4320
4478
|
|
|
4321
4479
|
\`\`\`typescript
|
|
4322
|
-
|
|
4323
|
-
tracer.handleNodeStart('agent', { query: 'Hello' });
|
|
4324
|
-
// ... node logic ...
|
|
4325
|
-
tracer.handleNodeEnd('agent', { result: 'response' });
|
|
4326
|
-
|
|
4327
|
-
// Track edge transitions
|
|
4328
|
-
tracer.handleEdgeTransition('agent', 'tools', {
|
|
4329
|
-
isConditional: true,
|
|
4330
|
-
condition: 'needsTools',
|
|
4331
|
-
conditionResult: true,
|
|
4332
|
-
});
|
|
4480
|
+
const handler = new MutagentCallbackHandler();
|
|
4333
4481
|
|
|
4334
|
-
|
|
4335
|
-
|
|
4482
|
+
for await (const event of graph.stream(
|
|
4483
|
+
{ input: 'Hello' },
|
|
4484
|
+
{ callbacks: [handler] },
|
|
4485
|
+
)) {
|
|
4486
|
+
console.log(event);
|
|
4487
|
+
}
|
|
4336
4488
|
\`\`\`
|
|
4337
4489
|
|
|
4338
4490
|
## Verification
|
|
@@ -4346,30 +4498,20 @@ mutagent integrate langgraph --verify
|
|
|
4346
4498
|
- Full guide: https://docs.mutagent.io/integrations/langgraph
|
|
4347
4499
|
- API Reference: https://docs.mutagent.io/sdk/tracing
|
|
4348
4500
|
- Dashboard: https://app.mutagent.io
|
|
4349
|
-
|
|
4350
|
-
## CLI Commands
|
|
4351
|
-
|
|
4352
|
-
\`\`\`bash
|
|
4353
|
-
# View workflow traces
|
|
4354
|
-
mutagent traces list --prompt <graph-id>
|
|
4355
|
-
|
|
4356
|
-
# Analyze node performance
|
|
4357
|
-
mutagent traces analyze <node-prompt-id>
|
|
4358
|
-
\`\`\`
|
|
4359
4501
|
`;
|
|
4360
4502
|
}
|
|
4361
4503
|
};
|
|
4362
4504
|
|
|
4363
4505
|
// src/lib/integrations/vercel-ai.ts
|
|
4364
|
-
import { readFileSync as
|
|
4506
|
+
import { readFileSync as readFileSync7, existsSync as existsSync7 } from "fs";
|
|
4365
4507
|
var vercelAiIntegration = {
|
|
4366
4508
|
name: "vercel-ai",
|
|
4367
4509
|
description: "Vercel AI SDK",
|
|
4368
4510
|
async detect() {
|
|
4369
4511
|
let hasAiSdk = false;
|
|
4370
|
-
if (
|
|
4512
|
+
if (existsSync7("package.json")) {
|
|
4371
4513
|
try {
|
|
4372
|
-
const pkg = JSON.parse(
|
|
4514
|
+
const pkg = JSON.parse(readFileSync7("package.json", "utf-8"));
|
|
4373
4515
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
4374
4516
|
hasAiSdk = "ai" in deps;
|
|
4375
4517
|
} catch {}
|
|
@@ -4385,7 +4527,7 @@ var vercelAiIntegration = {
|
|
|
4385
4527
|
name: mutagent-vercel-ai-integration
|
|
4386
4528
|
description: Integrate MutagenT with Vercel AI SDK for streaming and edge functions
|
|
4387
4529
|
framework: vercel-ai
|
|
4388
|
-
version:
|
|
4530
|
+
version: 2.0.0
|
|
4389
4531
|
---
|
|
4390
4532
|
|
|
4391
4533
|
# MutagenT + Vercel AI SDK Integration
|
|
@@ -4399,14 +4541,52 @@ version: 1.0.0
|
|
|
4399
4541
|
## Installation
|
|
4400
4542
|
|
|
4401
4543
|
\`\`\`bash
|
|
4402
|
-
|
|
4403
|
-
# If you need to install manually:
|
|
4404
|
-
bun add @mutagent/vercel-ai
|
|
4544
|
+
bun add @mutagent/vercel-ai @mutagent/sdk
|
|
4405
4545
|
# or
|
|
4406
|
-
npm install @mutagent/vercel-ai
|
|
4546
|
+
npm install @mutagent/vercel-ai @mutagent/sdk
|
|
4547
|
+
|
|
4548
|
+
# For Option A (OTel SpanExporter), also install:
|
|
4549
|
+
bun add @opentelemetry/sdk-trace-node @opentelemetry/sdk-trace-base
|
|
4407
4550
|
\`\`\`
|
|
4408
4551
|
|
|
4409
|
-
|
|
4552
|
+
---
|
|
4553
|
+
|
|
4554
|
+
## Option A (Recommended): OTel SpanExporter
|
|
4555
|
+
|
|
4556
|
+
Uses Vercel AI SDK's built-in \`experimental_telemetry\` with an OpenTelemetry exporter
|
|
4557
|
+
that sends spans directly to MutagenT.
|
|
4558
|
+
|
|
4559
|
+
\`\`\`typescript
|
|
4560
|
+
import { MutagentSpanExporter } from '@mutagent/vercel-ai';
|
|
4561
|
+
import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
|
|
4562
|
+
import { SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base';
|
|
4563
|
+
import { initTracing } from '@mutagent/sdk/tracing';
|
|
4564
|
+
import { generateText } from 'ai';
|
|
4565
|
+
import { openai } from '@ai-sdk/openai';
|
|
4566
|
+
|
|
4567
|
+
// Initialize MutagenT tracing
|
|
4568
|
+
initTracing({ apiKey: process.env.MUTAGENT_API_KEY! });
|
|
4569
|
+
|
|
4570
|
+
// Set up OTel with MutagenT exporter
|
|
4571
|
+
const provider = new NodeTracerProvider();
|
|
4572
|
+
provider.addSpanProcessor(
|
|
4573
|
+
new SimpleSpanProcessor(new MutagentSpanExporter())
|
|
4574
|
+
);
|
|
4575
|
+
provider.register();
|
|
4576
|
+
|
|
4577
|
+
// Use Vercel AI SDK normally with telemetry enabled
|
|
4578
|
+
const result = await generateText({
|
|
4579
|
+
model: openai('gpt-4'),
|
|
4580
|
+
prompt: 'Hello!',
|
|
4581
|
+
experimental_telemetry: { isEnabled: true },
|
|
4582
|
+
});
|
|
4583
|
+
\`\`\`
|
|
4584
|
+
|
|
4585
|
+
---
|
|
4586
|
+
|
|
4587
|
+
## Option B (Alternative): Middleware
|
|
4588
|
+
|
|
4589
|
+
Uses the Vercel AI SDK \`wrapLanguageModel\` middleware pattern.
|
|
4410
4590
|
|
|
4411
4591
|
\`\`\`typescript
|
|
4412
4592
|
// app/api/chat/route.ts
|
|
@@ -4415,13 +4595,10 @@ import { openai } from '@ai-sdk/openai';
|
|
|
4415
4595
|
import { createMutagentMiddleware } from '@mutagent/vercel-ai';
|
|
4416
4596
|
import { initTracing } from '@mutagent/sdk/tracing';
|
|
4417
4597
|
|
|
4418
|
-
// Initialize tracing (
|
|
4419
|
-
initTracing({
|
|
4420
|
-
apiKey: process.env.MUTAGENT_API_KEY!,
|
|
4421
|
-
endpoint: process.env.MUTAGENT_ENDPOINT,
|
|
4422
|
-
});
|
|
4598
|
+
// Initialize tracing (once at app startup)
|
|
4599
|
+
initTracing({ apiKey: process.env.MUTAGENT_API_KEY! });
|
|
4423
4600
|
|
|
4424
|
-
// Create middleware
|
|
4601
|
+
// Create middleware
|
|
4425
4602
|
const middleware = createMutagentMiddleware();
|
|
4426
4603
|
|
|
4427
4604
|
// Wrap your model with MutagenT middleware
|
|
@@ -4467,30 +4644,20 @@ mutagent integrate vercel-ai --verify
|
|
|
4467
4644
|
- Full guide: https://docs.mutagent.io/integrations/vercel-ai
|
|
4468
4645
|
- API Reference: https://docs.mutagent.io/sdk/tracing
|
|
4469
4646
|
- Dashboard: https://app.mutagent.io
|
|
4470
|
-
|
|
4471
|
-
## CLI Commands
|
|
4472
|
-
|
|
4473
|
-
\`\`\`bash
|
|
4474
|
-
# View streaming traces
|
|
4475
|
-
mutagent traces list --prompt <prompt-id>
|
|
4476
|
-
|
|
4477
|
-
# Export for analysis
|
|
4478
|
-
mutagent traces export --prompt <prompt-id> --format json
|
|
4479
|
-
\`\`\`
|
|
4480
4647
|
`;
|
|
4481
4648
|
}
|
|
4482
4649
|
};
|
|
4483
4650
|
|
|
4484
4651
|
// src/lib/integrations/openai.ts
|
|
4485
|
-
import { readFileSync as
|
|
4652
|
+
import { readFileSync as readFileSync8, existsSync as existsSync8 } from "fs";
|
|
4486
4653
|
var openaiIntegration = {
|
|
4487
4654
|
name: "openai",
|
|
4488
4655
|
description: "OpenAI SDK integration with automatic tracing",
|
|
4489
4656
|
async detect() {
|
|
4490
4657
|
let hasOpenAI = false;
|
|
4491
|
-
if (
|
|
4658
|
+
if (existsSync8("package.json")) {
|
|
4492
4659
|
try {
|
|
4493
|
-
const pkg = JSON.parse(
|
|
4660
|
+
const pkg = JSON.parse(readFileSync8("package.json", "utf-8"));
|
|
4494
4661
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
4495
4662
|
hasOpenAI = "openai" in deps;
|
|
4496
4663
|
} catch {}
|
|
@@ -4506,7 +4673,7 @@ var openaiIntegration = {
|
|
|
4506
4673
|
name: mutagent-openai-integration
|
|
4507
4674
|
description: Integrate MutagenT with OpenAI SDK for automatic tracing
|
|
4508
4675
|
framework: openai
|
|
4509
|
-
version:
|
|
4676
|
+
version: 2.0.0
|
|
4510
4677
|
---
|
|
4511
4678
|
|
|
4512
4679
|
# MutagenT + OpenAI SDK Integration
|
|
@@ -4520,41 +4687,28 @@ version: 1.0.0
|
|
|
4520
4687
|
## Installation
|
|
4521
4688
|
|
|
4522
4689
|
\`\`\`bash
|
|
4523
|
-
|
|
4524
|
-
# If you need to install manually:
|
|
4525
|
-
bun add @mutagent/openai
|
|
4690
|
+
bun add @mutagent/openai @mutagent/sdk
|
|
4526
4691
|
# or
|
|
4527
|
-
npm install @mutagent/openai
|
|
4528
|
-
\`\`\`
|
|
4529
|
-
|
|
4530
|
-
## Configuration
|
|
4531
|
-
|
|
4532
|
-
### 1. Environment Variables
|
|
4533
|
-
|
|
4534
|
-
\`\`\`env
|
|
4535
|
-
MUTAGENT_API_KEY=${config.apiKey}
|
|
4536
|
-
MUTAGENT_ENDPOINT=${config.endpoint}
|
|
4537
|
-
OPENAI_API_KEY=your-openai-api-key
|
|
4692
|
+
npm install @mutagent/openai @mutagent/sdk
|
|
4538
4693
|
\`\`\`
|
|
4539
4694
|
|
|
4540
|
-
|
|
4695
|
+
## Integration
|
|
4541
4696
|
|
|
4542
4697
|
\`\`\`typescript
|
|
4543
|
-
import
|
|
4698
|
+
import OpenAI from 'openai';
|
|
4699
|
+
import { observeOpenAI } from '@mutagent/openai';
|
|
4544
4700
|
import { initTracing } from '@mutagent/sdk/tracing';
|
|
4545
4701
|
|
|
4546
|
-
// Initialize tracing (
|
|
4547
|
-
initTracing({
|
|
4548
|
-
apiKey: process.env.MUTAGENT_API_KEY!,
|
|
4549
|
-
endpoint: process.env.MUTAGENT_ENDPOINT,
|
|
4550
|
-
});
|
|
4702
|
+
// Initialize tracing (once at app startup)
|
|
4703
|
+
initTracing({ apiKey: process.env.MUTAGENT_API_KEY! });
|
|
4551
4704
|
|
|
4552
|
-
//
|
|
4553
|
-
const openai = new
|
|
4554
|
-
|
|
4705
|
+
// Wrap the OpenAI client — ALL methods are automatically traced
|
|
4706
|
+
const openai = observeOpenAI(new OpenAI(), {
|
|
4707
|
+
sessionId: 'my-session', // optional
|
|
4708
|
+
userId: 'user-123', // optional
|
|
4555
4709
|
});
|
|
4556
4710
|
|
|
4557
|
-
//
|
|
4711
|
+
// Use exactly as normal — chat, embeddings, images, audio all work
|
|
4558
4712
|
const completion = await openai.chat.completions.create({
|
|
4559
4713
|
model: 'gpt-4o',
|
|
4560
4714
|
messages: [{ role: 'user', content: 'Hello!' }],
|
|
@@ -4563,21 +4717,12 @@ const completion = await openai.chat.completions.create({
|
|
|
4563
4717
|
console.log(completion.choices[0].message.content);
|
|
4564
4718
|
\`\`\`
|
|
4565
4719
|
|
|
4566
|
-
|
|
4720
|
+
ALL OpenAI SDK methods are preserved and traced automatically.
|
|
4721
|
+
No API changes — just wrap your client and everything is observed.
|
|
4567
4722
|
|
|
4568
|
-
|
|
4569
|
-
|
|
4570
|
-
\`\`\`typescript
|
|
4571
|
-
const response = await openai.chat.completions.create({
|
|
4572
|
-
model: 'gpt-4o',
|
|
4573
|
-
messages: [
|
|
4574
|
-
{ role: 'system', content: 'You are a helpful assistant.' },
|
|
4575
|
-
{ role: 'user', content: 'What is TypeScript?' },
|
|
4576
|
-
],
|
|
4577
|
-
});
|
|
4578
|
-
\`\`\`
|
|
4723
|
+
## Streaming
|
|
4579
4724
|
|
|
4580
|
-
|
|
4725
|
+
Streaming works out of the box with no extra configuration:
|
|
4581
4726
|
|
|
4582
4727
|
\`\`\`typescript
|
|
4583
4728
|
const stream = await openai.chat.completions.create({
|
|
@@ -4591,299 +4736,39 @@ for await (const chunk of stream) {
|
|
|
4591
4736
|
}
|
|
4592
4737
|
\`\`\`
|
|
4593
4738
|
|
|
4594
|
-
##
|
|
4595
|
-
|
|
4596
|
-
\`\`\`bash
|
|
4597
|
-
mutagent integrate openai --verify
|
|
4598
|
-
\`\`\`
|
|
4599
|
-
|
|
4600
|
-
## Documentation
|
|
4601
|
-
|
|
4602
|
-
- Full guide: https://docs.mutagent.io/integrations/openai
|
|
4603
|
-
- API Reference: https://docs.mutagent.io/sdk/tracing
|
|
4604
|
-
- Dashboard: https://app.mutagent.io
|
|
4605
|
-
|
|
4606
|
-
## CLI Reference
|
|
4607
|
-
|
|
4608
|
-
\`\`\`bash
|
|
4609
|
-
# View traces
|
|
4610
|
-
mutagent traces list --prompt <prompt-id>
|
|
4611
|
-
|
|
4612
|
-
# Analyze performance
|
|
4613
|
-
mutagent traces analyze <prompt-id>
|
|
4614
|
-
|
|
4615
|
-
# Export traces
|
|
4616
|
-
mutagent traces export --format json
|
|
4617
|
-
\`\`\`
|
|
4618
|
-
`;
|
|
4619
|
-
}
|
|
4620
|
-
};
|
|
4621
|
-
|
|
4622
|
-
// src/lib/integrations/claude-code.ts
|
|
4623
|
-
import { readFileSync as readFileSync10, existsSync as existsSync10 } from "fs";
|
|
4624
|
-
var claudeCodeIntegration = {
|
|
4625
|
-
name: "claude-code",
|
|
4626
|
-
description: "Claude Code SDK",
|
|
4627
|
-
async detect() {
|
|
4628
|
-
let hasClaudeCode = false;
|
|
4629
|
-
if (existsSync10("package.json")) {
|
|
4630
|
-
try {
|
|
4631
|
-
const pkg = JSON.parse(readFileSync10("package.json", "utf-8"));
|
|
4632
|
-
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
4633
|
-
hasClaudeCode = "@anthropic-ai/claude-code" in deps || "@anthropic-ai/sdk" in deps;
|
|
4634
|
-
} catch {}
|
|
4635
|
-
}
|
|
4636
|
-
return {
|
|
4637
|
-
detected: hasClaudeCode,
|
|
4638
|
-
confidence: hasClaudeCode ? 0.85 : 0,
|
|
4639
|
-
files: hasClaudeCode ? ["package.json"] : []
|
|
4640
|
-
};
|
|
4641
|
-
},
|
|
4642
|
-
async generate(config) {
|
|
4643
|
-
return `---
|
|
4644
|
-
name: mutagent-claude-code-integration
|
|
4645
|
-
description: Integrate MutagenT with Claude Code SDK
|
|
4646
|
-
framework: claude-code
|
|
4647
|
-
version: 1.0.0
|
|
4648
|
-
---
|
|
4649
|
-
|
|
4650
|
-
# MutagenT + Claude Code SDK Integration
|
|
4651
|
-
|
|
4652
|
-
## Prerequisites Verification
|
|
4653
|
-
|
|
4654
|
-
✓ MUTAGENT_API_KEY: ${config.apiKey.slice(0, 8)}...${config.apiKey.slice(-4)}
|
|
4655
|
-
✓ MUTAGENT_ENDPOINT: ${config.endpoint}
|
|
4656
|
-
✓ API Connection: Verified
|
|
4657
|
-
|
|
4658
|
-
## Installation
|
|
4659
|
-
|
|
4660
|
-
\`\`\`bash
|
|
4661
|
-
bun add @mutagent/sdk
|
|
4662
|
-
\`\`\`
|
|
4663
|
-
|
|
4664
|
-
## Basic Integration
|
|
4739
|
+
## Embeddings
|
|
4665
4740
|
|
|
4666
4741
|
\`\`\`typescript
|
|
4667
|
-
|
|
4668
|
-
|
|
4669
|
-
|
|
4670
|
-
const tracer = new MutagentTracer({
|
|
4671
|
-
apiKey: process.env.MUTAGENT_API_KEY!,
|
|
4672
|
-
endpoint: process.env.MUTAGENT_ENDPOINT,
|
|
4673
|
-
promptId: 'claude-prompt',
|
|
4674
|
-
});
|
|
4675
|
-
|
|
4676
|
-
const anthropic = new Anthropic({
|
|
4677
|
-
apiKey: process.env.ANTHROPIC_API_KEY!,
|
|
4742
|
+
const embedding = await openai.embeddings.create({
|
|
4743
|
+
model: 'text-embedding-3-small',
|
|
4744
|
+
input: 'Hello world',
|
|
4678
4745
|
});
|
|
4679
|
-
|
|
4680
|
-
// Wrap API calls
|
|
4681
|
-
const response = await tracer.wrap(
|
|
4682
|
-
() => anthropic.messages.create({
|
|
4683
|
-
model: 'claude-3-5-sonnet-20241022',
|
|
4684
|
-
max_tokens: 1024,
|
|
4685
|
-
messages: [{ role: 'user', content: 'Hello!' }],
|
|
4686
|
-
})
|
|
4687
|
-
);
|
|
4688
|
-
\`\`\`
|
|
4689
|
-
|
|
4690
|
-
## With Optimized Prompts
|
|
4691
|
-
|
|
4692
|
-
\`\`\`typescript
|
|
4693
|
-
import { getOptimizedPrompt } from '@mutagent/sdk';
|
|
4694
|
-
|
|
4695
|
-
const optimized = await getOptimizedPrompt('my-template');
|
|
4696
|
-
|
|
4697
|
-
const response = await tracer.wrap(
|
|
4698
|
-
() => anthropic.messages.create({
|
|
4699
|
-
model: 'claude-3-5-sonnet-20241022',
|
|
4700
|
-
system: optimized,
|
|
4701
|
-
messages,
|
|
4702
|
-
})
|
|
4703
|
-
);
|
|
4704
|
-
\`\`\`
|
|
4705
|
-
|
|
4706
|
-
## MCP Server Integration
|
|
4707
|
-
|
|
4708
|
-
For Claude Desktop / MCP:
|
|
4709
|
-
|
|
4710
|
-
\`\`\`json
|
|
4711
|
-
{
|
|
4712
|
-
"mcpServers": {
|
|
4713
|
-
"mutagent": {
|
|
4714
|
-
"command": "npx",
|
|
4715
|
-
"args": ["-y", "@mutagent/mcp-server"],
|
|
4716
|
-
"env": {
|
|
4717
|
-
"MUTAGENT_API_KEY": "${config.apiKey}"
|
|
4718
|
-
}
|
|
4719
|
-
}
|
|
4720
|
-
}
|
|
4721
|
-
}
|
|
4746
|
+
// Embedding call is traced automatically
|
|
4722
4747
|
\`\`\`
|
|
4723
4748
|
|
|
4724
4749
|
## Verification
|
|
4725
4750
|
|
|
4726
4751
|
\`\`\`bash
|
|
4727
|
-
mutagent integrate
|
|
4728
|
-
\`\`\`
|
|
4729
|
-
|
|
4730
|
-
## CLI Commands
|
|
4731
|
-
|
|
4732
|
-
\`\`\`bash
|
|
4733
|
-
# View Claude traces
|
|
4734
|
-
mutagent traces list --prompt <prompt-id>
|
|
4735
|
-
|
|
4736
|
-
# Optimize Claude prompts
|
|
4737
|
-
mutagent prompts optimize <prompt-id> --dataset <dataset-id>
|
|
4738
|
-
\`\`\`
|
|
4739
|
-
`;
|
|
4740
|
-
}
|
|
4741
|
-
};
|
|
4742
|
-
|
|
4743
|
-
// src/lib/integrations/generic.ts
|
|
4744
|
-
var genericIntegration = {
|
|
4745
|
-
name: "generic",
|
|
4746
|
-
description: "Generic OpenAI-compatible integration",
|
|
4747
|
-
async detect() {
|
|
4748
|
-
return {
|
|
4749
|
-
detected: false,
|
|
4750
|
-
confidence: 0,
|
|
4751
|
-
files: []
|
|
4752
|
-
};
|
|
4753
|
-
},
|
|
4754
|
-
async generate(config) {
|
|
4755
|
-
return `---
|
|
4756
|
-
name: mutagent-generic-integration
|
|
4757
|
-
description: Generic OpenAI-compatible API integration
|
|
4758
|
-
framework: generic
|
|
4759
|
-
version: 1.0.0
|
|
4760
|
-
---
|
|
4761
|
-
|
|
4762
|
-
# MutagenT + Generic (OpenAI-compatible) Integration
|
|
4763
|
-
|
|
4764
|
-
## Prerequisites Verification
|
|
4765
|
-
|
|
4766
|
-
✓ MUTAGENT_API_KEY: ${config.apiKey.slice(0, 8)}...${config.apiKey.slice(-4)}
|
|
4767
|
-
✓ MUTAGENT_ENDPOINT: ${config.endpoint}
|
|
4768
|
-
✓ API Connection: Verified
|
|
4769
|
-
|
|
4770
|
-
## Installation
|
|
4771
|
-
|
|
4772
|
-
\`\`\`bash
|
|
4773
|
-
bun add @mutagent/sdk
|
|
4774
|
-
\`\`\`
|
|
4775
|
-
|
|
4776
|
-
## Fetch-based Integration
|
|
4777
|
-
|
|
4778
|
-
\`\`\`typescript
|
|
4779
|
-
import { trace } from '@mutagent/sdk';
|
|
4780
|
-
|
|
4781
|
-
async function callLLM(prompt: string) {
|
|
4782
|
-
const span = trace.start({
|
|
4783
|
-
name: 'llm-call',
|
|
4784
|
-
promptId: 'my-prompt',
|
|
4785
|
-
input: { prompt },
|
|
4786
|
-
});
|
|
4787
|
-
|
|
4788
|
-
const start = Date.now();
|
|
4789
|
-
|
|
4790
|
-
try {
|
|
4791
|
-
const response = await fetch('https://api.openai.com/v1/chat/completions', {
|
|
4792
|
-
method: 'POST',
|
|
4793
|
-
headers: {
|
|
4794
|
-
'Authorization': \`Bearer \${process.env.OPENAI_API_KEY}\`,
|
|
4795
|
-
'Content-Type': 'application/json',
|
|
4796
|
-
},
|
|
4797
|
-
body: JSON.stringify({
|
|
4798
|
-
model: 'gpt-4o',
|
|
4799
|
-
messages: [{ role: 'user', content: prompt }],
|
|
4800
|
-
}),
|
|
4801
|
-
});
|
|
4802
|
-
|
|
4803
|
-
const data = await response.json();
|
|
4804
|
-
const latency = Date.now() - start;
|
|
4805
|
-
|
|
4806
|
-
span.end({
|
|
4807
|
-
output: data,
|
|
4808
|
-
latency,
|
|
4809
|
-
tokens: {
|
|
4810
|
-
input: data.usage?.prompt_tokens || 0,
|
|
4811
|
-
output: data.usage?.completion_tokens || 0,
|
|
4812
|
-
},
|
|
4813
|
-
});
|
|
4814
|
-
|
|
4815
|
-
return data;
|
|
4816
|
-
} catch (error) {
|
|
4817
|
-
span.error(error);
|
|
4818
|
-
throw error;
|
|
4819
|
-
}
|
|
4820
|
-
}
|
|
4821
|
-
\`\`\`
|
|
4822
|
-
|
|
4823
|
-
## With Optimized Prompts
|
|
4824
|
-
|
|
4825
|
-
\`\`\`typescript
|
|
4826
|
-
import { getOptimizedPrompt } from '@mutagent/sdk';
|
|
4827
|
-
|
|
4828
|
-
const optimized = await getOptimizedPrompt('template-id');
|
|
4829
|
-
const result = await callLLM(optimized);
|
|
4830
|
-
\`\`\`
|
|
4831
|
-
|
|
4832
|
-
## OpenAI SDK Wrapper
|
|
4833
|
-
|
|
4834
|
-
\`\`\`typescript
|
|
4835
|
-
import OpenAI from 'openai';
|
|
4836
|
-
import { wrapOpenAI } from '@mutagent/sdk/openai';
|
|
4837
|
-
|
|
4838
|
-
const openai = wrapOpenAI(
|
|
4839
|
-
new OpenAI({ apiKey: process.env.OPENAI_API_KEY }),
|
|
4840
|
-
{
|
|
4841
|
-
mutagentApiKey: process.env.MUTAGENT_API_KEY!,
|
|
4842
|
-
promptId: 'default',
|
|
4843
|
-
}
|
|
4844
|
-
);
|
|
4845
|
-
|
|
4846
|
-
// All calls are traced
|
|
4847
|
-
const completion = await openai.chat.completions.create({...});
|
|
4848
|
-
\`\`\`
|
|
4849
|
-
|
|
4850
|
-
## Verification
|
|
4851
|
-
|
|
4852
|
-
\`\`\`bash
|
|
4853
|
-
mutagent integrate generic --verify
|
|
4752
|
+
mutagent integrate openai --verify
|
|
4854
4753
|
\`\`\`
|
|
4855
4754
|
|
|
4856
|
-
##
|
|
4857
|
-
|
|
4858
|
-
\`\`\`bash
|
|
4859
|
-
# View traces
|
|
4860
|
-
mutagent traces list --prompt <prompt-id>
|
|
4755
|
+
## Documentation
|
|
4861
4756
|
|
|
4862
|
-
|
|
4863
|
-
|
|
4864
|
-
|
|
4757
|
+
- Full guide: https://docs.mutagent.io/integrations/openai
|
|
4758
|
+
- API Reference: https://docs.mutagent.io/sdk/tracing
|
|
4759
|
+
- Dashboard: https://app.mutagent.io
|
|
4865
4760
|
`;
|
|
4866
4761
|
}
|
|
4867
4762
|
};
|
|
4868
4763
|
|
|
4869
4764
|
// src/lib/integrations/registry.ts
|
|
4870
4765
|
var frameworkRegistry = new Map([
|
|
4871
|
-
["mastra", mastraIntegration],
|
|
4872
4766
|
["langchain", langchainIntegration],
|
|
4873
4767
|
["langgraph", langgraphIntegration],
|
|
4874
4768
|
["vercel-ai", vercelAiIntegration],
|
|
4875
|
-
["openai", openaiIntegration]
|
|
4876
|
-
["claude-code", claudeCodeIntegration],
|
|
4877
|
-
["generic", genericIntegration]
|
|
4769
|
+
["openai", openaiIntegration]
|
|
4878
4770
|
]);
|
|
4879
4771
|
var frameworkMetadata = [
|
|
4880
|
-
{
|
|
4881
|
-
name: "mastra",
|
|
4882
|
-
displayName: "Mastra",
|
|
4883
|
-
description: "Modern AI agent framework with observability built-in",
|
|
4884
|
-
npmPackage: "@mastra/core",
|
|
4885
|
-
website: "https://mastra.ai"
|
|
4886
|
-
},
|
|
4887
4772
|
{
|
|
4888
4773
|
name: "langchain",
|
|
4889
4774
|
displayName: "LangChain",
|
|
@@ -4897,7 +4782,7 @@ var frameworkMetadata = [
|
|
|
4897
4782
|
displayName: "LangGraph",
|
|
4898
4783
|
description: "Agent workflow framework built on LangChain",
|
|
4899
4784
|
npmPackage: "@langchain/langgraph",
|
|
4900
|
-
mutagentPackage: "@mutagent/
|
|
4785
|
+
mutagentPackage: "@mutagent/langchain",
|
|
4901
4786
|
website: "https://langchain-ai.github.io/langgraphjs/"
|
|
4902
4787
|
},
|
|
4903
4788
|
{
|
|
@@ -4915,20 +4800,6 @@ var frameworkMetadata = [
|
|
|
4915
4800
|
npmPackage: "openai",
|
|
4916
4801
|
mutagentPackage: "@mutagent/openai",
|
|
4917
4802
|
website: "https://platform.openai.com"
|
|
4918
|
-
},
|
|
4919
|
-
{
|
|
4920
|
-
name: "claude-code",
|
|
4921
|
-
displayName: "Claude Code SDK",
|
|
4922
|
-
description: "Native Anthropic integration for Claude",
|
|
4923
|
-
npmPackage: "@anthropic-ai/claude-code",
|
|
4924
|
-
website: "https://docs.anthropic.com"
|
|
4925
|
-
},
|
|
4926
|
-
{
|
|
4927
|
-
name: "generic",
|
|
4928
|
-
displayName: "Generic (OpenAI-compatible)",
|
|
4929
|
-
description: "Any OpenAI-compatible API endpoint",
|
|
4930
|
-
npmPackage: "openai",
|
|
4931
|
-
website: "https://platform.openai.com"
|
|
4932
4803
|
}
|
|
4933
4804
|
];
|
|
4934
4805
|
function getFramework(name) {
|
|
@@ -4940,16 +4811,16 @@ function getFrameworkMetadata(name) {
|
|
|
4940
4811
|
|
|
4941
4812
|
// src/commands/integrate.ts
|
|
4942
4813
|
function detectPackageManager() {
|
|
4943
|
-
if (
|
|
4814
|
+
if (existsSync9("bun.lockb") || existsSync9("bun.lock")) {
|
|
4944
4815
|
return "bun";
|
|
4945
4816
|
}
|
|
4946
|
-
if (
|
|
4817
|
+
if (existsSync9("pnpm-lock.yaml")) {
|
|
4947
4818
|
return "pnpm";
|
|
4948
4819
|
}
|
|
4949
|
-
if (
|
|
4820
|
+
if (existsSync9("yarn.lock")) {
|
|
4950
4821
|
return "yarn";
|
|
4951
4822
|
}
|
|
4952
|
-
if (
|
|
4823
|
+
if (existsSync9("package-lock.json")) {
|
|
4953
4824
|
return "npm";
|
|
4954
4825
|
}
|
|
4955
4826
|
try {
|
|
@@ -4978,8 +4849,6 @@ AI AGENT: Please determine the framework by:
|
|
|
4978
4849
|
- \`@langchain/langgraph\` -> use \`mutagent integrate langgraph\`
|
|
4979
4850
|
- \`ai\` (vercel) -> use \`mutagent integrate vercel-ai\`
|
|
4980
4851
|
- \`openai\` -> use \`mutagent integrate openai\`
|
|
4981
|
-
- \`@mastra/core\` -> use \`mutagent integrate mastra\`
|
|
4982
|
-
- \`@anthropic-ai/claude-code\` -> use \`mutagent integrate claude-code\`
|
|
4983
4852
|
|
|
4984
4853
|
### Option 2: Ask the user
|
|
4985
4854
|
"Which AI framework are you using?"
|
|
@@ -4987,9 +4856,6 @@ AI AGENT: Please determine the framework by:
|
|
|
4987
4856
|
- LangGraph
|
|
4988
4857
|
- Vercel AI SDK
|
|
4989
4858
|
- OpenAI SDK
|
|
4990
|
-
- Mastra
|
|
4991
|
-
- Claude Code
|
|
4992
|
-
- Generic (OpenAI-compatible)
|
|
4993
4859
|
|
|
4994
4860
|
### Next Steps
|
|
4995
4861
|
1. Once determined, run: \`mutagent integrate <framework>\`
|
|
@@ -5006,12 +4872,12 @@ Examples:
|
|
|
5006
4872
|
${chalk9.dim("$")} mutagent integrate langchain --output ./INTEGRATION.md
|
|
5007
4873
|
${chalk9.dim("$")} mutagent integrate --list # List available frameworks
|
|
5008
4874
|
|
|
5009
|
-
Frameworks:
|
|
4875
|
+
Frameworks: langchain, langgraph, vercel-ai, openai
|
|
5010
4876
|
|
|
5011
4877
|
This is a SKILL LOADER for AI coding agents (Claude Code, Cursor, etc.).
|
|
5012
4878
|
It returns INSTRUCTIONS that AI agents execute - it does NOT auto-install packages.
|
|
5013
4879
|
`);
|
|
5014
|
-
integrate.argument("[framework]", "Framework name (
|
|
4880
|
+
integrate.argument("[framework]", "Framework name (langchain, langgraph, vercel-ai, openai)").option("-o, --output <path>", "Save instructions to file").option("--raw", "Output raw markdown (no formatting)").option("--verify", "Verify integration after setup").action(async (frameworkArg, options) => {
|
|
5015
4881
|
const isJson = getJsonFlag(integrate);
|
|
5016
4882
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5017
4883
|
try {
|
|
@@ -5109,7 +4975,7 @@ AI AGENT: Execute these steps using Bash for install, Write/Edit for code change
|
|
|
5109
4975
|
init_sdk_client();
|
|
5110
4976
|
import { Command as Command6 } from "commander";
|
|
5111
4977
|
import chalk10 from "chalk";
|
|
5112
|
-
import { readFileSync as
|
|
4978
|
+
import { readFileSync as readFileSync9 } from "fs";
|
|
5113
4979
|
init_errors();
|
|
5114
4980
|
function createAgentsCommand() {
|
|
5115
4981
|
const agents = new Command6("agents").description("Manage AI agents").addHelpText("after", `
|
|
@@ -5220,20 +5086,48 @@ System Prompt:`));
|
|
|
5220
5086
|
handleError(error, isJson);
|
|
5221
5087
|
}
|
|
5222
5088
|
});
|
|
5223
|
-
agents.command("create").description("Create a new agent").option("-f, --file <path>", "Create from JSON file").option("-n, --name <name>", "Agent name").option("-s, --slug <slug>", "Agent slug (URL-friendly identifier)").option("-p, --system-prompt <prompt>", "System prompt").option("-m, --model <model>", "Model (claude-sonnet-4-5, claude-opus-4-5, claude-haiku-4-5)").option("
|
|
5089
|
+
agents.command("create").description("Create a new agent").option("-d, --data <json>", "Agent as JSON string (recommended for CI/scripts/agents)").option("-f, --file <path>", "Create from JSON file").option("-n, --name <name>", "Agent name").option("-s, --slug <slug>", "Agent slug (URL-friendly identifier)").option("-p, --system-prompt <prompt>", "System prompt").option("-m, --model <model>", "Model (claude-sonnet-4-5, claude-opus-4-5, claude-haiku-4-5)").option("--description <desc>", "Agent description").addHelpText("after", `
|
|
5224
5090
|
Examples:
|
|
5225
|
-
${chalk10.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "
|
|
5226
|
-
${chalk10.dim("$")} mutagent agents create
|
|
5227
|
-
${chalk10.dim("$")} mutagent agents create --
|
|
5091
|
+
${chalk10.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "You are a code reviewer..."
|
|
5092
|
+
${chalk10.dim("$")} mutagent agents create -d '{"name":"Code Reviewer","slug":"code-reviewer","systemPrompt":"You are a code reviewer..."}'
|
|
5093
|
+
${chalk10.dim("$")} mutagent agents create --file agent.json ${chalk10.dim("# full agent object as JSON file")}
|
|
5228
5094
|
|
|
5229
|
-
|
|
5095
|
+
Expected JSON (--data):
|
|
5096
|
+
${chalk10.dim('{"name":"<name>","slug":"<slug>","systemPrompt":"<system prompt>","model":"<model-id>","description":"<description>"}')}
|
|
5097
|
+
|
|
5098
|
+
Input Methods (pick one, priority order):
|
|
5099
|
+
--name/--slug/... Individual flags ${chalk10.green("(recommended)")}
|
|
5100
|
+
-d, --data Inline JSON object (CI/scripts/agents)
|
|
5101
|
+
--file Load from JSON file
|
|
5102
|
+
|
|
5103
|
+
${chalk10.yellow("Note: Prefer individual flags or --data over --file to avoid stale files living in your repo.")}
|
|
5104
|
+
${chalk10.red("Required: name, slug, systemPrompt.")} ${chalk10.dim("--data and --file are mutually exclusive. CLI flags override --data fields.")}
|
|
5230
5105
|
`).action(async (options) => {
|
|
5231
5106
|
const isJson = getJsonFlag(agents);
|
|
5232
5107
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5233
5108
|
try {
|
|
5234
5109
|
let data;
|
|
5235
|
-
if (options.file) {
|
|
5236
|
-
|
|
5110
|
+
if (options.data && options.file) {
|
|
5111
|
+
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --data and --file together", "Use --data for inline JSON or --file for file-based input, not both");
|
|
5112
|
+
}
|
|
5113
|
+
if (options.data) {
|
|
5114
|
+
try {
|
|
5115
|
+
data = JSON.parse(options.data);
|
|
5116
|
+
} catch {
|
|
5117
|
+
throw new MutagentError("INVALID_JSON", "Invalid JSON in --data flag", `Provide a valid JSON object, e.g., '{"name":"my-agent","slug":"my-agent","systemPrompt":"You are..."}'`);
|
|
5118
|
+
}
|
|
5119
|
+
if (options.name)
|
|
5120
|
+
data.name = options.name;
|
|
5121
|
+
if (options.slug)
|
|
5122
|
+
data.slug = options.slug;
|
|
5123
|
+
if (options.systemPrompt)
|
|
5124
|
+
data.systemPrompt = options.systemPrompt;
|
|
5125
|
+
if (options.model)
|
|
5126
|
+
data.model = options.model;
|
|
5127
|
+
if (options.description)
|
|
5128
|
+
data.description = options.description;
|
|
5129
|
+
} else if (options.file) {
|
|
5130
|
+
const content = readFileSync9(options.file, "utf-8");
|
|
5237
5131
|
data = JSON.parse(content);
|
|
5238
5132
|
} else if (options.name && options.slug && options.systemPrompt) {
|
|
5239
5133
|
data = {
|
|
@@ -5246,7 +5140,7 @@ ${chalk10.dim("Required: --name, --slug, and --system-prompt (or --file).")}
|
|
|
5246
5140
|
if (options.description)
|
|
5247
5141
|
data.description = options.description;
|
|
5248
5142
|
} else {
|
|
5249
|
-
throw new MutagentError("MISSING_ARGUMENTS", "Either --file or (--name, --slug, and --system-prompt) are required", "Use --file to load from JSON or provide --name, --slug, and --system-prompt");
|
|
5143
|
+
throw new MutagentError("MISSING_ARGUMENTS", "Either --data, --file, or (--name, --slug, and --system-prompt) are required", "Use --data for inline JSON, --file to load from JSON, or provide --name, --slug, and --system-prompt");
|
|
5250
5144
|
}
|
|
5251
5145
|
const client = getSDKClient();
|
|
5252
5146
|
const agent = await client.createAgent(data);
|
|
@@ -5256,18 +5150,46 @@ ${chalk10.dim("Required: --name, --slug, and --system-prompt (or --file).")}
|
|
|
5256
5150
|
handleError(error, isJson);
|
|
5257
5151
|
}
|
|
5258
5152
|
});
|
|
5259
|
-
agents.command("update").description("Update an agent").argument("<id>", "Agent ID").option("-f, --file <path>", "Update from JSON file").option("-n, --name <name>", "New name").option("-p, --system-prompt <prompt>", "New system prompt").option("-m, --model <model>", "New model").option("
|
|
5153
|
+
agents.command("update").description("Update an agent").argument("<id>", "Agent ID").option("-d, --data <json>", "Agent updates as JSON string (CI/scripts/agents)").option("-f, --file <path>", "Update from JSON file").option("-n, --name <name>", "New name").option("-p, --system-prompt <prompt>", "New system prompt").option("-m, --model <model>", "New model").option("--description <desc>", "New description").option("-s, --status <status>", "New status (active, paused, archived)").addHelpText("after", `
|
|
5260
5154
|
Examples:
|
|
5261
5155
|
${chalk10.dim("$")} mutagent agents update <id> --name "New Name"
|
|
5262
|
-
${chalk10.dim("$")} mutagent agents update <id> --status
|
|
5263
|
-
${chalk10.dim("$")} mutagent agents update <id>
|
|
5156
|
+
${chalk10.dim("$")} mutagent agents update <id> --system-prompt "Updated prompt" --status active
|
|
5157
|
+
${chalk10.dim("$")} mutagent agents update <id> -d '{"name":"New Name","systemPrompt":"Updated prompt"}'
|
|
5158
|
+
${chalk10.dim("$")} mutagent agents update <id> --file updated-agent.json ${chalk10.dim("# full agent object")}
|
|
5159
|
+
|
|
5160
|
+
Input Methods (pick one, priority order):
|
|
5161
|
+
--name/--system-prompt/... Individual flags ${chalk10.green("(recommended)")}
|
|
5162
|
+
-d, --data Inline JSON object (CI/scripts/agents)
|
|
5163
|
+
--file Load from JSON file
|
|
5164
|
+
|
|
5165
|
+
${chalk10.yellow("Note: Prefer individual flags or --data over --file to avoid stale files living in your repo.")}
|
|
5166
|
+
${chalk10.dim("--data and --file are mutually exclusive. CLI flags override --data fields.")}
|
|
5264
5167
|
`).action(async (id, options) => {
|
|
5265
5168
|
const isJson = getJsonFlag(agents);
|
|
5266
5169
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5267
5170
|
try {
|
|
5268
5171
|
let data = {};
|
|
5269
|
-
if (options.file) {
|
|
5270
|
-
|
|
5172
|
+
if (options.data && options.file) {
|
|
5173
|
+
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --data and --file together", "Use --data for inline JSON or --file for file-based input, not both");
|
|
5174
|
+
}
|
|
5175
|
+
if (options.data) {
|
|
5176
|
+
try {
|
|
5177
|
+
data = JSON.parse(options.data);
|
|
5178
|
+
} catch {
|
|
5179
|
+
throw new MutagentError("INVALID_JSON", "Invalid JSON in --data flag", `Provide a valid JSON object, e.g., '{"name":"new-name","systemPrompt":"Updated prompt"}'`);
|
|
5180
|
+
}
|
|
5181
|
+
if (options.name)
|
|
5182
|
+
data.name = options.name;
|
|
5183
|
+
if (options.systemPrompt)
|
|
5184
|
+
data.systemPrompt = options.systemPrompt;
|
|
5185
|
+
if (options.model)
|
|
5186
|
+
data.model = options.model;
|
|
5187
|
+
if (options.description)
|
|
5188
|
+
data.description = options.description;
|
|
5189
|
+
if (options.status)
|
|
5190
|
+
data.status = options.status;
|
|
5191
|
+
} else if (options.file) {
|
|
5192
|
+
const content = readFileSync9(options.file, "utf-8");
|
|
5271
5193
|
data = JSON.parse(content);
|
|
5272
5194
|
} else {
|
|
5273
5195
|
if (options.name)
|
|
@@ -5282,7 +5204,7 @@ Examples:
|
|
|
5282
5204
|
data.status = options.status;
|
|
5283
5205
|
}
|
|
5284
5206
|
if (Object.keys(data).length === 0) {
|
|
5285
|
-
throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --file, --name, --system-prompt, --model, --description, or --status");
|
|
5207
|
+
throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --data, --file, --name, --system-prompt, --model, --description, or --status");
|
|
5286
5208
|
}
|
|
5287
5209
|
const client = getSDKClient();
|
|
5288
5210
|
const agent = await client.updateAgent(id, data);
|
|
@@ -5323,12 +5245,15 @@ ${chalk10.dim("Tip: Use --force to skip confirmation (required for non-interacti
|
|
|
5323
5245
|
handleError(error, isJson);
|
|
5324
5246
|
}
|
|
5325
5247
|
});
|
|
5326
|
-
const conversations =
|
|
5248
|
+
const conversations = new Command6("conversations").description("Manage conversations for agents").addHelpText("after", `
|
|
5327
5249
|
Examples:
|
|
5328
5250
|
${chalk10.dim("$")} mutagent agents conversations list <agent-id>
|
|
5329
5251
|
${chalk10.dim("$")} mutagent agents conversations create <agent-id>
|
|
5330
5252
|
${chalk10.dim("$")} mutagent agents conversations messages <agent-id> <conversation-id>
|
|
5331
|
-
`)
|
|
5253
|
+
`).action(() => {
|
|
5254
|
+
conversations.help();
|
|
5255
|
+
});
|
|
5256
|
+
agents.addCommand(conversations);
|
|
5332
5257
|
conversations.command("list").description("List conversations for an agent").argument("<agent-id>", "Agent ID").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").addHelpText("after", `
|
|
5333
5258
|
Examples:
|
|
5334
5259
|
${chalk10.dim("$")} mutagent agents conversations list <agent-id>
|
|
@@ -5477,6 +5402,8 @@ Examples:
|
|
|
5477
5402
|
init_config();
|
|
5478
5403
|
import { Command as Command7 } from "commander";
|
|
5479
5404
|
import chalk11 from "chalk";
|
|
5405
|
+
init_errors();
|
|
5406
|
+
var VALID_CONFIG_KEYS = ["apiKey", "endpoint", "format", "timeout", "defaultWorkspace", "defaultOrganization"];
|
|
5480
5407
|
function createConfigCommand() {
|
|
5481
5408
|
const config = new Command7("config").description("Manage CLI configuration").addHelpText("after", `
|
|
5482
5409
|
Examples:
|
|
@@ -5509,21 +5436,27 @@ ${chalk11.dim("Keys: apiKey, endpoint, format, timeout, defaultWorkspace, defaul
|
|
|
5509
5436
|
`).action((key) => {
|
|
5510
5437
|
const isJson = getJsonFlag(config);
|
|
5511
5438
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
5512
|
-
|
|
5513
|
-
|
|
5514
|
-
|
|
5515
|
-
|
|
5516
|
-
|
|
5517
|
-
|
|
5518
|
-
|
|
5519
|
-
|
|
5520
|
-
|
|
5521
|
-
|
|
5522
|
-
|
|
5523
|
-
|
|
5439
|
+
try {
|
|
5440
|
+
const cfg = loadConfig();
|
|
5441
|
+
if (!(key in cfg)) {
|
|
5442
|
+
throw new MutagentError("INVALID_ARGUMENTS", `Unknown config key: "${key}"`, `Valid keys: ${VALID_CONFIG_KEYS.join(", ")}
|
|
5443
|
+
` + "Example: mutagent config get endpoint");
|
|
5444
|
+
}
|
|
5445
|
+
const value = cfg[key];
|
|
5446
|
+
const stringValue = typeof value === "object" ? JSON.stringify(value) : String(value ?? "");
|
|
5447
|
+
const displayValue = key === "apiKey" && value ? `${stringValue.slice(0, 8)}...` : stringValue;
|
|
5448
|
+
if (isJson) {
|
|
5449
|
+
output.output({ [key]: displayValue });
|
|
5450
|
+
} else {
|
|
5451
|
+
output.info(displayValue);
|
|
5452
|
+
}
|
|
5453
|
+
} catch (error) {
|
|
5454
|
+
handleError(error, isJson);
|
|
5524
5455
|
}
|
|
5525
5456
|
});
|
|
5526
|
-
const set = new Command7("set").description("Set configuration value")
|
|
5457
|
+
const set = new Command7("set").description("Set configuration value").action(() => {
|
|
5458
|
+
set.help();
|
|
5459
|
+
});
|
|
5527
5460
|
set.command("workspace").description("Set default workspace ID").argument("<id>", "Workspace ID to set as default").addHelpText("after", `
|
|
5528
5461
|
Examples:
|
|
5529
5462
|
${chalk11.dim("$")} mutagent config set workspace <workspace-id>
|
|
@@ -5562,7 +5495,7 @@ ${chalk11.dim("Persists organization ID for org-scoped API keys.")}
|
|
|
5562
5495
|
init_sdk_client();
|
|
5563
5496
|
import { Command as Command8 } from "commander";
|
|
5564
5497
|
import chalk12 from "chalk";
|
|
5565
|
-
import { readFileSync as
|
|
5498
|
+
import { readFileSync as readFileSync10 } from "fs";
|
|
5566
5499
|
init_errors();
|
|
5567
5500
|
function parseSSELine(line) {
|
|
5568
5501
|
if (!line || line.startsWith(":")) {
|
|
@@ -5615,12 +5548,13 @@ Examples:
|
|
|
5615
5548
|
${chalk12.dim("$")} mutagent playground run <prompt-id> --system "You are helpful" --human "Hello"
|
|
5616
5549
|
${chalk12.dim("$")} mutagent playground run <prompt-id> --input '{}' --model gpt-4-turbo --json
|
|
5617
5550
|
|
|
5618
|
-
Input Methods (pick one):
|
|
5551
|
+
Input Methods (pick one, priority order):
|
|
5552
|
+
--system/--human Quick system + user message ${chalk12.green("(recommended)")}
|
|
5619
5553
|
--input '{"key":"value"}' Inline JSON variables
|
|
5620
|
-
--file input.json Load from JSON file
|
|
5621
|
-
--system/--human Quick system + user message
|
|
5622
5554
|
--messages '[...]' Full messages array
|
|
5555
|
+
--file input.json Load from JSON file
|
|
5623
5556
|
|
|
5557
|
+
${chalk12.yellow("Note: Prefer --system/--human or --input over --file to avoid stale files living in your repo.")}
|
|
5624
5558
|
${chalk12.dim(`Hint: Test before evaluating: mutagent playground run <id> --input '{"key":"value"}'`)}
|
|
5625
5559
|
`).action(async (promptId, options) => {
|
|
5626
5560
|
const isJson = getJsonFlag(playground);
|
|
@@ -5724,7 +5658,7 @@ function parsePromptStyleInput(options) {
|
|
|
5724
5658
|
function getInputJson(options) {
|
|
5725
5659
|
if (options.file) {
|
|
5726
5660
|
try {
|
|
5727
|
-
return
|
|
5661
|
+
return readFileSync10(options.file, "utf-8");
|
|
5728
5662
|
} catch (err) {
|
|
5729
5663
|
throw new MutagentError("FILE_READ_ERROR", `Failed to read input file: ${options.file}`, err instanceof Error ? err.message : "Check the file path and permissions");
|
|
5730
5664
|
}
|
|
@@ -5952,6 +5886,10 @@ Subcommands:
|
|
|
5952
5886
|
list, get, test
|
|
5953
5887
|
|
|
5954
5888
|
Note: Provider management (create, update, delete) is available in the Admin Panel only.
|
|
5889
|
+
|
|
5890
|
+
${chalk14.yellow("Note:")} The providers module is not yet active. This is a placeholder
|
|
5891
|
+
for future external provider configuration. The server currently uses
|
|
5892
|
+
built-in provider settings.
|
|
5955
5893
|
`);
|
|
5956
5894
|
providers.command("list").description("List all providers").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").option("-t, --type <type>", "Filter by provider type").addHelpText("after", `
|
|
5957
5895
|
Examples:
|
|
@@ -5980,6 +5918,8 @@ Examples:
|
|
|
5980
5918
|
}));
|
|
5981
5919
|
output.output({ ...result, data: withLinks });
|
|
5982
5920
|
} else {
|
|
5921
|
+
console.log(chalk14.yellow("Note: The providers module is not yet active. This is a placeholder for future external provider configuration."));
|
|
5922
|
+
console.log("");
|
|
5983
5923
|
if (result.data.length === 0) {
|
|
5984
5924
|
output.info("No providers configured.");
|
|
5985
5925
|
output.info(`Configure at: ${providerSettingsLink()}`);
|
|
@@ -6013,6 +5953,8 @@ Examples:
|
|
|
6013
5953
|
if (isJson) {
|
|
6014
5954
|
output.output({ ...provider, _links: providerLinks(provider.id) });
|
|
6015
5955
|
} else {
|
|
5956
|
+
console.log(chalk14.yellow("Note: The providers module is not yet active. This is a placeholder for future external provider configuration."));
|
|
5957
|
+
console.log("");
|
|
6016
5958
|
const formatted = {
|
|
6017
5959
|
id: provider.id,
|
|
6018
5960
|
name: provider.name,
|
|
@@ -6042,6 +5984,8 @@ ${chalk14.dim("Tests connectivity and lists available models for the provider.")
|
|
|
6042
5984
|
try {
|
|
6043
5985
|
const client = getSDKClient();
|
|
6044
5986
|
if (!isJson) {
|
|
5987
|
+
console.log(chalk14.yellow("Note: The providers module is not yet active. This is a placeholder for future external provider configuration."));
|
|
5988
|
+
console.log("");
|
|
6045
5989
|
output.info(`Testing provider ${id}...`);
|
|
6046
5990
|
}
|
|
6047
5991
|
const result = await client.testProvider(id);
|
|
@@ -6077,7 +6021,7 @@ init_config();
|
|
|
6077
6021
|
import { Command as Command11 } from "commander";
|
|
6078
6022
|
import inquirer3 from "inquirer";
|
|
6079
6023
|
import chalk15 from "chalk";
|
|
6080
|
-
import { existsSync as
|
|
6024
|
+
import { existsSync as existsSync10, mkdirSync as mkdirSync3, readFileSync as readFileSync11, writeFileSync as writeFileSync4 } from "fs";
|
|
6081
6025
|
import { execSync as execSync2 } from "child_process";
|
|
6082
6026
|
import { join as join5 } from "path";
|
|
6083
6027
|
init_errors();
|
|
@@ -6130,16 +6074,16 @@ var FRAMEWORK_DETECTION_MAP = {
|
|
|
6130
6074
|
}
|
|
6131
6075
|
};
|
|
6132
6076
|
function detectPackageManager2(cwd = process.cwd()) {
|
|
6133
|
-
if (
|
|
6077
|
+
if (existsSync10(join5(cwd, "bun.lockb")) || existsSync10(join5(cwd, "bun.lock"))) {
|
|
6134
6078
|
return "bun";
|
|
6135
6079
|
}
|
|
6136
|
-
if (
|
|
6080
|
+
if (existsSync10(join5(cwd, "pnpm-lock.yaml"))) {
|
|
6137
6081
|
return "pnpm";
|
|
6138
6082
|
}
|
|
6139
|
-
if (
|
|
6083
|
+
if (existsSync10(join5(cwd, "yarn.lock"))) {
|
|
6140
6084
|
return "yarn";
|
|
6141
6085
|
}
|
|
6142
|
-
if (
|
|
6086
|
+
if (existsSync10(join5(cwd, "package-lock.json"))) {
|
|
6143
6087
|
return "npm";
|
|
6144
6088
|
}
|
|
6145
6089
|
try {
|
|
@@ -6161,12 +6105,12 @@ function getInstallCommand2(pm, packages) {
|
|
|
6161
6105
|
}
|
|
6162
6106
|
function detectFrameworkFromPackageJson(cwd = process.cwd()) {
|
|
6163
6107
|
const pkgPath = join5(cwd, "package.json");
|
|
6164
|
-
if (!
|
|
6108
|
+
if (!existsSync10(pkgPath)) {
|
|
6165
6109
|
return null;
|
|
6166
6110
|
}
|
|
6167
6111
|
let pkg;
|
|
6168
6112
|
try {
|
|
6169
|
-
pkg = JSON.parse(
|
|
6113
|
+
pkg = JSON.parse(readFileSync11(pkgPath, "utf-8"));
|
|
6170
6114
|
} catch {
|
|
6171
6115
|
return null;
|
|
6172
6116
|
}
|
|
@@ -6182,7 +6126,7 @@ function detectFrameworkFromPackageJson(cwd = process.cwd()) {
|
|
|
6182
6126
|
return null;
|
|
6183
6127
|
}
|
|
6184
6128
|
function hasRcConfig(cwd = process.cwd()) {
|
|
6185
|
-
return
|
|
6129
|
+
return existsSync10(join5(cwd, ".mutagentrc.json"));
|
|
6186
6130
|
}
|
|
6187
6131
|
function writeRcConfig(config, cwd = process.cwd()) {
|
|
6188
6132
|
const rcPath = join5(cwd, ".mutagentrc.json");
|
|
@@ -6350,6 +6294,32 @@ Modes:
|
|
|
6350
6294
|
output.info('Ready! Run "mutagent --help" to see available commands.');
|
|
6351
6295
|
}
|
|
6352
6296
|
}
|
|
6297
|
+
if (!isNonInteractive) {
|
|
6298
|
+
const skillPath = join5(cwd, ".claude/skills/mutagent-cli/SKILL.md");
|
|
6299
|
+
if (!existsSync10(skillPath)) {
|
|
6300
|
+
const { installSkill } = await inquirer3.prompt([{
|
|
6301
|
+
type: "confirm",
|
|
6302
|
+
name: "installSkill",
|
|
6303
|
+
message: "Install MutagenT skill for Claude Code? (Teaches AI agents how to use the CLI)",
|
|
6304
|
+
default: true
|
|
6305
|
+
}]);
|
|
6306
|
+
if (installSkill) {
|
|
6307
|
+
try {
|
|
6308
|
+
const skillDir = join5(cwd, ".claude/skills/mutagent-cli");
|
|
6309
|
+
if (!existsSync10(skillDir)) {
|
|
6310
|
+
mkdirSync3(skillDir, { recursive: true });
|
|
6311
|
+
}
|
|
6312
|
+
execSync2("node " + join5(cwd, "node_modules/.bin/mutagent") + " skills install", {
|
|
6313
|
+
cwd,
|
|
6314
|
+
stdio: "ignore"
|
|
6315
|
+
});
|
|
6316
|
+
output.success("Installed MutagenT CLI skill for Claude Code");
|
|
6317
|
+
} catch {
|
|
6318
|
+
output.info("Install skill manually: mutagent skills install");
|
|
6319
|
+
}
|
|
6320
|
+
}
|
|
6321
|
+
}
|
|
6322
|
+
}
|
|
6353
6323
|
if (isJson) {
|
|
6354
6324
|
output.output({
|
|
6355
6325
|
success: true,
|
|
@@ -6478,7 +6448,7 @@ Scanning ${scanPath}...
|
|
|
6478
6448
|
// src/commands/skills.ts
|
|
6479
6449
|
import { Command as Command13 } from "commander";
|
|
6480
6450
|
import chalk17 from "chalk";
|
|
6481
|
-
import { existsSync as
|
|
6451
|
+
import { existsSync as existsSync11, mkdirSync as mkdirSync4, writeFileSync as writeFileSync5 } from "fs";
|
|
6482
6452
|
import { join as join6 } from "path";
|
|
6483
6453
|
var SKILL_FRONTMATTER = `---
|
|
6484
6454
|
name: mutagent-cli
|
|
@@ -6493,6 +6463,22 @@ var SKILL_BODY = `# MutagenT CLI Skill
|
|
|
6493
6463
|
|
|
6494
6464
|
## Quick Reference
|
|
6495
6465
|
Run \`mutagent --help\` for full command list and examples.
|
|
6466
|
+
Run \`mutagent <command> --help\` for on-the-fly instructions before using any command.
|
|
6467
|
+
|
|
6468
|
+
Key commands:
|
|
6469
|
+
- \`mutagent auth login\` — Authenticate
|
|
6470
|
+
- \`mutagent prompts list\` — List prompts
|
|
6471
|
+
- \`mutagent usage\` — Check plan limits and remaining optimization runs
|
|
6472
|
+
- \`mutagent explore\` — Discover prompts in codebase
|
|
6473
|
+
- \`mutagent integrate <framework>\` — Framework integration instructions
|
|
6474
|
+
|
|
6475
|
+
## Agent Usage Guidelines
|
|
6476
|
+
|
|
6477
|
+
- **ALWAYS** run \`mutagent prompts list --json\` and \`mutagent usage --json\` BEFORE creating or modifying resources
|
|
6478
|
+
- **ALWAYS** use the \`--json\` flag for machine-readable output
|
|
6479
|
+
- **ALWAYS** confirm with the user before write operations (create, update, delete, optimize)
|
|
6480
|
+
- **CHECK** \`mutagent usage\` before running \`optimize start\` to verify remaining optimization runs
|
|
6481
|
+
- Format dashboard links as markdown: \`[View on Dashboard](url)\`
|
|
6496
6482
|
|
|
6497
6483
|
## Post-Onboarding Decision Tree
|
|
6498
6484
|
|
|
@@ -6511,8 +6497,9 @@ After \`mutagent auth login\`, the user lands in one of 3 paths:
|
|
|
6511
6497
|
3. \`mutagent prompts create --data '{...}'\` — Upload with REQUIRED outputSchema
|
|
6512
6498
|
4. Guided eval creator OR \`mutagent prompts evaluation create\`
|
|
6513
6499
|
5. \`mutagent prompts dataset add\` — Upload/curate datasets (named)
|
|
6514
|
-
6. \`mutagent
|
|
6515
|
-
7.
|
|
6500
|
+
6. \`mutagent usage --json\` — Verify remaining optimization runs
|
|
6501
|
+
7. \`mutagent prompts optimize start\` — Run optimization
|
|
6502
|
+
8. Review scorecard → Apply/Reject optimized prompt
|
|
6516
6503
|
|
|
6517
6504
|
### Path C: Manual
|
|
6518
6505
|
User uses CLI commands directly. Run \`mutagent --help\`.
|
|
@@ -6585,8 +6572,8 @@ that teaches coding agents how to use the MutagenT CLI effectively.
|
|
|
6585
6572
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
6586
6573
|
const skillDir = join6(process.cwd(), SKILL_DIR);
|
|
6587
6574
|
const skillPath = join6(skillDir, SKILL_FILE);
|
|
6588
|
-
if (!
|
|
6589
|
-
|
|
6575
|
+
if (!existsSync11(skillDir)) {
|
|
6576
|
+
mkdirSync4(skillDir, { recursive: true });
|
|
6590
6577
|
}
|
|
6591
6578
|
const content = `${SKILL_FRONTMATTER}
|
|
6592
6579
|
|
|
@@ -6610,16 +6597,120 @@ ${SKILL_BODY}
|
|
|
6610
6597
|
return skills;
|
|
6611
6598
|
}
|
|
6612
6599
|
|
|
6600
|
+
// src/commands/usage.ts
|
|
6601
|
+
init_config();
|
|
6602
|
+
import { Command as Command14 } from "commander";
|
|
6603
|
+
import chalk18 from "chalk";
|
|
6604
|
+
init_errors();
|
|
6605
|
+
init_sdk_client();
|
|
6606
|
+
var TRIAL_OPTIMIZATION_LIMIT = 5;
|
|
6607
|
+
var BILLING_URL = "https://app.mutagent.io/settings/billing";
|
|
6608
|
+
function renderProgressBar(used, limit, width = 30) {
|
|
6609
|
+
const ratio = Math.min(used / limit, 1);
|
|
6610
|
+
const filled = Math.round(ratio * width);
|
|
6611
|
+
const empty = width - filled;
|
|
6612
|
+
const bar = "█".repeat(filled) + "░".repeat(empty);
|
|
6613
|
+
const percent = Math.round(ratio * 100);
|
|
6614
|
+
return `${bar} ${String(percent)}%`;
|
|
6615
|
+
}
|
|
6616
|
+
function createUsageCommand() {
|
|
6617
|
+
const usage = new Command14("usage").description("Show resource counts and optimization run limits").addHelpText("after", `
|
|
6618
|
+
Examples:
|
|
6619
|
+
${chalk18.dim("$")} mutagent usage
|
|
6620
|
+
${chalk18.dim("$")} mutagent usage --json
|
|
6621
|
+
`);
|
|
6622
|
+
usage.action(async () => {
|
|
6623
|
+
const isJson = getJsonFlag(usage);
|
|
6624
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
6625
|
+
try {
|
|
6626
|
+
const apiKey = getApiKey();
|
|
6627
|
+
if (!apiKey) {
|
|
6628
|
+
throw new AuthenticationError;
|
|
6629
|
+
}
|
|
6630
|
+
const spinner = await createSpinner("Fetching usage data...", isJson).start();
|
|
6631
|
+
const client = getSDKClient();
|
|
6632
|
+
const prompts = await client.listPrompts();
|
|
6633
|
+
const promptCount = prompts.length;
|
|
6634
|
+
let datasetCount = 0;
|
|
6635
|
+
let evaluationCount = 0;
|
|
6636
|
+
if (promptCount > 0) {
|
|
6637
|
+
const results = await Promise.all(prompts.map(async (prompt) => {
|
|
6638
|
+
const id = String(prompt.id);
|
|
6639
|
+
const [datasets, evaluations] = await Promise.all([
|
|
6640
|
+
client.listDatasets(id).catch(() => []),
|
|
6641
|
+
client.listEvaluations(id).catch(() => [])
|
|
6642
|
+
]);
|
|
6643
|
+
return { datasets: datasets.length, evaluations: evaluations.length };
|
|
6644
|
+
}));
|
|
6645
|
+
for (const r of results) {
|
|
6646
|
+
datasetCount += r.datasets;
|
|
6647
|
+
evaluationCount += r.evaluations;
|
|
6648
|
+
}
|
|
6649
|
+
}
|
|
6650
|
+
const optimizationLimit = TRIAL_OPTIMIZATION_LIMIT;
|
|
6651
|
+
const optimizationUsed = 0;
|
|
6652
|
+
const optimizationRemaining = optimizationLimit - optimizationUsed;
|
|
6653
|
+
if (spinner && typeof spinner.stop === "function") {
|
|
6654
|
+
spinner.stop();
|
|
6655
|
+
}
|
|
6656
|
+
if (isJson) {
|
|
6657
|
+
output.output({
|
|
6658
|
+
resources: {
|
|
6659
|
+
prompts: promptCount,
|
|
6660
|
+
datasets: datasetCount,
|
|
6661
|
+
evaluations: evaluationCount
|
|
6662
|
+
},
|
|
6663
|
+
optimizationRuns: {
|
|
6664
|
+
used: optimizationUsed,
|
|
6665
|
+
limit: optimizationLimit,
|
|
6666
|
+
remaining: optimizationRemaining,
|
|
6667
|
+
plan: "trial"
|
|
6668
|
+
},
|
|
6669
|
+
_links: {
|
|
6670
|
+
billing: BILLING_URL
|
|
6671
|
+
}
|
|
6672
|
+
});
|
|
6673
|
+
} else {
|
|
6674
|
+
console.log("");
|
|
6675
|
+
console.log(chalk18.bold("\uD83D\uDCCA MutagenT Usage"));
|
|
6676
|
+
console.log(chalk18.dim("─".repeat(45)));
|
|
6677
|
+
console.log("");
|
|
6678
|
+
console.log(chalk18.bold("Resources:"));
|
|
6679
|
+
console.log(` Prompts: ${chalk18.cyan(String(promptCount))}`);
|
|
6680
|
+
console.log(` Datasets: ${chalk18.cyan(String(datasetCount))}`);
|
|
6681
|
+
console.log(` Evaluations: ${chalk18.cyan(String(evaluationCount))}`);
|
|
6682
|
+
console.log("");
|
|
6683
|
+
console.log(chalk18.bold(`Optimization Runs (${chalk18.yellow("trial")} plan):`));
|
|
6684
|
+
console.log(` Remaining: ${chalk18.cyan(String(optimizationRemaining))} / ${String(optimizationLimit)}`);
|
|
6685
|
+
console.log(` ${renderProgressBar(optimizationUsed, optimizationLimit)}`);
|
|
6686
|
+
console.log("");
|
|
6687
|
+
console.log(chalk18.yellow(` ⚠ ${String(optimizationRemaining)} optimization runs remaining`));
|
|
6688
|
+
console.log(chalk18.dim(` ℹ Optimization run counts are approximate`));
|
|
6689
|
+
console.log(` Upgrade: ${chalk18.underline(BILLING_URL)}`);
|
|
6690
|
+
console.log("");
|
|
6691
|
+
}
|
|
6692
|
+
} catch (error) {
|
|
6693
|
+
handleError(error, isJson);
|
|
6694
|
+
}
|
|
6695
|
+
});
|
|
6696
|
+
return usage;
|
|
6697
|
+
}
|
|
6698
|
+
|
|
6613
6699
|
// src/bin/cli.ts
|
|
6614
6700
|
init_config();
|
|
6701
|
+
import { existsSync as existsSync12 } from "fs";
|
|
6615
6702
|
var cliVersion = "0.1.1";
|
|
6616
|
-
|
|
6617
|
-
|
|
6618
|
-
|
|
6619
|
-
|
|
6620
|
-
|
|
6621
|
-
|
|
6622
|
-
|
|
6703
|
+
if (process.env.CLI_VERSION) {
|
|
6704
|
+
cliVersion = process.env.CLI_VERSION;
|
|
6705
|
+
} else {
|
|
6706
|
+
try {
|
|
6707
|
+
const __dirname2 = dirname(fileURLToPath(import.meta.url));
|
|
6708
|
+
const pkgPath = join7(__dirname2, "..", "..", "package.json");
|
|
6709
|
+
const pkg = JSON.parse(readFileSync12(pkgPath, "utf-8"));
|
|
6710
|
+
cliVersion = pkg.version ?? cliVersion;
|
|
6711
|
+
} catch {}
|
|
6712
|
+
}
|
|
6713
|
+
var program = new Command15;
|
|
6623
6714
|
program.name("mutagent").description(`MutagenT CLI - AI-native prompt optimization platform
|
|
6624
6715
|
|
|
6625
6716
|
Documentation: https://docs.mutagent.io/cli
|
|
@@ -6628,45 +6719,47 @@ program.name("mutagent").description(`MutagenT CLI - AI-native prompt optimizati
|
|
|
6628
6719
|
showGlobalOptions: true
|
|
6629
6720
|
});
|
|
6630
6721
|
program.addHelpText("after", `
|
|
6631
|
-
${
|
|
6632
|
-
Export your API key: ${
|
|
6633
|
-
Or pass inline: ${
|
|
6634
|
-
Machine-readable output: ${
|
|
6635
|
-
Disable prompts: ${
|
|
6636
|
-
Set default workspace: ${
|
|
6637
|
-
Set default org: ${
|
|
6722
|
+
${chalk19.yellow("Non-Interactive Mode (CI/CD & Coding Agents):")}
|
|
6723
|
+
Export your API key: ${chalk19.green("export MUTAGENT_API_KEY=mt_your_key_here")}
|
|
6724
|
+
Or pass inline: ${chalk19.green("mutagent prompts list --api-key mt_your_key")}
|
|
6725
|
+
Machine-readable output: ${chalk19.green("mutagent prompts list --json")}
|
|
6726
|
+
Disable prompts: ${chalk19.green("mutagent prompts list --non-interactive")}
|
|
6727
|
+
Set default workspace: ${chalk19.green("mutagent config set workspace <workspace-id>")}
|
|
6728
|
+
Set default org: ${chalk19.green("mutagent config set org <org-id>")}
|
|
6638
6729
|
`);
|
|
6639
6730
|
program.addHelpText("after", `
|
|
6640
|
-
${
|
|
6641
|
-
${
|
|
6731
|
+
${chalk19.yellow("Workflows:")}
|
|
6732
|
+
${chalk19.bold("Evaluate → Optimize Loop:")}
|
|
6642
6733
|
1. mutagent prompts create --name "..." --raw-file prompt.txt
|
|
6643
6734
|
2. mutagent prompts dataset add <prompt-id> --name "..." --file data.json
|
|
6644
6735
|
3. mutagent prompts evaluation create <prompt-id> --name "..." --file criteria.json
|
|
6645
6736
|
4. mutagent prompts optimize start <prompt-id> --dataset <id> --max-iterations 3
|
|
6646
6737
|
|
|
6647
|
-
${
|
|
6738
|
+
${chalk19.bold("Quick Test:")}
|
|
6648
6739
|
mutagent playground run <prompt-id> --input '{"key":"value"}'
|
|
6649
6740
|
|
|
6650
|
-
${
|
|
6651
|
-
${
|
|
6652
|
-
${
|
|
6653
|
-
${
|
|
6654
|
-
${
|
|
6741
|
+
${chalk19.bold("Prerequisites for Optimization:")}
|
|
6742
|
+
${chalk19.green("✓")} Prompt with input/output parameters
|
|
6743
|
+
${chalk19.green("✓")} Dataset with items (input + expectedOutput pairs)
|
|
6744
|
+
${chalk19.green("✓")} Evaluation with criteria (field-level, input/output focused)
|
|
6745
|
+
${chalk19.dim("•")} LLM provider ${chalk19.dim("(only when server uses external providers)")}
|
|
6655
6746
|
`);
|
|
6656
6747
|
program.addHelpText("after", `
|
|
6657
|
-
${
|
|
6658
|
-
${
|
|
6659
|
-
${
|
|
6660
|
-
${
|
|
6661
|
-
${
|
|
6662
|
-
${
|
|
6663
|
-
${
|
|
6664
|
-
${
|
|
6665
|
-
${
|
|
6666
|
-
${!hasCredentials() ?
|
|
6748
|
+
${chalk19.cyan("┌─ AI AGENT INTEGRATION HINT ────────────────────────────────────────────────┐")}
|
|
6749
|
+
${chalk19.cyan("│")} ${chalk19.cyan("│")}
|
|
6750
|
+
${chalk19.cyan("│")} Frameworks: langchain, langgraph, vercel-ai, openai ${chalk19.cyan("│")}
|
|
6751
|
+
${chalk19.cyan("│")} ${chalk19.cyan("│")}
|
|
6752
|
+
${chalk19.cyan("│")} Get integration guide: mutagent integrate <framework> ${chalk19.cyan("│")}
|
|
6753
|
+
${chalk19.cyan("│")} Verify setup: mutagent integrate <framework> --verify ${chalk19.cyan("│")}
|
|
6754
|
+
${chalk19.cyan("│")} Use --json for AI parsing: mutagent <command> --json ${chalk19.cyan("│")}
|
|
6755
|
+
${chalk19.cyan("│")} ${chalk19.cyan("│")}
|
|
6756
|
+
${chalk19.cyan("└────────────────────────────────────────────────────────────────────────────┘")}
|
|
6757
|
+
${!hasCredentials() ? chalk19.yellow(`
|
|
6667
6758
|
Warning: Not authenticated. Run: mutagent auth login --browser
|
|
6668
|
-
`) : ""}${!hasRcConfig() ?
|
|
6759
|
+
`) : ""}${!hasRcConfig() ? chalk19.green(`
|
|
6669
6760
|
Get started: mutagent init
|
|
6761
|
+
`) : ""}${!existsSync12(join7(process.cwd(), ".claude/skills/mutagent-cli/SKILL.md")) ? chalk19.magenta(`
|
|
6762
|
+
Using Claude Code? Install the MutagenT skill: mutagent skills install
|
|
6670
6763
|
`) : ""}`);
|
|
6671
6764
|
program.hook("preAction", (thisCommand) => {
|
|
6672
6765
|
const globalOpts = thisCommand.optsWithGlobals();
|
|
@@ -6693,7 +6786,8 @@ program.addCommand(createWorkspacesCommand());
|
|
|
6693
6786
|
program.addCommand(createProvidersCommand());
|
|
6694
6787
|
program.addCommand(createExploreCommand());
|
|
6695
6788
|
program.addCommand(createSkillsCommand());
|
|
6789
|
+
program.addCommand(createUsageCommand());
|
|
6696
6790
|
program.parse();
|
|
6697
6791
|
|
|
6698
|
-
//# debugId=
|
|
6792
|
+
//# debugId=88B4F3961CA6D43E64756E2164756E21
|
|
6699
6793
|
//# sourceMappingURL=cli.js.map
|