@mutagent/cli 0.1.15 → 0.1.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/bin/cli.js CHANGED
@@ -464,6 +464,7 @@ class SDKClientWrapper {
464
464
  systemPrompt: data.systemPrompt ?? undefined,
465
465
  humanPrompt: data.humanPrompt ?? undefined,
466
466
  rawPrompt: data.rawPrompt ?? undefined,
467
+ inputSchema: data.inputSchema,
467
468
  outputSchema: data.outputSchema,
468
469
  metadata: data.metadata,
469
470
  tags: data.tags ?? undefined
@@ -529,11 +530,13 @@ class SDKClientWrapper {
529
530
  labels: item.labels,
530
531
  metadata: item.metadata
531
532
  }));
533
+ const missingExpectedOutputCount = mappedItems.filter((item) => item.expectedOutput === undefined).length;
532
534
  await this.request(`/api/prompts/datasets/${String(dataset.id)}/items/bulk`, {
533
535
  method: "POST",
534
536
  body: JSON.stringify({ items: mappedItems })
535
537
  });
536
538
  itemCount = mappedItems.length;
539
+ return { ...dataset, itemCount, missingExpectedOutputCount };
537
540
  }
538
541
  return { ...dataset, itemCount };
539
542
  } catch (error) {
@@ -574,6 +577,13 @@ class SDKClientWrapper {
574
577
  this.handleError(error);
575
578
  }
576
579
  }
580
+ async deleteEvaluation(evaluationId) {
581
+ try {
582
+ await this.request(`/api/prompts/evaluations/${evaluationId}`, { method: "DELETE" });
583
+ } catch (error) {
584
+ this.handleError(error);
585
+ }
586
+ }
577
587
  async getEvaluationResults(runId) {
578
588
  try {
579
589
  return await this.request(`/api/prompts/evaluations/${runId}/result`);
@@ -588,8 +598,8 @@ class SDKClientWrapper {
588
598
  body: JSON.stringify({
589
599
  datasetId: parseInt(datasetId, 10),
590
600
  config: {
591
- maxIterations: config?.maxIterations ?? 3,
592
- targetScore: config?.targetScore,
601
+ maxIterations: config?.maxIterations ?? 1,
602
+ targetScore: config?.targetScore ?? 0.8,
593
603
  patience: config?.patience,
594
604
  model: config?.model,
595
605
  ...config?.evalModel ? { tuningParams: { evaluationModel: config.evalModel } } : {}
@@ -849,6 +859,12 @@ class SDKClientWrapper {
849
859
  this.handleError(error);
850
860
  }
851
861
  }
862
+ getCurrentWorkspaceId() {
863
+ return this.workspaceId;
864
+ }
865
+ getCurrentOrgId() {
866
+ return this.organizationId;
867
+ }
852
868
  async testProvider(id) {
853
869
  try {
854
870
  const response = await this.sdk.providerConfigs.testProvider({
@@ -922,9 +938,9 @@ var init_sdk_client = __esm(() => {
922
938
  });
923
939
 
924
940
  // src/bin/cli.ts
925
- import { Command as Command14 } from "commander";
926
- import chalk18 from "chalk";
927
- import { readFileSync as readFileSync14 } from "fs";
941
+ import { Command as Command15 } from "commander";
942
+ import chalk19 from "chalk";
943
+ import { readFileSync as readFileSync12 } from "fs";
928
944
  import { join as join7, dirname } from "path";
929
945
  import { fileURLToPath } from "url";
930
946
 
@@ -1058,6 +1074,24 @@ ${String(data.length)} result(s)`));
1058
1074
  }
1059
1075
  }
1060
1076
  }
1077
+ function createSpinner(text, isJson) {
1078
+ if (isJson) {
1079
+ const noop = () => {
1080
+ return;
1081
+ };
1082
+ return {
1083
+ start: noop,
1084
+ stop: noop,
1085
+ succeed: noop,
1086
+ fail: noop
1087
+ };
1088
+ }
1089
+ const start = async () => {
1090
+ const { default: ora } = await import("ora");
1091
+ return ora(text).start();
1092
+ };
1093
+ return { start };
1094
+ }
1061
1095
 
1062
1096
  // src/commands/auth.ts
1063
1097
  init_errors();
@@ -1779,12 +1813,10 @@ async function runIntegrationPath() {
1779
1813
  name: "framework",
1780
1814
  message: "Which AI framework are you using?",
1781
1815
  choices: [
1782
- { name: "Mastra", value: "mastra" },
1783
1816
  { name: "LangChain", value: "langchain" },
1784
1817
  { name: "LangGraph", value: "langgraph" },
1785
1818
  { name: "Vercel AI SDK", value: "vercel-ai" },
1786
- { name: "Claude Code", value: "claude-code" },
1787
- { name: "Other / Generic", value: "generic" }
1819
+ { name: "OpenAI SDK", value: "openai" }
1788
1820
  ]
1789
1821
  }]);
1790
1822
  console.log("");
@@ -2419,17 +2451,39 @@ function isValidJsonSchema(schema) {
2419
2451
  function buildSchemaFromVariables(variables) {
2420
2452
  const properties = {};
2421
2453
  for (const variable of variables) {
2422
- const prop = { type: variable.type };
2423
- if (variable.description) {
2424
- prop.description = variable.description;
2425
- }
2426
- properties[variable.name] = prop;
2454
+ properties[variable.name] = {
2455
+ type: variable.type,
2456
+ description: variable.description
2457
+ };
2427
2458
  }
2428
2459
  return {
2429
2460
  type: "object",
2430
2461
  properties
2431
2462
  };
2432
2463
  }
2464
+ function validateSchemaDescriptions(schema) {
2465
+ if (schema === null || schema === undefined)
2466
+ return [];
2467
+ if (typeof schema !== "object" || Array.isArray(schema))
2468
+ return [];
2469
+ const obj = schema;
2470
+ const properties = obj.properties;
2471
+ if (!properties || typeof properties !== "object" || Array.isArray(properties))
2472
+ return [];
2473
+ const missing = [];
2474
+ const props = properties;
2475
+ for (const [key, value] of Object.entries(props)) {
2476
+ if (value === null || value === undefined || typeof value !== "object") {
2477
+ missing.push(key);
2478
+ continue;
2479
+ }
2480
+ const prop = value;
2481
+ if (typeof prop.description !== "string" || prop.description.trim() === "") {
2482
+ missing.push(key);
2483
+ }
2484
+ }
2485
+ return missing;
2486
+ }
2433
2487
  function formatSchemaWarning(fieldName) {
2434
2488
  return [
2435
2489
  `${fieldName} doesn't appear to be a valid JSON Schema.`,
@@ -2534,9 +2588,17 @@ async function runGuidedEvalCreator(promptId) {
2534
2588
  }
2535
2589
  }]);
2536
2590
  let targetField;
2537
- if (allFields.length > 0) {
2591
+ const usedParams = new Set(criteria.map((c) => c.evaluationParameter));
2592
+ const availableFields = allFields.filter((f) => {
2593
+ const param = f.startsWith("output.") ? f.slice("output.".length) : f.startsWith("input.") ? f.slice("input.".length) : f;
2594
+ return !usedParams.has(param);
2595
+ });
2596
+ if (availableFields.length === 0 && allFields.length > 0) {
2597
+ console.log(chalk5.yellow(" All detected schema fields are already used by existing criteria."));
2598
+ }
2599
+ if (availableFields.length > 0) {
2538
2600
  const fieldChoices = [
2539
- ...allFields.map((f) => ({ name: f, value: f })),
2601
+ ...availableFields.map((f) => ({ name: f, value: f })),
2540
2602
  { name: "(custom field name)", value: "__custom__" }
2541
2603
  ];
2542
2604
  const { field } = await inquirer3.prompt([{
@@ -2593,10 +2655,15 @@ async function runGuidedEvalCreator(promptId) {
2593
2655
  } else {
2594
2656
  scoringRubric = rubric;
2595
2657
  }
2658
+ const evaluationParameter = targetField.startsWith("output.") ? targetField.slice("output.".length) : targetField.startsWith("input.") ? targetField.slice("input.".length) : targetField;
2659
+ if (criteria.some((c) => c.evaluationParameter === evaluationParameter)) {
2660
+ console.log(chalk5.red(` Error: "${evaluationParameter}" is already used by another criterion. Each criterion must target a unique output field.`));
2661
+ continue;
2662
+ }
2596
2663
  criteria.push({
2597
2664
  name: criterionName.trim(),
2598
- targetField,
2599
- scoringRubric,
2665
+ description: scoringRubric,
2666
+ evaluationParameter,
2600
2667
  weight: 1
2601
2668
  });
2602
2669
  const { continueAdding } = await inquirer3.prompt([{
@@ -2618,7 +2685,7 @@ async function runGuidedEvalCreator(promptId) {
2618
2685
  console.log(` Type: ${evalType}`);
2619
2686
  console.log(` Criteria: ${String(criteria.length)}`);
2620
2687
  for (const c of criteria) {
2621
- console.log(` - ${chalk5.cyan(c.name)} → ${c.targetField}`);
2688
+ console.log(` - ${chalk5.cyan(c.name)} → ${c.evaluationParameter}`);
2622
2689
  }
2623
2690
  console.log("");
2624
2691
  const { confirmed } = await inquirer3.prompt([{
@@ -2691,6 +2758,38 @@ function renderScorecard(data) {
2691
2758
  console.log(line(` Prompt: ${chalk6.cyan(truncateText(optimizedText, 38))}`));
2692
2759
  console.log(line(` Score: ${formatScore(bestScore)}${formatScoreChange(originalScore, bestScore)}`));
2693
2760
  console.log(separator);
2761
+ if (data.criteriaScores && data.criteriaScores.length > 0) {
2762
+ console.log(line(chalk6.dim(" Criterion Before After Change")));
2763
+ console.log(line(chalk6.dim(" " + "─".repeat(45))));
2764
+ for (const c of data.criteriaScores) {
2765
+ const name = c.name.length > 16 ? c.name.substring(0, 13) + "..." : c.name;
2766
+ const paddedName = name + " ".repeat(18 - name.length);
2767
+ const beforeStr = c.before !== undefined ? c.before.toFixed(2) : "N/A ";
2768
+ const afterStr = c.after !== undefined ? c.after.toFixed(2) : "N/A ";
2769
+ const changeStr = c.before !== undefined && c.after !== undefined && c.before > 0 ? (() => {
2770
+ const pct = Math.round((c.after - c.before) / c.before * 100);
2771
+ if (pct > 0)
2772
+ return chalk6.green(`+${String(pct)}%`);
2773
+ if (pct < 0)
2774
+ return chalk6.red(`${String(pct)}%`);
2775
+ return chalk6.dim("0%");
2776
+ })() : "";
2777
+ console.log(line(` ${paddedName}${beforeStr} ${afterStr} ${changeStr}`));
2778
+ }
2779
+ console.log(line(chalk6.dim(" " + "─".repeat(45))));
2780
+ const overallBefore = originalScore !== undefined ? originalScore.toFixed(2) : "N/A ";
2781
+ const overallAfter = bestScore !== undefined ? bestScore.toFixed(2) : "N/A ";
2782
+ const overallChange = originalScore !== undefined && bestScore !== undefined && originalScore > 0 ? (() => {
2783
+ const pct = Math.round((bestScore - originalScore) / originalScore * 100);
2784
+ if (pct > 0)
2785
+ return chalk6.green(`+${String(pct)}%`);
2786
+ if (pct < 0)
2787
+ return chalk6.red(`${String(pct)}%`);
2788
+ return chalk6.dim("0%");
2789
+ })() : "";
2790
+ console.log(line(` ${"Overall" + " ".repeat(11)}${overallBefore} ${overallAfter} ${overallChange}`));
2791
+ console.log(separator);
2792
+ }
2694
2793
  const statusStr = job.status === "completed" ? chalk6.green("completed") : chalk6.yellow(job.status);
2695
2794
  console.log(line(`Status: ${statusStr} | Iterations: ${String(iterations)}`));
2696
2795
  if (job.config?.model) {
@@ -2699,19 +2798,16 @@ function renderScorecard(data) {
2699
2798
  if (data.scoreProgression && data.scoreProgression.length > 0) {
2700
2799
  console.log(line(""));
2701
2800
  console.log(line(chalk6.dim("Score Progression:")));
2702
- const progression = data.scoreProgression.map((s, i) => `#${String(i + 1)}: ${s.toFixed(2)}`).join(" ");
2703
- if (progression.length > boxWidth - 4) {
2704
- const mid = Math.ceil(data.scoreProgression.length / 2);
2705
- const line1 = data.scoreProgression.slice(0, mid).map((s, i) => `#${String(i + 1)}: ${s.toFixed(2)}`).join(" ");
2706
- const line2 = data.scoreProgression.slice(mid).map((s, i) => `#${String(i + mid + 1)}: ${s.toFixed(2)}`).join(" ");
2707
- console.log(line(chalk6.dim(line1)));
2708
- console.log(line(chalk6.dim(line2)));
2709
- } else {
2710
- console.log(line(chalk6.dim(progression)));
2801
+ const barWidth = 10;
2802
+ for (let i = 0;i < data.scoreProgression.length; i++) {
2803
+ const s = data.scoreProgression[i] ?? 0;
2804
+ const filled = Math.round(s * barWidth);
2805
+ const bar = "█".repeat(filled) + "░".repeat(barWidth - filled);
2806
+ console.log(line(chalk6.dim(` #${String(i + 1)}: ${bar} ${s.toFixed(2)}`)));
2711
2807
  }
2712
2808
  }
2713
2809
  console.log(separator);
2714
- console.log(line(`Dashboard: ${chalk6.underline(optimizerLink(job.jobId))}`));
2810
+ console.log(line(`Dashboard: ${chalk6.underline(optimizerLink(job.id))}`));
2715
2811
  console.log(bottomBorder);
2716
2812
  console.log("");
2717
2813
  }
@@ -2747,7 +2843,7 @@ function updateMutationContext(updater) {
2747
2843
  } catch {}
2748
2844
  }
2749
2845
  var PREREQUISITES_TEXT = `
2750
- ${chalk7.yellow("Prerequisites:")}
2846
+ ${chalk7.red("Prerequisites (required):")}
2751
2847
  1. Evaluation criteria defined ${chalk7.dim("(via dashboard or evaluation create)")}
2752
2848
  2. Dataset uploaded ${chalk7.dim("mutagent prompts dataset list <prompt-id>")}
2753
2849
  ${chalk7.dim("Note: LLM provider config is only required when the server uses external providers (USE_EXT_PROVIDERS=true)")}`;
@@ -2780,18 +2876,6 @@ function isSchemaEmpty(schema) {
2780
2876
  return true;
2781
2877
  return false;
2782
2878
  }
2783
- function warnMissingSchemas(data, output) {
2784
- if (isSchemaEmpty(data.inputSchema)) {
2785
- output.warn("No inputSchema provided. Optimization requires inputSchema with defined variables.");
2786
- } else if (!isValidJsonSchema(data.inputSchema)) {
2787
- output.warn(formatSchemaWarning("inputSchema"));
2788
- }
2789
- if (isSchemaEmpty(data.outputSchema)) {
2790
- output.warn("No outputSchema provided. This may limit optimization effectiveness.");
2791
- } else if (!isValidJsonSchema(data.outputSchema)) {
2792
- output.warn(formatSchemaWarning("outputSchema"));
2793
- }
2794
- }
2795
2879
  async function collectSchemaInteractively() {
2796
2880
  const inquirer3 = (await import("inquirer")).default;
2797
2881
  const variables = [];
@@ -2820,13 +2904,18 @@ async function collectSchemaInteractively() {
2820
2904
  {
2821
2905
  type: "input",
2822
2906
  name: "description",
2823
- message: "Description (optional):"
2907
+ message: "Description (required):",
2908
+ validate: (input) => {
2909
+ if (!input.trim())
2910
+ return "Variable description is required";
2911
+ return true;
2912
+ }
2824
2913
  }
2825
2914
  ]);
2826
2915
  variables.push({
2827
2916
  name: answers.name.trim(),
2828
2917
  type: answers.type,
2829
- description: answers.description.trim() || undefined
2918
+ description: answers.description.trim()
2830
2919
  });
2831
2920
  const continueAnswer = await inquirer3.prompt([{
2832
2921
  type: "confirm",
@@ -2938,7 +3027,7 @@ Examples:
2938
3027
  Subcommands:
2939
3028
  list, get, create, update, delete
2940
3029
  dataset list|add|remove
2941
- evaluation list|create|results
3030
+ evaluation list|create|delete|results
2942
3031
  optimize start|status|results
2943
3032
  `);
2944
3033
  prompts.command("list").description("List all prompts").option("-l, --limit <n>", "Limit results", "50").addHelpText("after", `
@@ -2963,14 +3052,22 @@ ${chalk7.dim("Tip: Use --json for machine-readable output (AI agents, CI pipelin
2963
3052
  }));
2964
3053
  output.output(withLinks);
2965
3054
  } else {
2966
- const formatted = limited.map((p) => ({
2967
- id: p.id,
2968
- name: p.name,
2969
- version: p.version,
2970
- updated: p.updatedAt ? new Date(p.updatedAt).toLocaleDateString() : "N/A",
2971
- url: promptLink(p.id)
2972
- }));
2973
- output.output(formatted);
3055
+ const wsId = client.getCurrentWorkspaceId();
3056
+ if (wsId) {
3057
+ output.info(`Workspace: ${wsId}`);
3058
+ }
3059
+ if (limited.length === 0) {
3060
+ output.info("No prompts found in current workspace. Run 'mutagent config list' to check workspace context.");
3061
+ } else {
3062
+ const formatted = limited.map((p) => ({
3063
+ id: p.id,
3064
+ name: p.name,
3065
+ version: p.version,
3066
+ updated: p.updatedAt ? new Date(p.updatedAt).toLocaleDateString() : "N/A",
3067
+ url: promptLink(p.id)
3068
+ }));
3069
+ output.output(formatted);
3070
+ }
2974
3071
  }
2975
3072
  } catch (error) {
2976
3073
  handleError(error, isJson);
@@ -3008,20 +3105,24 @@ ${chalk7.dim("Tip: Combine --with-datasets and --with-evals to fetch all nested
3008
3105
  });
3009
3106
  prompts.command("create").description("Create a new prompt").option("-d, --data <json>", "Prompt as JSON string (recommended — curl-style inline)").option("-f, --file <path>", "Create from JSON file").option("--raw-file <path>", "Create from plain text file (used as rawPrompt)").option("-n, --name <name>", "Prompt name").option("-c, --content <content>", "Prompt content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).option("--output-schema <json>", "Output schema as JSON string (required for optimization)").addHelpText("after", `
3010
3107
  Examples:
3011
- ${chalk7.dim("$")} mutagent prompts create -d '{"name":"summarizer","systemPrompt":"Summarize","humanPrompt":"{{text}}","outputSchema":{"type":"object","properties":{"summary":{"type":"string"}}}}'
3012
- ${chalk7.dim("$")} mutagent prompts create --name "my-prompt" --system "You are helpful" --human "Hello" --output-schema '{"type":"object","properties":{"result":{"type":"string"}}}'
3013
- ${chalk7.dim("$")} mutagent prompts create --file prompt.json
3014
- ${chalk7.dim("$")} mutagent prompts create --name "raw-prompt" --raw "Summarize: {{text}}"
3108
+ ${chalk7.dim("$")} mutagent prompts create --name "my-prompt" --system "You are helpful" --human "{{input}}" --output-schema '{"type":"object","properties":{"result":{"type":"string","description":"The result"}}}'
3109
+ ${chalk7.dim("$")} mutagent prompts create --name "raw-prompt" --raw "Summarize: {{text}}" --output-schema '{"type":"object","properties":{"summary":{"type":"string","description":"Summary"}}}'
3110
+ ${chalk7.dim("$")} mutagent prompts create -d '{"name":"summarizer","systemPrompt":"Summarize","humanPrompt":"{{text}}","outputSchema":{"type":"object","properties":{"summary":{"type":"string","description":"Summary"}}}}'
3111
+ ${chalk7.dim("$")} mutagent prompts create --file prompt.json ${chalk7.dim("# full prompt object as JSON file")}
3015
3112
 
3016
3113
  Prompt Input Methods (pick one, priority order):
3017
- -d, --data Inline JSON object (recommended for CI/scripts)
3018
- --file Load from JSON file (full prompt object)
3019
- --raw-file Load plain text file as raw prompt
3020
- --system/--human Structured system + user message pair
3114
+ --system/--human Structured system + user message pair ${chalk7.green("(recommended)")}
3021
3115
  --raw Single raw prompt text with {{variables}}
3116
+ -d, --data Inline JSON object (CI/scripts/agents)
3022
3117
  --messages Full messages array as JSON
3118
+ --raw-file Load plain text file as raw prompt
3119
+ --file Load from JSON file (full prompt object)
3120
+
3121
+ Expected JSON (--data):
3122
+ ${chalk7.dim(`'{"name":"my-prompt","systemPrompt":"You are...","humanPrompt":"{{input}}","outputSchema":{"type":"object","properties":{"result":{"type":"string","description":"The result"}}},"inputSchema":{"type":"object","properties":{"input":{"type":"string","description":"User input"}}}}'`)}
3023
3123
 
3024
- ${chalk7.dim("Note: --data and --file are mutually exclusive. outputSchema is required (include in --data/--file or use --output-schema).")}
3124
+ ${chalk7.yellow("Note: Prefer --system/--human or --data over --file to avoid stale files living in your repo.")}
3125
+ ${chalk7.red("outputSchema is required.")} ${chalk7.dim("--data and --file are mutually exclusive.")}
3025
3126
  `).action(async (options) => {
3026
3127
  const isJson = getJsonFlag(prompts);
3027
3128
  const output = new OutputFormatter(isJson ? "json" : "table");
@@ -3120,17 +3221,32 @@ ${chalk7.dim("Note: --data and --file are mutually exclusive. outputSchema is re
3120
3221
  if (schema) {
3121
3222
  data.inputSchema = schema;
3122
3223
  }
3123
- } else {
3124
- output.warn("No inputSchema provided. Optimization requires inputSchema with defined variables.");
3125
3224
  }
3126
3225
  }
3127
- warnMissingSchemas(data, output);
3226
+ if (isSchemaEmpty(data.inputSchema)) {
3227
+ throw new MutagentError("VALIDATION_ERROR", "inputSchema is required. Optimization cannot run without defined input variables.", `Provide inputSchema via --data, --file, or interactive mode. Example:
3228
+ --data '{"inputSchema":{"type":"object","properties":{"query":{"type":"string","description":"User query"}}}}'
3229
+ Or use interactive mode (TTY) to define variables step by step.`);
3230
+ } else if (!isValidJsonSchema(data.inputSchema)) {
3231
+ throw new MutagentError("VALIDATION_ERROR", "inputSchema is not a valid JSON Schema object.", formatSchemaWarning("inputSchema"));
3232
+ } else {
3233
+ const missingDescs = validateSchemaDescriptions(data.inputSchema);
3234
+ if (missingDescs.length > 0) {
3235
+ throw new MutagentError("VALIDATION_ERROR", `inputSchema properties missing descriptions: ${missingDescs.join(", ")}`, `Add a 'description' field to each property in your inputSchema. Example: { "properties": { "field": { "type": "string", "description": "What this field contains" } } }`);
3236
+ }
3237
+ }
3238
+ if (isSchemaEmpty(data.outputSchema)) {
3239
+ output.warn("No outputSchema provided. This may limit optimization effectiveness.");
3240
+ } else if (!isValidJsonSchema(data.outputSchema)) {
3241
+ output.warn(formatSchemaWarning("outputSchema"));
3242
+ }
3128
3243
  const client = getSDKClient();
3129
3244
  const prompt = await client.createPrompt(data);
3130
3245
  if (isJson) {
3131
3246
  output.output({ ...prompt, _links: promptLinks(prompt.id) });
3132
3247
  } else {
3133
3248
  output.success(`Created prompt: ${prompt.name} (id: ${String(prompt.id)})`);
3249
+ output.info(`Workspace: ${client.getCurrentWorkspaceId() ?? "auto-inferred from API key"}`);
3134
3250
  const hints = formatCreationHints({
3135
3251
  resourceType: "Prompt",
3136
3252
  id: prompt.id,
@@ -3152,15 +3268,17 @@ ${chalk7.dim("Note: --data and --file are mutually exclusive. outputSchema is re
3152
3268
  handleError(error, isJson);
3153
3269
  }
3154
3270
  });
3155
- prompts.command("update").description("Update a prompt").argument("<id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Update fields as JSON string (recommended — curl-style inline)").option("-f, --file <path>", "Update from JSON file").option("--raw-file <path>", "Update from plain text file (used as rawPrompt)").option("-n, --name <name>", "New name").option("-c, --content <content>", "New content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).addHelpText("after", `
3271
+ prompts.command("update").description("Update a prompt").argument("<id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Update fields as JSON string (recommended — curl-style inline)").option("-f, --file <path>", "Update from JSON file").option("--raw-file <path>", "Update from plain text file (used as rawPrompt)").option("-n, --name <name>", "New name").option("-c, --content <content>", "New content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).option("--input-schema <json>", "Input schema as JSON string").option("--input-schema-file <path>", "Input schema from JSON file").option("--output-schema <json>", "Output schema as JSON string").option("--output-schema-file <path>", "Output schema from JSON file").addHelpText("after", `
3156
3272
  Examples:
3157
- ${chalk7.dim("$")} mutagent prompts update <id> -d '{"name":"new-name","systemPrompt":"Updated prompt"}'
3273
+ ${chalk7.dim("$")} mutagent prompts update <id> --system "Updated system prompt" --human "{{input}}"
3158
3274
  ${chalk7.dim("$")} mutagent prompts update <id> --name "new-name"
3159
- ${chalk7.dim("$")} mutagent prompts update <id> --system "Updated system prompt"
3160
- ${chalk7.dim("$")} mutagent prompts update <id> --raw-file updated-prompt.txt
3161
- ${chalk7.dim("$")} mutagent prompts update <id> --file updated-prompt.json
3275
+ ${chalk7.dim("$")} mutagent prompts update <id> --raw "Summarize: {{text}}"
3276
+ ${chalk7.dim("$")} mutagent prompts update <id> -d '{"name":"new-name","systemPrompt":"Updated prompt"}'
3277
+ ${chalk7.dim("$")} mutagent prompts update <id> --input-schema '{"type":"object","properties":{"text":{"type":"string","description":"Input text"}}}'
3278
+ ${chalk7.dim("$")} mutagent prompts update <id> --file updated-prompt.json ${chalk7.dim("# full prompt object")}
3162
3279
 
3163
- ${chalk7.dim("Note: --data and --file are mutually exclusive. CLI flags (--name) override --data fields.")}
3280
+ ${chalk7.yellow("Note: Prefer --system/--human or --data over --file to avoid stale files living in your repo.")}
3281
+ ${chalk7.dim("--data and --file are mutually exclusive. CLI flags (--name) override --data fields.")}
3164
3282
  `).action(async (id, options) => {
3165
3283
  const isJson = getJsonFlag(prompts);
3166
3284
  const output = new OutputFormatter(isJson ? "json" : "table");
@@ -3214,8 +3332,52 @@ ${chalk7.dim("Note: --data and --file are mutually exclusive. CLI flags (--name)
3214
3332
  data.rawPrompt = options.content;
3215
3333
  }
3216
3334
  }
3335
+ if (options.inputSchema && options.inputSchemaFile) {
3336
+ throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --input-schema and --input-schema-file together", "Use --input-schema for inline JSON or --input-schema-file for file-based input, not both");
3337
+ }
3338
+ if (options.inputSchema) {
3339
+ try {
3340
+ data.inputSchema = JSON.parse(options.inputSchema);
3341
+ } catch {
3342
+ throw new MutagentError("INVALID_JSON", "Invalid JSON in --input-schema flag", `Provide a valid JSON Schema, e.g., '{"type":"object","properties":{"text":{"type":"string","description":"Input text"}}}'`);
3343
+ }
3344
+ } else if (options.inputSchemaFile) {
3345
+ if (!existsSync4(options.inputSchemaFile)) {
3346
+ throw new MutagentError("FILE_NOT_FOUND", `File not found: ${options.inputSchemaFile}`, "Check the file path and try again");
3347
+ }
3348
+ try {
3349
+ data.inputSchema = JSON.parse(readFileSync4(options.inputSchemaFile, "utf-8"));
3350
+ } catch {
3351
+ throw new MutagentError("INVALID_JSON", `Failed to parse JSON from ${options.inputSchemaFile}`, "Ensure the file contains valid JSON Schema");
3352
+ }
3353
+ }
3354
+ if (options.outputSchema && options.outputSchemaFile) {
3355
+ throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --output-schema and --output-schema-file together", "Use --output-schema for inline JSON or --output-schema-file for file-based input, not both");
3356
+ }
3357
+ if (options.outputSchema) {
3358
+ try {
3359
+ data.outputSchema = JSON.parse(options.outputSchema);
3360
+ } catch {
3361
+ throw new MutagentError("INVALID_JSON", "Invalid JSON in --output-schema flag", `Provide a valid JSON Schema, e.g., '{"type":"object","properties":{"result":{"type":"string"}}}'`);
3362
+ }
3363
+ } else if (options.outputSchemaFile) {
3364
+ if (!existsSync4(options.outputSchemaFile)) {
3365
+ throw new MutagentError("FILE_NOT_FOUND", `File not found: ${options.outputSchemaFile}`, "Check the file path and try again");
3366
+ }
3367
+ try {
3368
+ data.outputSchema = JSON.parse(readFileSync4(options.outputSchemaFile, "utf-8"));
3369
+ } catch {
3370
+ throw new MutagentError("INVALID_JSON", `Failed to parse JSON from ${options.outputSchemaFile}`, "Ensure the file contains valid JSON Schema");
3371
+ }
3372
+ }
3373
+ if (data.inputSchema && isValidJsonSchema(data.inputSchema)) {
3374
+ const missingDescs = validateSchemaDescriptions(data.inputSchema);
3375
+ if (missingDescs.length > 0) {
3376
+ throw new MutagentError("VALIDATION_ERROR", `inputSchema properties missing descriptions: ${missingDescs.join(", ")}`, `Add a 'description' field to each property in your inputSchema. Example: { "properties": { "field": { "type": "string", "description": "What this field contains" } } }`);
3377
+ }
3378
+ }
3217
3379
  if (Object.keys(data).length === 0) {
3218
- throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --data, --file, --raw-file, --name, --raw, --system/--human, or --messages");
3380
+ throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --data, --file, --raw-file, --name, --raw, --system/--human, --messages, --input-schema, or --output-schema");
3219
3381
  }
3220
3382
  const promptContent = data.rawPrompt ?? data.systemPrompt ?? data.humanPrompt ?? "";
3221
3383
  if (promptContent && !isJson) {
@@ -3277,12 +3439,15 @@ ${chalk7.dim("Tip: Use --force to skip confirmation (required for non-interactiv
3277
3439
  handleError(error, isJson);
3278
3440
  }
3279
3441
  });
3280
- const dataset = prompts.command("dataset").description("Manage datasets for prompts").addHelpText("after", `
3442
+ const dataset = new Command3("dataset").description("Manage datasets for prompts").addHelpText("after", `
3281
3443
  Examples:
3282
3444
  ${chalk7.dim("$")} mutagent prompts dataset list <prompt-id>
3283
3445
  ${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.jsonl
3284
3446
  ${chalk7.dim("$")} mutagent prompts dataset remove <prompt-id> <dataset-id>
3285
- `);
3447
+ `).action(() => {
3448
+ dataset.help();
3449
+ });
3450
+ prompts.addCommand(dataset);
3286
3451
  dataset.command("list").description("List datasets for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").addHelpText("after", `
3287
3452
  Examples:
3288
3453
  ${chalk7.dim("$")} mutagent prompts dataset list <prompt-id>
@@ -3312,10 +3477,9 @@ Examples:
3312
3477
  });
3313
3478
  dataset.command("add").description("Add dataset to a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").option("-f, --file <path>", "Dataset file (JSON array, JSONL, or CSV)").option("-d, --data <json>", "Inline JSON array of dataset items").option("-n, --name <name>", "Dataset name (default: timestamped)").addHelpText("after", `
3314
3479
  Examples:
3315
- ${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.json
3316
- ${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.jsonl --name "My Dataset"
3317
- ${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.csv
3318
3480
  ${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> -d '[{"input":{"text":"hello"},"expectedOutput":{"result":"world"}}]'
3481
+ ${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> -d '[{"input":{"text":"hello"},"expectedOutput":{"result":"world"}}]' --name "My Dataset"
3482
+ ${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.jsonl ${chalk7.dim("# also supports .json, .csv")}
3319
3483
 
3320
3484
  Supported file formats:
3321
3485
  ${chalk7.dim(".json")} JSON array of objects: [{"input": {...}, "expectedOutput": {...}}, ...]
@@ -3326,7 +3490,12 @@ Inline data format (-d):
3326
3490
  JSON array of objects, e.g.:
3327
3491
  ${chalk7.dim('[{"input": {"text": "hello"}, "expectedOutput": {"result": "world"}}]')}
3328
3492
 
3329
- ${chalk7.dim("Note: --file and -d are mutually exclusive. Provide one or the other.")}
3493
+ Expected item format:
3494
+ ${chalk7.dim('{"input": {"<field>": "<value>"}, "expectedOutput": {"<field>": "<value>"}}')}
3495
+
3496
+ ${chalk7.red("Required: --data or --file must be provided.")}
3497
+ ${chalk7.yellow("Note: Prefer -d/--data for inline JSON over --file to avoid stale files living in your repo.")}
3498
+ ${chalk7.dim("--file and -d are mutually exclusive.")}
3330
3499
  `).action(async (promptId, options) => {
3331
3500
  const isJson = getJsonFlag(prompts);
3332
3501
  const output = new OutputFormatter(isJson ? "json" : "table");
@@ -3386,6 +3555,9 @@ ${chalk7.dim("Note: --file and -d are mutually exclusive. Provide one or the oth
3386
3555
  if (datasetResult.itemCount !== undefined && datasetResult.itemCount > 0) {
3387
3556
  output.info(`Items uploaded: ${String(datasetResult.itemCount)}`);
3388
3557
  }
3558
+ if (datasetResult.missingExpectedOutputCount !== undefined && datasetResult.missingExpectedOutputCount > 0) {
3559
+ output.warn(`${String(datasetResult.missingExpectedOutputCount)} of ${String(datasetResult.itemCount ?? 0)} items have no expectedOutput. Optimization may fail without expected outputs.`);
3560
+ }
3389
3561
  const hints = formatCreationHints({
3390
3562
  resourceType: "Dataset",
3391
3563
  id: datasetResult.id,
@@ -3419,12 +3591,16 @@ Examples:
3419
3591
  handleError(error, isJson);
3420
3592
  }
3421
3593
  });
3422
- const evaluation = prompts.command("evaluation").description("Manage evaluations for prompts").addHelpText("after", `
3594
+ const evaluation = new Command3("evaluation").description("Manage evaluations for prompts").addHelpText("after", `
3423
3595
  Examples:
3424
3596
  ${chalk7.dim("$")} mutagent prompts evaluation list <prompt-id>
3425
3597
  ${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "My Eval"
3598
+ ${chalk7.dim("$")} mutagent prompts evaluation delete <evaluation-id>
3426
3599
  ${chalk7.dim("$")} mutagent prompts evaluation results <run-id>
3427
- `);
3600
+ `).action(() => {
3601
+ evaluation.help();
3602
+ });
3603
+ prompts.addCommand(evaluation);
3428
3604
  evaluation.command("list").description("List evaluations for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").addHelpText("after", `
3429
3605
  Examples:
3430
3606
  ${chalk7.dim("$")} mutagent prompts evaluation list <prompt-id>
@@ -3454,17 +3630,19 @@ Examples:
3454
3630
  });
3455
3631
  evaluation.command("create").description("Create an evaluation configuration for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Evaluation as JSON string (recommended — curl-style inline)").option("-n, --name <name>", "Evaluation name (required unless --guided)").option("-f, --file <path>", "Load evaluation criteria from JSON file").option("--description <text>", "Evaluation description").option("--dataset <id>", "Dataset ID to associate (from: mutagent prompts dataset list)").option("--guided", "Interactive guided mode — build criteria step by step").addHelpText("after", `
3456
3632
  Examples:
3457
- ${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --guided
3458
- ${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --data '{"name":"Accuracy","evalConfig":{"criteria":[...]}}' --name "Accuracy"
3459
- ${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Accuracy Check"
3633
+ ${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --guided ${chalk7.dim("# recommended: interactive walkthrough")}
3634
+ ${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Accuracy" -d '{"evalConfig":{"criteria":[{"name":"Accuracy","description":"Score 1.0 if output matches expected","evaluationParameter":"result"}]}}'
3460
3635
  ${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Full Eval" --file criteria.json --dataset <dataset-id>
3461
- ${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "QA" --description "Quality assurance eval" --json
3462
3636
 
3463
- Criteria file format (--file or --data):
3464
- ${chalk7.dim('{ "name": "...", "evalConfig": { "criteria": [...] }, "llmConfig": { "model": "gpt-4" } }')}
3637
+ Expected JSON (--data):
3638
+ ${chalk7.dim('{"evalConfig":{"criteria":[{"name":"<name>","description":"<scoring rubric>","evaluationParameter":"<output field to score>"}]}}')}
3639
+
3640
+ ${chalk7.red("Each criterion MUST have: name, description, evaluationParameter")}
3641
+ evaluationParameter maps to output schema fields (e.g., "result", "classification")
3465
3642
 
3466
- ${chalk7.dim("Note: --data and --file are mutually exclusive. CLI flags (--name, --description) override --data fields.")}
3467
- ${chalk7.dim("Tip: Use --guided for an interactive walkthrough of criteria creation.")}
3643
+ ${chalk7.yellow("Note: Prefer --guided or --data over --file to avoid stale files living in your repo.")}
3644
+ ${chalk7.red("Required: --name (unless --guided). Criteria must include evaluationParameter.")}
3645
+ ${chalk7.dim("--data and --file are mutually exclusive. CLI flags (--name, --description) override --data fields.")}
3468
3646
  ${chalk7.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent prompts dataset list <prompt-id>")}
3469
3647
  `).action(async (promptId, options) => {
3470
3648
  const isJson = getJsonFlag(prompts);
@@ -3549,6 +3727,79 @@ ${chalk7.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent
3549
3727
  evalData.name = options.name;
3550
3728
  if (options.description)
3551
3729
  evalData.description = options.description;
3730
+ const criteria = evalData.evalConfig?.criteria;
3731
+ if (!criteria || !Array.isArray(criteria) || criteria.length === 0) {
3732
+ let fieldsHint = "";
3733
+ try {
3734
+ const client2 = getSDKClient();
3735
+ const prompt = await client2.getPrompt(promptId);
3736
+ if (prompt.outputSchema && typeof prompt.outputSchema === "object") {
3737
+ const props = prompt.outputSchema.properties;
3738
+ if (props && typeof props === "object") {
3739
+ const fields = Object.keys(props);
3740
+ if (fields.length > 0) {
3741
+ fieldsHint = `
3742
+ Detected output fields from prompt schema: ${fields.join(", ")}
3743
+ `;
3744
+ }
3745
+ }
3746
+ }
3747
+ } catch {}
3748
+ throw new MutagentError("VALIDATION_ERROR", "Evaluation criteria are required. Provide criteria via --data, --file, or use --guided mode.", `Each criterion needs: name, description, evaluationParameter (the output field to score).
3749
+ ` + fieldsHint + `
3750
+ Example JSON (--data flag):
3751
+ --data '{"evalConfig":{"criteria":[{"name":"Accuracy","description":"Score 1.0 if output matches expected, 0.0 otherwise","evaluationParameter":"classification"}]}}'
3752
+
3753
+ Or use --guided for interactive creation: mutagent prompts evaluation create <id> --guided`);
3754
+ }
3755
+ for (const c of criteria) {
3756
+ const hasDescription = c.description ?? c.scoringRubric;
3757
+ const hasEvalParam = c.evaluationParameter ?? c.targetField;
3758
+ if (!c.name || !hasDescription || !hasEvalParam) {
3759
+ const cName = typeof c.name === "string" ? c.name : "unnamed";
3760
+ throw new MutagentError("VALIDATION_ERROR", `Criterion "${cName}" is missing required fields. Need: name, description (or scoringRubric), evaluationParameter (or targetField).`, 'evaluationParameter maps to the output field to evaluate (e.g., "classification", "riskLevel"). Use --guided for interactive creation.');
3761
+ }
3762
+ if (c.scoringRubric && !c.description) {
3763
+ c.description = c.scoringRubric;
3764
+ delete c.scoringRubric;
3765
+ }
3766
+ if (c.targetField && !c.evaluationParameter) {
3767
+ const tf = c.targetField;
3768
+ c.evaluationParameter = tf.startsWith("output.") ? tf.slice("output.".length) : tf.startsWith("input.") ? tf.slice("input.".length) : tf;
3769
+ delete c.targetField;
3770
+ }
3771
+ }
3772
+ const seenParams = new Set;
3773
+ const duplicateParams = [];
3774
+ for (const c of criteria) {
3775
+ const param = c.evaluationParameter;
3776
+ if (param) {
3777
+ if (seenParams.has(param)) {
3778
+ duplicateParams.push(param);
3779
+ }
3780
+ seenParams.add(param);
3781
+ }
3782
+ }
3783
+ if (duplicateParams.length > 0) {
3784
+ let availableFields = [];
3785
+ try {
3786
+ const client2 = getSDKClient();
3787
+ const prompt = await client2.getPrompt(promptId);
3788
+ if (prompt.outputSchema && typeof prompt.outputSchema === "object") {
3789
+ const props = prompt.outputSchema.properties;
3790
+ if (props && typeof props === "object") {
3791
+ availableFields = Object.keys(props);
3792
+ }
3793
+ }
3794
+ } catch {}
3795
+ const uniqueDupes = [...new Set(duplicateParams)];
3796
+ const fieldsHint = availableFields.length > 0 ? `
3797
+ Available output fields: ${availableFields.join(", ")}` : "";
3798
+ throw new MutagentError("VALIDATION_ERROR", `Duplicate evaluationParameter: "${uniqueDupes.join('", "')}". Each criterion must target a unique output field.`, "Each criterion scores a different output field. Fix by changing the evaluationParameter to a unique field." + fieldsHint + `
3799
+
3800
+ Example:
3801
+ --data '{"evalConfig":{"criteria":[{"name":"Accuracy","description":"...","evaluationParameter":"classification"},{"name":"Confidence","description":"...","evaluationParameter":"confidence"}]}}'`);
3802
+ }
3552
3803
  if (options.dataset) {
3553
3804
  evalData.datasetId = parseInt(options.dataset, 10);
3554
3805
  if (isNaN(evalData.datasetId)) {
@@ -3596,15 +3847,37 @@ Examples:
3596
3847
  handleError(error, isJson);
3597
3848
  }
3598
3849
  });
3599
- const optimize = prompts.command("optimize").description("Manage prompt optimization jobs").addHelpText("after", `
3850
+ evaluation.command("delete").description("Delete an evaluation").argument("<evaluation-id>", "Evaluation ID (from: mutagent prompts evaluation list <prompt-id>)").addHelpText("after", `
3851
+ Examples:
3852
+ ${chalk7.dim("$")} mutagent prompts evaluation delete <evaluation-id>
3853
+ ${chalk7.dim("$")} mutagent prompts evaluation delete <evaluation-id> --json
3854
+ `).action(async (evaluationId) => {
3855
+ const isJson = getJsonFlag(prompts);
3856
+ const output = new OutputFormatter(isJson ? "json" : "table");
3857
+ try {
3858
+ const client = getSDKClient();
3859
+ await client.deleteEvaluation(evaluationId);
3860
+ if (isJson) {
3861
+ output.output({ success: true, message: "Evaluation deleted successfully", evaluationId });
3862
+ } else {
3863
+ output.success(`Evaluation ${evaluationId} deleted successfully`);
3864
+ }
3865
+ } catch (error) {
3866
+ handleError(error, isJson);
3867
+ }
3868
+ });
3869
+ const optimize = new Command3("optimize").description("Manage prompt optimization jobs").addHelpText("after", `
3600
3870
  Examples:
3601
3871
  ${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id>
3602
3872
  ${chalk7.dim("$")} mutagent prompts optimize status <job-id>
3603
3873
  ${chalk7.dim("$")} mutagent prompts optimize results <job-id>
3604
3874
 
3605
3875
  Workflow: start -> status (poll) -> results
3606
- `);
3607
- optimize.command("start").description("Start prompt optimization").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").requiredOption("-d, --dataset <id>", "Dataset ID for optimization (from: mutagent prompts dataset list <prompt-id>)").option("--max-iterations <n>", "Max optimization iterations (default: 3)").option("--target-score <n>", "Target accuracy 0-1 (e.g., 0.9)").option("--patience <n>", "Iterations without improvement before stopping").option("--model <model-id>", 'Target LLM model (e.g., "claude-sonnet-4-5-20250929")').option("--eval-model <model-id>", "Evaluation model (defaults to target model)").addHelpText("after", `
3876
+ `).action(() => {
3877
+ optimize.help();
3878
+ });
3879
+ prompts.addCommand(optimize);
3880
+ optimize.command("start").description("Start prompt optimization").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").requiredOption("-d, --dataset <id>", "Dataset ID for optimization (from: mutagent prompts dataset list <prompt-id>)").option("--max-iterations <n>", "Max optimization iterations (default: 1)").option("--target-score <n>", "Target accuracy 0-1 (default: 0.8)").option("--patience <n>", "Iterations without improvement before stopping").option("--model <model-id>", 'Target LLM model (e.g., "claude-sonnet-4-5-20250929")').option("--eval-model <model-id>", "Evaluation model (defaults to target model)").addHelpText("after", `
3608
3881
  Examples:
3609
3882
  ${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id>
3610
3883
  ${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --max-iterations 5
@@ -3613,11 +3886,45 @@ Examples:
3613
3886
  ${PREREQUISITES_TEXT}
3614
3887
 
3615
3888
  ${chalk7.dim("Monitor progress with: mutagent prompts optimize status <job-id>")}
3889
+
3890
+ ${chalk7.dim("AI Agent Note: After running commands, present CLI output to the user as a status report. Use --json for structured parsing.")}
3616
3891
  `).action(async (promptId, options) => {
3617
3892
  const isJson = getJsonFlag(prompts);
3618
3893
  const output = new OutputFormatter(isJson ? "json" : "table");
3619
3894
  try {
3620
3895
  const client = getSDKClient();
3896
+ if (!isJson) {
3897
+ output.info("Running pre-flight checks...");
3898
+ const prompt = await client.getPrompt(promptId);
3899
+ if (isSchemaEmpty(prompt.outputSchema)) {
3900
+ throw new MutagentError("MISSING_OUTPUT_SCHEMA", "Prompt is missing outputSchema — required for optimization", `Update your prompt: mutagent prompts update ${promptId} -d '{"outputSchema":{"type":"object","properties":{"result":{"type":"string"}}}}'`);
3901
+ }
3902
+ if (isSchemaEmpty(prompt.inputSchema)) {
3903
+ output.warn("Prompt has no inputSchema. Optimization works best with defined input variables.");
3904
+ }
3905
+ try {
3906
+ const evals = await client.listEvaluations(promptId);
3907
+ if (evals.length === 0) {
3908
+ throw new MutagentError("MISSING_EVALUATION", "No evaluations found for this prompt — required for optimization", `Create an evaluation: mutagent prompts evaluation create ${promptId} --guided`);
3909
+ }
3910
+ const validEvals = evals.filter((e) => {
3911
+ const config = e.evalConfig;
3912
+ const criteria = config?.criteria;
3913
+ if (!criteria || criteria.length === 0)
3914
+ return false;
3915
+ return criteria.some((c) => c.evaluationParameter ?? c.targetField);
3916
+ });
3917
+ if (validEvals.length === 0) {
3918
+ output.warn(`Evaluations exist but none have evaluationParameter set. Optimization works best with criteria targeting specific output fields. Re-create: mutagent prompts evaluation create ${promptId} --guided`);
3919
+ } else {
3920
+ output.info(`Found ${String(validEvals.length)} evaluation(s) with proper criteria.`);
3921
+ }
3922
+ } catch (e) {
3923
+ if (e instanceof MutagentError)
3924
+ throw e;
3925
+ output.warn("Could not validate evaluations. Optimization will use default evaluation.");
3926
+ }
3927
+ }
3621
3928
  const job = await client.startOptimization(promptId, options.dataset, {
3622
3929
  maxIterations: options.maxIterations ? parseInt(options.maxIterations, 10) : undefined,
3623
3930
  targetScore: options.targetScore ? parseFloat(options.targetScore) : undefined,
@@ -3629,27 +3936,55 @@ ${chalk7.dim("Monitor progress with: mutagent prompts optimize status <job-id>")
3629
3936
  output.output({
3630
3937
  ...job,
3631
3938
  _links: {
3632
- dashboard: optimizerLink(job.jobId),
3633
- api: `/api/prompts/${promptId}/optimizations/${job.jobId}`
3939
+ dashboard: optimizerLink(job.id),
3940
+ api: `/api/prompts/${promptId}/optimizations/${job.id}`
3634
3941
  }
3635
3942
  });
3636
3943
  } else {
3637
- output.success(`Started optimization job: ${job.jobId}`);
3944
+ output.success(`Started optimization job: ${job.id}`);
3638
3945
  const hints = formatCreationHints({
3639
3946
  resourceType: "Optimization",
3640
- id: job.jobId,
3641
- dashboardUrl: optimizerLink(job.jobId),
3642
- apiPath: `/api/prompts/${promptId}/optimizations/${job.jobId}`
3947
+ id: job.id,
3948
+ dashboardUrl: optimizerLink(job.id),
3949
+ apiPath: `/api/prompts/${promptId}/optimizations/${job.id}`
3643
3950
  });
3644
3951
  console.log(hints);
3645
3952
  }
3646
3953
  } catch (error) {
3647
3954
  if (error instanceof ApiError) {
3648
3955
  const messages = parseValidationErrors(error);
3956
+ const errorText = messages.join(" ");
3957
+ const suggestions = [];
3958
+ if (errorText.includes("Output Schema is not present") || errorText.includes("outputSchema")) {
3959
+ suggestions.push(`Update prompt: mutagent prompts update ${promptId} -d '{"outputSchema":{"type":"object","properties":{...}}}'`);
3960
+ }
3961
+ if (errorText.includes("Outputs are not valid") || errorText.includes("Parameters Mismatch")) {
3962
+ suggestions.push("Dataset items may not match the prompt schema. Check expectedOutput fields in your dataset.");
3963
+ }
3964
+ if (errorText.includes("Inputs are not valid")) {
3965
+ suggestions.push("Dataset item inputs don't match the prompt's inputSchema. Verify field names and types.");
3966
+ }
3967
+ if (errorText.includes("has no items") || errorText.includes("Dataset") && errorText.includes("not found")) {
3968
+ suggestions.push(`Upload data: mutagent prompts dataset add ${promptId} --file data.json`);
3969
+ }
3970
+ if (errorText.includes("Evaluations are not present")) {
3971
+ suggestions.push(`Create evaluation: mutagent prompts evaluation create ${promptId} --guided`);
3972
+ }
3973
+ if (errorText.includes("Trial") && errorText.includes("limit")) {
3974
+ suggestions.push("Trial optimization limit reached. Contact support to upgrade.");
3975
+ }
3649
3976
  if (!isJson) {
3650
- console.error(chalk7.red("Optimization failed. Missing requirements:"));
3977
+ console.error(chalk7.red(`
3978
+ Optimization failed:`));
3651
3979
  for (const msg of messages) {
3652
- console.error(chalk7.red(` - ${msg}`));
3980
+ console.error(chalk7.red(` ${msg}`));
3981
+ }
3982
+ if (suggestions.length > 0) {
3983
+ console.error(chalk7.yellow(`
3984
+ Suggested fixes:`));
3985
+ for (const s of suggestions) {
3986
+ console.error(chalk7.yellow(` → ${s}`));
3987
+ }
3653
3988
  }
3654
3989
  console.error("");
3655
3990
  console.error(PREREQUISITES_TEXT);
@@ -3894,235 +4229,179 @@ ${chalk8.dim("Exports to stdout by default. Use --output to save to a file.")}
3894
4229
  init_config();
3895
4230
  import { Command as Command5 } from "commander";
3896
4231
  import chalk9 from "chalk";
3897
- import { writeFileSync as writeFileSync3, existsSync as existsSync11 } from "fs";
4232
+ import { writeFileSync as writeFileSync3, existsSync as existsSync9 } from "fs";
3898
4233
  import { execSync } from "child_process";
3899
4234
  init_errors();
3900
4235
 
3901
- // src/lib/integrations/mastra.ts
4236
+ // src/lib/integrations/langchain.ts
3902
4237
  import { readFileSync as readFileSync5, existsSync as existsSync5 } from "fs";
3903
- function renderPrerequisites(config) {
3904
- const keyPreview = config.apiKey.slice(0, 8) + "..." + config.apiKey.slice(-4);
3905
- return `✓ MUTAGENT_API_KEY: ${keyPreview}
3906
- ✓ MUTAGENT_ENDPOINT: ${config.endpoint}
3907
- ✓ API Connection: Verified`;
3908
- }
3909
- var mastraIntegration = {
3910
- name: "mastra",
3911
- description: "Mastra AI framework",
4238
+ var langchainIntegration = {
4239
+ name: "langchain",
4240
+ description: "LangChain framework",
3912
4241
  async detect() {
3913
- const files = ["mastra.config.ts", "mastra.config.js"];
3914
- const found = [];
3915
- for (const file of files) {
3916
- if (existsSync5(file)) {
3917
- found.push(file);
3918
- }
3919
- }
3920
- let hasMastraDep = false;
4242
+ let hasLangchain = false;
3921
4243
  if (existsSync5("package.json")) {
3922
4244
  try {
3923
4245
  const pkg = JSON.parse(readFileSync5("package.json", "utf-8"));
3924
4246
  const deps = { ...pkg.dependencies, ...pkg.devDependencies };
3925
- hasMastraDep = "@mastra/core" in deps;
4247
+ hasLangchain = "langchain" in deps || "@langchain/core" in deps;
3926
4248
  } catch {}
3927
4249
  }
3928
4250
  return {
3929
- detected: found.length > 0 || hasMastraDep,
3930
- confidence: found.length > 0 ? 0.95 : hasMastraDep ? 0.7 : 0,
3931
- files: found
4251
+ detected: hasLangchain,
4252
+ confidence: hasLangchain ? 0.9 : 0,
4253
+ files: hasLangchain ? ["package.json"] : []
3932
4254
  };
3933
4255
  },
3934
4256
  async generate(config) {
3935
4257
  return `---
3936
- name: mutagent-mastra-integration
3937
- description: Integrate MutagenT with Mastra framework for prompt optimization and trace observability
3938
- framework: mastra
4258
+ name: mutagent-langchain-integration
4259
+ description: Integrate MutagenT with LangChain for prompt optimization and trace observability
4260
+ framework: langchain
3939
4261
  version: 1.0.0
3940
4262
  ---
3941
4263
 
3942
- # MutagenT + Mastra Integration
4264
+ # MutagenT + LangChain Integration
3943
4265
 
3944
4266
  ## Prerequisites Verification
3945
4267
 
3946
- ${renderPrerequisites(config)}
4268
+ - MUTAGENT_API_KEY: ${config.apiKey.slice(0, 8)}...${config.apiKey.slice(-4)}
4269
+ - MUTAGENT_ENDPOINT: ${config.endpoint}
4270
+ - API Connection: Verified
3947
4271
 
3948
4272
  ## Installation
3949
4273
 
3950
4274
  \`\`\`bash
3951
- bun add @mutagent/sdk
4275
+ # The @mutagent/langchain package was installed automatically
4276
+ # If you need to install manually:
4277
+ bun add @mutagent/langchain
3952
4278
  # or
3953
- npm install @mutagent/sdk
4279
+ npm install @mutagent/langchain
3954
4280
  \`\`\`
3955
4281
 
3956
4282
  ## Configuration
3957
4283
 
3958
4284
  ### 1. Environment Variables
3959
4285
 
3960
- Add to your \`.env\` file:
3961
-
3962
4286
  \`\`\`env
3963
4287
  MUTAGENT_API_KEY=${config.apiKey}
3964
4288
  MUTAGENT_ENDPOINT=${config.endpoint}
3965
4289
  \`\`\`
3966
4290
 
3967
- ### 2. Mastra Config Integration
3968
-
3969
- Update \`mastra.config.ts\`:
4291
+ ### 2. LangChain Callback Handler
3970
4292
 
3971
4293
  \`\`\`typescript
3972
- import { defineConfig } from '@mastra/core';
3973
- import { MutagentObserver } from '@mutagent/sdk/mastra';
3974
-
3975
- export default defineConfig({
3976
- observers: [
3977
- new MutagentObserver({
3978
- apiKey: process.env.MUTAGENT_API_KEY!,
3979
- endpoint: process.env.MUTAGENT_ENDPOINT,
3980
- // Optional: auto-capture all agent traces
3981
- captureTraces: true,
3982
- })
3983
- ],
3984
- });
3985
- \`\`\`
3986
-
3987
- ### 3. Per-Agent Configuration
4294
+ import { ChatOpenAI } from '@langchain/openai';
4295
+ import { MutagentCallbackHandler } from '@mutagent/langchain';
4296
+ import { initTracing } from '@mutagent/sdk/tracing';
3988
4297
 
3989
- For specific agents only:
4298
+ // Initialize tracing (or set MUTAGENT_API_KEY env var for auto-init)
4299
+ initTracing({
4300
+ apiKey: process.env.MUTAGENT_API_KEY!,
4301
+ endpoint: process.env.MUTAGENT_ENDPOINT,
4302
+ });
3990
4303
 
3991
- \`\`\`typescript
3992
- import { Agent } from '@mastra/core';
3993
- import { withMutagent } from '@mutagent/sdk/mastra';
4304
+ // Create callback handler (no args needed — uses SDK tracing)
4305
+ const handler = new MutagentCallbackHandler();
3994
4306
 
3995
- const agent = new Agent({
3996
- name: 'support-agent',
3997
- instructions: '...',
4307
+ const llm = new ChatOpenAI({
4308
+ callbacks: [handler],
3998
4309
  });
3999
4310
 
4000
- // Wrap with MutagenT observability
4001
- export default withMutagent(agent, {
4002
- promptId: 'support-template',
4003
- autoOptimize: true,
4004
- });
4311
+ const result = await llm.invoke('Hello world');
4312
+ // Traces automatically captured via LangChain callbacks
4005
4313
  \`\`\`
4006
4314
 
4007
- ## Usage
4008
-
4009
- ### Trace Collection (Replaces Langfuse)
4010
-
4011
- All agent calls are automatically traced to MutagenT:
4315
+ ### 3. Chain Integration
4012
4316
 
4013
4317
  \`\`\`typescript
4014
- const result = await agent.generate('How do I reset my password?');
4015
- // Trace automatically captured with latency, tokens, input/output
4016
- \`\`\`
4017
-
4018
- ### Prompt Optimization
4318
+ import { ChatOpenAI } from '@langchain/openai';
4319
+ import { PromptTemplate } from '@langchain/core/prompts';
4320
+ import { MutagentCallbackHandler } from '@mutagent/langchain';
4019
4321
 
4020
- \`\`\`typescript
4021
- import { optimizePrompt } from '@mutagent/sdk';
4322
+ const handler = new MutagentCallbackHandler();
4022
4323
 
4023
- // Run optimization for a prompt
4024
- const optimized = await optimizePrompt({
4025
- promptId: 'support-template',
4026
- datasetId: 'support-tickets',
4027
- metric: 'response_quality',
4324
+ const llm = new ChatOpenAI({
4325
+ callbacks: [handler],
4028
4326
  });
4029
4327
 
4030
- // Update agent with optimized prompt
4031
- agent.instructions = optimized.content;
4032
- \`\`\`
4033
-
4034
- ### Manual Tracing
4035
-
4036
- For custom traces:
4037
-
4038
- \`\`\`typescript
4039
- import { trace } from '@mutagent/sdk';
4040
-
4041
- const span = trace.start({
4042
- name: 'custom-operation',
4043
- promptId: 'my-prompt',
4044
- });
4328
+ const template = PromptTemplate.fromTemplate('Answer the question: {question}');
4329
+ const chain = template.pipe(llm);
4045
4330
 
4046
- try {
4047
- const result = await doSomething();
4048
- span.end({ output: result });
4049
- } catch (error) {
4050
- span.error(error);
4051
- }
4331
+ const result = await chain.invoke(
4332
+ { question: 'What is TypeScript?' },
4333
+ { callbacks: [handler] },
4334
+ );
4052
4335
  \`\`\`
4053
4336
 
4054
4337
  ## Migration from Langfuse
4055
4338
 
4056
- Replace Langfuse imports:
4339
+ Replace Langfuse callback:
4057
4340
 
4058
4341
  \`\`\`typescript
4059
4342
  // Before
4060
- import { Langfuse } from 'langfuse';
4343
+ import { CallbackHandler } from 'langfuse-langchain';
4344
+ const callbacks = [new CallbackHandler()];
4061
4345
 
4062
4346
  // After
4063
- import { MutagentObserver } from '@mutagent/sdk/mastra';
4347
+ import { MutagentCallbackHandler } from '@mutagent/langchain';
4348
+ const callbacks = [new MutagentCallbackHandler()];
4064
4349
  \`\`\`
4065
4350
 
4066
4351
  ## Verification
4067
4352
 
4068
- Run this to verify your integration:
4069
-
4070
4353
  \`\`\`bash
4071
- mutagent integrate mastra --verify
4354
+ mutagent integrate langchain --verify
4072
4355
  \`\`\`
4073
4356
 
4074
- ## Next Steps
4357
+ ## Documentation
4075
4358
 
4076
- 1. Create your first prompt: \`mutagent prompts create --interactive\`
4077
- 2. Add a dataset: \`mutagent prompts datasets:add <prompt-id> --file data.jsonl\`
4078
- 3. Run optimization: \`mutagent prompts optimize <prompt-id> --dataset <dataset-id>\`
4079
- 4. View traces: \`mutagent traces list --prompt <prompt-id>\`
4359
+ - Full guide: https://docs.mutagent.io/integrations/langchain
4360
+ - API Reference: https://docs.mutagent.io/sdk/tracing
4361
+ - Dashboard: https://app.mutagent.io
4080
4362
 
4081
- ## CLI Commands Reference
4363
+ ## CLI Reference
4082
4364
 
4083
4365
  \`\`\`bash
4084
- # List prompts
4085
- mutagent prompts list
4086
-
4087
- # Get prompt with traces
4088
- mutagent prompts get <id> --with-datasets
4366
+ # View traces
4367
+ mutagent traces list --prompt <prompt-id>
4089
4368
 
4090
- # Analyze traces
4369
+ # Analyze performance
4091
4370
  mutagent traces analyze <prompt-id>
4092
4371
  \`\`\`
4093
4372
  `;
4094
4373
  }
4095
4374
  };
4096
4375
 
4097
- // src/lib/integrations/langchain.ts
4376
+ // src/lib/integrations/langgraph.ts
4098
4377
  import { readFileSync as readFileSync6, existsSync as existsSync6 } from "fs";
4099
- var langchainIntegration = {
4100
- name: "langchain",
4101
- description: "LangChain framework",
4378
+ var langgraphIntegration = {
4379
+ name: "langgraph",
4380
+ description: "LangGraph agent workflow framework",
4102
4381
  async detect() {
4103
- let hasLangchain = false;
4382
+ let hasLanggraph = false;
4104
4383
  if (existsSync6("package.json")) {
4105
4384
  try {
4106
4385
  const pkg = JSON.parse(readFileSync6("package.json", "utf-8"));
4107
4386
  const deps = { ...pkg.dependencies, ...pkg.devDependencies };
4108
- hasLangchain = "langchain" in deps || "@langchain/core" in deps;
4387
+ hasLanggraph = "@langchain/langgraph" in deps;
4109
4388
  } catch {}
4110
4389
  }
4111
4390
  return {
4112
- detected: hasLangchain,
4113
- confidence: hasLangchain ? 0.9 : 0,
4114
- files: hasLangchain ? ["package.json"] : []
4391
+ detected: hasLanggraph,
4392
+ confidence: hasLanggraph ? 0.95 : 0,
4393
+ files: hasLanggraph ? ["package.json"] : []
4115
4394
  };
4116
4395
  },
4117
4396
  async generate(config) {
4118
4397
  return `---
4119
- name: mutagent-langchain-integration
4120
- description: Integrate MutagenT with LangChain for prompt optimization and trace observability
4121
- framework: langchain
4122
- version: 1.0.0
4398
+ name: mutagent-langgraph-integration
4399
+ description: Integrate MutagenT with LangGraph for agent workflow observability
4400
+ framework: langgraph
4401
+ version: 2.0.0
4123
4402
  ---
4124
4403
 
4125
- # MutagenT + LangChain Integration
4404
+ # MutagenT + LangGraph Integration
4126
4405
 
4127
4406
  ## Prerequisites Verification
4128
4407
 
@@ -4130,173 +4409,51 @@ version: 1.0.0
4130
4409
  - MUTAGENT_ENDPOINT: ${config.endpoint}
4131
4410
  - API Connection: Verified
4132
4411
 
4412
+ > **Note**: LangGraph is built on LangChain. The same \`MutagentCallbackHandler\` from
4413
+ > \`@mutagent/langchain\` works for both LangChain and LangGraph — no separate package needed.
4414
+
4133
4415
  ## Installation
4134
4416
 
4135
4417
  \`\`\`bash
4136
- # The @mutagent/langchain package was installed automatically
4137
- # If you need to install manually:
4138
- bun add @mutagent/langchain
4418
+ bun add @mutagent/langchain @mutagent/sdk
4139
4419
  # or
4140
- npm install @mutagent/langchain
4420
+ npm install @mutagent/langchain @mutagent/sdk
4141
4421
  \`\`\`
4142
4422
 
4143
- ## Configuration
4144
-
4145
- ### 1. Environment Variables
4146
-
4147
- \`\`\`env
4148
- MUTAGENT_API_KEY=${config.apiKey}
4149
- MUTAGENT_ENDPOINT=${config.endpoint}
4150
- \`\`\`
4423
+ > **Deprecation Notice**: The \`@mutagent/langgraph\` package is deprecated.
4424
+ > Use \`@mutagent/langchain\` instead — it supports both LangChain and LangGraph.
4151
4425
 
4152
- ### 2. LangChain Callback Handler
4426
+ ## Integration
4153
4427
 
4154
4428
  \`\`\`typescript
4155
- import { ChatOpenAI } from '@langchain/openai';
4156
4429
  import { MutagentCallbackHandler } from '@mutagent/langchain';
4157
4430
  import { initTracing } from '@mutagent/sdk/tracing';
4158
4431
 
4159
- // Initialize tracing (or set MUTAGENT_API_KEY env var for auto-init)
4160
- initTracing({
4161
- apiKey: process.env.MUTAGENT_API_KEY!,
4162
- endpoint: process.env.MUTAGENT_ENDPOINT,
4163
- });
4164
-
4165
- // Create callback handler (no args needed — uses SDK tracing)
4166
- const handler = new MutagentCallbackHandler();
4432
+ // Initialize tracing (once at app startup)
4433
+ initTracing({ apiKey: process.env.MUTAGENT_API_KEY! });
4167
4434
 
4168
- const llm = new ChatOpenAI({
4169
- callbacks: [handler],
4435
+ // Create the handler
4436
+ const handler = new MutagentCallbackHandler({
4437
+ sessionId: 'my-session', // optional
4438
+ userId: 'user-123', // optional
4170
4439
  });
4171
4440
 
4172
- const result = await llm.invoke('Hello world');
4173
- // Traces automatically captured via LangChain callbacks
4441
+ // Pass to any LangGraph invoke/stream call
4442
+ const result = await graph.invoke(input, { callbacks: [handler] });
4174
4443
  \`\`\`
4175
4444
 
4176
- ### 3. Chain Integration
4445
+ ## Full Graph Example
4177
4446
 
4178
4447
  \`\`\`typescript
4448
+ import { StateGraph, Annotation } from '@langchain/langgraph';
4179
4449
  import { ChatOpenAI } from '@langchain/openai';
4180
- import { PromptTemplate } from '@langchain/core/prompts';
4181
4450
  import { MutagentCallbackHandler } from '@mutagent/langchain';
4182
-
4183
- const handler = new MutagentCallbackHandler();
4184
-
4185
- const llm = new ChatOpenAI({
4186
- callbacks: [handler],
4187
- });
4188
-
4189
- const template = PromptTemplate.fromTemplate('Answer the question: {question}');
4190
- const chain = template.pipe(llm);
4191
-
4192
- const result = await chain.invoke(
4193
- { question: 'What is TypeScript?' },
4194
- { callbacks: [handler] },
4195
- );
4196
- \`\`\`
4197
-
4198
- ## Migration from Langfuse
4199
-
4200
- Replace Langfuse callback:
4201
-
4202
- \`\`\`typescript
4203
- // Before
4204
- import { CallbackHandler } from 'langfuse-langchain';
4205
- const callbacks = [new CallbackHandler()];
4206
-
4207
- // After
4208
- import { MutagentCallbackHandler } from '@mutagent/langchain';
4209
- const callbacks = [new MutagentCallbackHandler()];
4210
- \`\`\`
4211
-
4212
- ## Verification
4213
-
4214
- \`\`\`bash
4215
- mutagent integrate langchain --verify
4216
- \`\`\`
4217
-
4218
- ## Documentation
4219
-
4220
- - Full guide: https://docs.mutagent.io/integrations/langchain
4221
- - API Reference: https://docs.mutagent.io/sdk/tracing
4222
- - Dashboard: https://app.mutagent.io
4223
-
4224
- ## CLI Reference
4225
-
4226
- \`\`\`bash
4227
- # View traces
4228
- mutagent traces list --prompt <prompt-id>
4229
-
4230
- # Analyze performance
4231
- mutagent traces analyze <prompt-id>
4232
- \`\`\`
4233
- `;
4234
- }
4235
- };
4236
-
4237
- // src/lib/integrations/langgraph.ts
4238
- import { readFileSync as readFileSync7, existsSync as existsSync7 } from "fs";
4239
- var langgraphIntegration = {
4240
- name: "langgraph",
4241
- description: "LangGraph agent workflow framework",
4242
- async detect() {
4243
- let hasLanggraph = false;
4244
- if (existsSync7("package.json")) {
4245
- try {
4246
- const pkg = JSON.parse(readFileSync7("package.json", "utf-8"));
4247
- const deps = { ...pkg.dependencies, ...pkg.devDependencies };
4248
- hasLanggraph = "@langchain/langgraph" in deps;
4249
- } catch {}
4250
- }
4251
- return {
4252
- detected: hasLanggraph,
4253
- confidence: hasLanggraph ? 0.95 : 0,
4254
- files: hasLanggraph ? ["package.json"] : []
4255
- };
4256
- },
4257
- async generate(config) {
4258
- return `---
4259
- name: mutagent-langgraph-integration
4260
- description: Integrate MutagenT with LangGraph for agent workflow observability
4261
- framework: langgraph
4262
- version: 1.0.0
4263
- ---
4264
-
4265
- # MutagenT + LangGraph Integration
4266
-
4267
- ## Prerequisites Verification
4268
-
4269
- - MUTAGENT_API_KEY: ${config.apiKey.slice(0, 8)}...${config.apiKey.slice(-4)}
4270
- - MUTAGENT_ENDPOINT: ${config.endpoint}
4271
- - API Connection: Verified
4272
-
4273
- ## Installation
4274
-
4275
- \`\`\`bash
4276
- # The @mutagent/langgraph package was installed automatically
4277
- # If you need to install manually:
4278
- bun add @mutagent/langgraph
4279
- # or
4280
- npm install @mutagent/langgraph
4281
- \`\`\`
4282
-
4283
- ## Configuration
4284
-
4285
- ### Graph with MutagenT Tracer
4286
-
4287
- \`\`\`typescript
4288
- import { StateGraph, Annotation } from '@langchain/langgraph';
4289
- import { MutagentGraphTracer } from '@mutagent/langgraph';
4290
4451
  import { initTracing } from '@mutagent/sdk/tracing';
4291
4452
 
4292
- // Initialize tracing (or set MUTAGENT_API_KEY env var for auto-init)
4293
- initTracing({
4294
- apiKey: process.env.MUTAGENT_API_KEY!,
4295
- endpoint: process.env.MUTAGENT_ENDPOINT,
4296
- });
4453
+ // Initialize tracing (once at app startup)
4454
+ initTracing({ apiKey: process.env.MUTAGENT_API_KEY! });
4297
4455
 
4298
- // Create tracer (no args needed — uses SDK tracing)
4299
- const tracer = new MutagentGraphTracer();
4456
+ const handler = new MutagentCallbackHandler();
4300
4457
 
4301
4458
  // Define your graph as usual
4302
4459
  const StateAnnotation = Annotation.Root({
@@ -4310,29 +4467,24 @@ const graph = new StateGraph(StateAnnotation)
4310
4467
  .addEdge('__start__', 'agent')
4311
4468
  .compile();
4312
4469
 
4313
- // Use lifecycle methods for tracing
4314
- tracer.handleGraphStart('agent-workflow', { input: 'Hello' });
4315
- const result = await graph.invoke({ input: 'Hello' });
4316
- tracer.handleGraphEnd({ output: result.output });
4470
+ // All nodes, edges, and LLM calls are automatically traced
4471
+ const result = await graph.invoke(
4472
+ { input: 'Hello' },
4473
+ { callbacks: [handler] },
4474
+ );
4317
4475
  \`\`\`
4318
4476
 
4319
- ### Per-Node and Edge Tracing
4477
+ ## Streaming
4320
4478
 
4321
4479
  \`\`\`typescript
4322
- // Track individual nodes
4323
- tracer.handleNodeStart('agent', { query: 'Hello' });
4324
- // ... node logic ...
4325
- tracer.handleNodeEnd('agent', { result: 'response' });
4326
-
4327
- // Track edge transitions
4328
- tracer.handleEdgeTransition('agent', 'tools', {
4329
- isConditional: true,
4330
- condition: 'needsTools',
4331
- conditionResult: true,
4332
- });
4480
+ const handler = new MutagentCallbackHandler();
4333
4481
 
4334
- tracer.handleNodeStart('tools', { toolName: 'search' });
4335
- tracer.handleNodeEnd('tools', { result: 'found' });
4482
+ for await (const event of graph.stream(
4483
+ { input: 'Hello' },
4484
+ { callbacks: [handler] },
4485
+ )) {
4486
+ console.log(event);
4487
+ }
4336
4488
  \`\`\`
4337
4489
 
4338
4490
  ## Verification
@@ -4346,30 +4498,20 @@ mutagent integrate langgraph --verify
4346
4498
  - Full guide: https://docs.mutagent.io/integrations/langgraph
4347
4499
  - API Reference: https://docs.mutagent.io/sdk/tracing
4348
4500
  - Dashboard: https://app.mutagent.io
4349
-
4350
- ## CLI Commands
4351
-
4352
- \`\`\`bash
4353
- # View workflow traces
4354
- mutagent traces list --prompt <graph-id>
4355
-
4356
- # Analyze node performance
4357
- mutagent traces analyze <node-prompt-id>
4358
- \`\`\`
4359
4501
  `;
4360
4502
  }
4361
4503
  };
4362
4504
 
4363
4505
  // src/lib/integrations/vercel-ai.ts
4364
- import { readFileSync as readFileSync8, existsSync as existsSync8 } from "fs";
4506
+ import { readFileSync as readFileSync7, existsSync as existsSync7 } from "fs";
4365
4507
  var vercelAiIntegration = {
4366
4508
  name: "vercel-ai",
4367
4509
  description: "Vercel AI SDK",
4368
4510
  async detect() {
4369
4511
  let hasAiSdk = false;
4370
- if (existsSync8("package.json")) {
4512
+ if (existsSync7("package.json")) {
4371
4513
  try {
4372
- const pkg = JSON.parse(readFileSync8("package.json", "utf-8"));
4514
+ const pkg = JSON.parse(readFileSync7("package.json", "utf-8"));
4373
4515
  const deps = { ...pkg.dependencies, ...pkg.devDependencies };
4374
4516
  hasAiSdk = "ai" in deps;
4375
4517
  } catch {}
@@ -4385,7 +4527,7 @@ var vercelAiIntegration = {
4385
4527
  name: mutagent-vercel-ai-integration
4386
4528
  description: Integrate MutagenT with Vercel AI SDK for streaming and edge functions
4387
4529
  framework: vercel-ai
4388
- version: 1.0.0
4530
+ version: 2.0.0
4389
4531
  ---
4390
4532
 
4391
4533
  # MutagenT + Vercel AI SDK Integration
@@ -4399,14 +4541,52 @@ version: 1.0.0
4399
4541
  ## Installation
4400
4542
 
4401
4543
  \`\`\`bash
4402
- # The @mutagent/vercel-ai package was installed automatically
4403
- # If you need to install manually:
4404
- bun add @mutagent/vercel-ai
4544
+ bun add @mutagent/vercel-ai @mutagent/sdk
4405
4545
  # or
4406
- npm install @mutagent/vercel-ai
4546
+ npm install @mutagent/vercel-ai @mutagent/sdk
4547
+
4548
+ # For Option A (OTel SpanExporter), also install:
4549
+ bun add @opentelemetry/sdk-trace-node @opentelemetry/sdk-trace-base
4407
4550
  \`\`\`
4408
4551
 
4409
- ## Integration
4552
+ ---
4553
+
4554
+ ## Option A (Recommended): OTel SpanExporter
4555
+
4556
+ Uses Vercel AI SDK's built-in \`experimental_telemetry\` with an OpenTelemetry exporter
4557
+ that sends spans directly to MutagenT.
4558
+
4559
+ \`\`\`typescript
4560
+ import { MutagentSpanExporter } from '@mutagent/vercel-ai';
4561
+ import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
4562
+ import { SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base';
4563
+ import { initTracing } from '@mutagent/sdk/tracing';
4564
+ import { generateText } from 'ai';
4565
+ import { openai } from '@ai-sdk/openai';
4566
+
4567
+ // Initialize MutagenT tracing
4568
+ initTracing({ apiKey: process.env.MUTAGENT_API_KEY! });
4569
+
4570
+ // Set up OTel with MutagenT exporter
4571
+ const provider = new NodeTracerProvider();
4572
+ provider.addSpanProcessor(
4573
+ new SimpleSpanProcessor(new MutagentSpanExporter())
4574
+ );
4575
+ provider.register();
4576
+
4577
+ // Use Vercel AI SDK normally with telemetry enabled
4578
+ const result = await generateText({
4579
+ model: openai('gpt-4'),
4580
+ prompt: 'Hello!',
4581
+ experimental_telemetry: { isEnabled: true },
4582
+ });
4583
+ \`\`\`
4584
+
4585
+ ---
4586
+
4587
+ ## Option B (Alternative): Middleware
4588
+
4589
+ Uses the Vercel AI SDK \`wrapLanguageModel\` middleware pattern.
4410
4590
 
4411
4591
  \`\`\`typescript
4412
4592
  // app/api/chat/route.ts
@@ -4415,13 +4595,10 @@ import { openai } from '@ai-sdk/openai';
4415
4595
  import { createMutagentMiddleware } from '@mutagent/vercel-ai';
4416
4596
  import { initTracing } from '@mutagent/sdk/tracing';
4417
4597
 
4418
- // Initialize tracing (or set MUTAGENT_API_KEY env var for auto-init)
4419
- initTracing({
4420
- apiKey: process.env.MUTAGENT_API_KEY!,
4421
- endpoint: process.env.MUTAGENT_ENDPOINT,
4422
- });
4598
+ // Initialize tracing (once at app startup)
4599
+ initTracing({ apiKey: process.env.MUTAGENT_API_KEY! });
4423
4600
 
4424
- // Create middleware (no args needed — uses SDK tracing)
4601
+ // Create middleware
4425
4602
  const middleware = createMutagentMiddleware();
4426
4603
 
4427
4604
  // Wrap your model with MutagenT middleware
@@ -4467,30 +4644,20 @@ mutagent integrate vercel-ai --verify
4467
4644
  - Full guide: https://docs.mutagent.io/integrations/vercel-ai
4468
4645
  - API Reference: https://docs.mutagent.io/sdk/tracing
4469
4646
  - Dashboard: https://app.mutagent.io
4470
-
4471
- ## CLI Commands
4472
-
4473
- \`\`\`bash
4474
- # View streaming traces
4475
- mutagent traces list --prompt <prompt-id>
4476
-
4477
- # Export for analysis
4478
- mutagent traces export --prompt <prompt-id> --format json
4479
- \`\`\`
4480
4647
  `;
4481
4648
  }
4482
4649
  };
4483
4650
 
4484
4651
  // src/lib/integrations/openai.ts
4485
- import { readFileSync as readFileSync9, existsSync as existsSync9 } from "fs";
4652
+ import { readFileSync as readFileSync8, existsSync as existsSync8 } from "fs";
4486
4653
  var openaiIntegration = {
4487
4654
  name: "openai",
4488
4655
  description: "OpenAI SDK integration with automatic tracing",
4489
4656
  async detect() {
4490
4657
  let hasOpenAI = false;
4491
- if (existsSync9("package.json")) {
4658
+ if (existsSync8("package.json")) {
4492
4659
  try {
4493
- const pkg = JSON.parse(readFileSync9("package.json", "utf-8"));
4660
+ const pkg = JSON.parse(readFileSync8("package.json", "utf-8"));
4494
4661
  const deps = { ...pkg.dependencies, ...pkg.devDependencies };
4495
4662
  hasOpenAI = "openai" in deps;
4496
4663
  } catch {}
@@ -4506,7 +4673,7 @@ var openaiIntegration = {
4506
4673
  name: mutagent-openai-integration
4507
4674
  description: Integrate MutagenT with OpenAI SDK for automatic tracing
4508
4675
  framework: openai
4509
- version: 1.0.0
4676
+ version: 2.0.0
4510
4677
  ---
4511
4678
 
4512
4679
  # MutagenT + OpenAI SDK Integration
@@ -4520,41 +4687,28 @@ version: 1.0.0
4520
4687
  ## Installation
4521
4688
 
4522
4689
  \`\`\`bash
4523
- # The @mutagent/openai package was installed automatically
4524
- # If you need to install manually:
4525
- bun add @mutagent/openai
4690
+ bun add @mutagent/openai @mutagent/sdk
4526
4691
  # or
4527
- npm install @mutagent/openai
4528
- \`\`\`
4529
-
4530
- ## Configuration
4531
-
4532
- ### 1. Environment Variables
4533
-
4534
- \`\`\`env
4535
- MUTAGENT_API_KEY=${config.apiKey}
4536
- MUTAGENT_ENDPOINT=${config.endpoint}
4537
- OPENAI_API_KEY=your-openai-api-key
4692
+ npm install @mutagent/openai @mutagent/sdk
4538
4693
  \`\`\`
4539
4694
 
4540
- ### 2. Traced OpenAI Client
4695
+ ## Integration
4541
4696
 
4542
4697
  \`\`\`typescript
4543
- import { MutagentOpenAI } from '@mutagent/openai';
4698
+ import OpenAI from 'openai';
4699
+ import { observeOpenAI } from '@mutagent/openai';
4544
4700
  import { initTracing } from '@mutagent/sdk/tracing';
4545
4701
 
4546
- // Initialize tracing (or set MUTAGENT_API_KEY env var for auto-init)
4547
- initTracing({
4548
- apiKey: process.env.MUTAGENT_API_KEY!,
4549
- endpoint: process.env.MUTAGENT_ENDPOINT,
4550
- });
4702
+ // Initialize tracing (once at app startup)
4703
+ initTracing({ apiKey: process.env.MUTAGENT_API_KEY! });
4551
4704
 
4552
- // Create a traced OpenAI client (drop-in replacement for OpenAI)
4553
- const openai = new MutagentOpenAI({
4554
- apiKey: process.env.OPENAI_API_KEY,
4705
+ // Wrap the OpenAI client ALL methods are automatically traced
4706
+ const openai = observeOpenAI(new OpenAI(), {
4707
+ sessionId: 'my-session', // optional
4708
+ userId: 'user-123', // optional
4555
4709
  });
4556
4710
 
4557
- // All calls are automatically traced
4711
+ // Use exactly as normal — chat, embeddings, images, audio all work
4558
4712
  const completion = await openai.chat.completions.create({
4559
4713
  model: 'gpt-4o',
4560
4714
  messages: [{ role: 'user', content: 'Hello!' }],
@@ -4563,21 +4717,12 @@ const completion = await openai.chat.completions.create({
4563
4717
  console.log(completion.choices[0].message.content);
4564
4718
  \`\`\`
4565
4719
 
4566
- ## Usage Examples
4720
+ ALL OpenAI SDK methods are preserved and traced automatically.
4721
+ No API changes — just wrap your client and everything is observed.
4567
4722
 
4568
- ### Chat Completion
4569
-
4570
- \`\`\`typescript
4571
- const response = await openai.chat.completions.create({
4572
- model: 'gpt-4o',
4573
- messages: [
4574
- { role: 'system', content: 'You are a helpful assistant.' },
4575
- { role: 'user', content: 'What is TypeScript?' },
4576
- ],
4577
- });
4578
- \`\`\`
4723
+ ## Streaming
4579
4724
 
4580
- ### Streaming
4725
+ Streaming works out of the box with no extra configuration:
4581
4726
 
4582
4727
  \`\`\`typescript
4583
4728
  const stream = await openai.chat.completions.create({
@@ -4591,299 +4736,39 @@ for await (const chunk of stream) {
4591
4736
  }
4592
4737
  \`\`\`
4593
4738
 
4594
- ## Verification
4595
-
4596
- \`\`\`bash
4597
- mutagent integrate openai --verify
4598
- \`\`\`
4599
-
4600
- ## Documentation
4601
-
4602
- - Full guide: https://docs.mutagent.io/integrations/openai
4603
- - API Reference: https://docs.mutagent.io/sdk/tracing
4604
- - Dashboard: https://app.mutagent.io
4605
-
4606
- ## CLI Reference
4607
-
4608
- \`\`\`bash
4609
- # View traces
4610
- mutagent traces list --prompt <prompt-id>
4611
-
4612
- # Analyze performance
4613
- mutagent traces analyze <prompt-id>
4614
-
4615
- # Export traces
4616
- mutagent traces export --format json
4617
- \`\`\`
4618
- `;
4619
- }
4620
- };
4621
-
4622
- // src/lib/integrations/claude-code.ts
4623
- import { readFileSync as readFileSync10, existsSync as existsSync10 } from "fs";
4624
- var claudeCodeIntegration = {
4625
- name: "claude-code",
4626
- description: "Claude Code SDK",
4627
- async detect() {
4628
- let hasClaudeCode = false;
4629
- if (existsSync10("package.json")) {
4630
- try {
4631
- const pkg = JSON.parse(readFileSync10("package.json", "utf-8"));
4632
- const deps = { ...pkg.dependencies, ...pkg.devDependencies };
4633
- hasClaudeCode = "@anthropic-ai/claude-code" in deps || "@anthropic-ai/sdk" in deps;
4634
- } catch {}
4635
- }
4636
- return {
4637
- detected: hasClaudeCode,
4638
- confidence: hasClaudeCode ? 0.85 : 0,
4639
- files: hasClaudeCode ? ["package.json"] : []
4640
- };
4641
- },
4642
- async generate(config) {
4643
- return `---
4644
- name: mutagent-claude-code-integration
4645
- description: Integrate MutagenT with Claude Code SDK
4646
- framework: claude-code
4647
- version: 1.0.0
4648
- ---
4649
-
4650
- # MutagenT + Claude Code SDK Integration
4651
-
4652
- ## Prerequisites Verification
4653
-
4654
- ✓ MUTAGENT_API_KEY: ${config.apiKey.slice(0, 8)}...${config.apiKey.slice(-4)}
4655
- ✓ MUTAGENT_ENDPOINT: ${config.endpoint}
4656
- ✓ API Connection: Verified
4657
-
4658
- ## Installation
4659
-
4660
- \`\`\`bash
4661
- bun add @mutagent/sdk
4662
- \`\`\`
4663
-
4664
- ## Basic Integration
4739
+ ## Embeddings
4665
4740
 
4666
4741
  \`\`\`typescript
4667
- import Anthropic from '@anthropic-ai/sdk';
4668
- import { MutagentTracer } from '@mutagent/sdk/claude';
4669
-
4670
- const tracer = new MutagentTracer({
4671
- apiKey: process.env.MUTAGENT_API_KEY!,
4672
- endpoint: process.env.MUTAGENT_ENDPOINT,
4673
- promptId: 'claude-prompt',
4674
- });
4675
-
4676
- const anthropic = new Anthropic({
4677
- apiKey: process.env.ANTHROPIC_API_KEY!,
4742
+ const embedding = await openai.embeddings.create({
4743
+ model: 'text-embedding-3-small',
4744
+ input: 'Hello world',
4678
4745
  });
4679
-
4680
- // Wrap API calls
4681
- const response = await tracer.wrap(
4682
- () => anthropic.messages.create({
4683
- model: 'claude-3-5-sonnet-20241022',
4684
- max_tokens: 1024,
4685
- messages: [{ role: 'user', content: 'Hello!' }],
4686
- })
4687
- );
4688
- \`\`\`
4689
-
4690
- ## With Optimized Prompts
4691
-
4692
- \`\`\`typescript
4693
- import { getOptimizedPrompt } from '@mutagent/sdk';
4694
-
4695
- const optimized = await getOptimizedPrompt('my-template');
4696
-
4697
- const response = await tracer.wrap(
4698
- () => anthropic.messages.create({
4699
- model: 'claude-3-5-sonnet-20241022',
4700
- system: optimized,
4701
- messages,
4702
- })
4703
- );
4704
- \`\`\`
4705
-
4706
- ## MCP Server Integration
4707
-
4708
- For Claude Desktop / MCP:
4709
-
4710
- \`\`\`json
4711
- {
4712
- "mcpServers": {
4713
- "mutagent": {
4714
- "command": "npx",
4715
- "args": ["-y", "@mutagent/mcp-server"],
4716
- "env": {
4717
- "MUTAGENT_API_KEY": "${config.apiKey}"
4718
- }
4719
- }
4720
- }
4721
- }
4746
+ // Embedding call is traced automatically
4722
4747
  \`\`\`
4723
4748
 
4724
4749
  ## Verification
4725
4750
 
4726
4751
  \`\`\`bash
4727
- mutagent integrate claude-code --verify
4728
- \`\`\`
4729
-
4730
- ## CLI Commands
4731
-
4732
- \`\`\`bash
4733
- # View Claude traces
4734
- mutagent traces list --prompt <prompt-id>
4735
-
4736
- # Optimize Claude prompts
4737
- mutagent prompts optimize <prompt-id> --dataset <dataset-id>
4738
- \`\`\`
4739
- `;
4740
- }
4741
- };
4742
-
4743
- // src/lib/integrations/generic.ts
4744
- var genericIntegration = {
4745
- name: "generic",
4746
- description: "Generic OpenAI-compatible integration",
4747
- async detect() {
4748
- return {
4749
- detected: false,
4750
- confidence: 0,
4751
- files: []
4752
- };
4753
- },
4754
- async generate(config) {
4755
- return `---
4756
- name: mutagent-generic-integration
4757
- description: Generic OpenAI-compatible API integration
4758
- framework: generic
4759
- version: 1.0.0
4760
- ---
4761
-
4762
- # MutagenT + Generic (OpenAI-compatible) Integration
4763
-
4764
- ## Prerequisites Verification
4765
-
4766
- ✓ MUTAGENT_API_KEY: ${config.apiKey.slice(0, 8)}...${config.apiKey.slice(-4)}
4767
- ✓ MUTAGENT_ENDPOINT: ${config.endpoint}
4768
- ✓ API Connection: Verified
4769
-
4770
- ## Installation
4771
-
4772
- \`\`\`bash
4773
- bun add @mutagent/sdk
4774
- \`\`\`
4775
-
4776
- ## Fetch-based Integration
4777
-
4778
- \`\`\`typescript
4779
- import { trace } from '@mutagent/sdk';
4780
-
4781
- async function callLLM(prompt: string) {
4782
- const span = trace.start({
4783
- name: 'llm-call',
4784
- promptId: 'my-prompt',
4785
- input: { prompt },
4786
- });
4787
-
4788
- const start = Date.now();
4789
-
4790
- try {
4791
- const response = await fetch('https://api.openai.com/v1/chat/completions', {
4792
- method: 'POST',
4793
- headers: {
4794
- 'Authorization': \`Bearer \${process.env.OPENAI_API_KEY}\`,
4795
- 'Content-Type': 'application/json',
4796
- },
4797
- body: JSON.stringify({
4798
- model: 'gpt-4o',
4799
- messages: [{ role: 'user', content: prompt }],
4800
- }),
4801
- });
4802
-
4803
- const data = await response.json();
4804
- const latency = Date.now() - start;
4805
-
4806
- span.end({
4807
- output: data,
4808
- latency,
4809
- tokens: {
4810
- input: data.usage?.prompt_tokens || 0,
4811
- output: data.usage?.completion_tokens || 0,
4812
- },
4813
- });
4814
-
4815
- return data;
4816
- } catch (error) {
4817
- span.error(error);
4818
- throw error;
4819
- }
4820
- }
4821
- \`\`\`
4822
-
4823
- ## With Optimized Prompts
4824
-
4825
- \`\`\`typescript
4826
- import { getOptimizedPrompt } from '@mutagent/sdk';
4827
-
4828
- const optimized = await getOptimizedPrompt('template-id');
4829
- const result = await callLLM(optimized);
4830
- \`\`\`
4831
-
4832
- ## OpenAI SDK Wrapper
4833
-
4834
- \`\`\`typescript
4835
- import OpenAI from 'openai';
4836
- import { wrapOpenAI } from '@mutagent/sdk/openai';
4837
-
4838
- const openai = wrapOpenAI(
4839
- new OpenAI({ apiKey: process.env.OPENAI_API_KEY }),
4840
- {
4841
- mutagentApiKey: process.env.MUTAGENT_API_KEY!,
4842
- promptId: 'default',
4843
- }
4844
- );
4845
-
4846
- // All calls are traced
4847
- const completion = await openai.chat.completions.create({...});
4848
- \`\`\`
4849
-
4850
- ## Verification
4851
-
4852
- \`\`\`bash
4853
- mutagent integrate generic --verify
4752
+ mutagent integrate openai --verify
4854
4753
  \`\`\`
4855
4754
 
4856
- ## CLI Commands
4857
-
4858
- \`\`\`bash
4859
- # View traces
4860
- mutagent traces list --prompt <prompt-id>
4755
+ ## Documentation
4861
4756
 
4862
- # Run evaluation
4863
- mutagent prompts evals:run <prompt-id> --dataset <dataset-id>
4864
- \`\`\`
4757
+ - Full guide: https://docs.mutagent.io/integrations/openai
4758
+ - API Reference: https://docs.mutagent.io/sdk/tracing
4759
+ - Dashboard: https://app.mutagent.io
4865
4760
  `;
4866
4761
  }
4867
4762
  };
4868
4763
 
4869
4764
  // src/lib/integrations/registry.ts
4870
4765
  var frameworkRegistry = new Map([
4871
- ["mastra", mastraIntegration],
4872
4766
  ["langchain", langchainIntegration],
4873
4767
  ["langgraph", langgraphIntegration],
4874
4768
  ["vercel-ai", vercelAiIntegration],
4875
- ["openai", openaiIntegration],
4876
- ["claude-code", claudeCodeIntegration],
4877
- ["generic", genericIntegration]
4769
+ ["openai", openaiIntegration]
4878
4770
  ]);
4879
4771
  var frameworkMetadata = [
4880
- {
4881
- name: "mastra",
4882
- displayName: "Mastra",
4883
- description: "Modern AI agent framework with observability built-in",
4884
- npmPackage: "@mastra/core",
4885
- website: "https://mastra.ai"
4886
- },
4887
4772
  {
4888
4773
  name: "langchain",
4889
4774
  displayName: "LangChain",
@@ -4897,7 +4782,7 @@ var frameworkMetadata = [
4897
4782
  displayName: "LangGraph",
4898
4783
  description: "Agent workflow framework built on LangChain",
4899
4784
  npmPackage: "@langchain/langgraph",
4900
- mutagentPackage: "@mutagent/langgraph",
4785
+ mutagentPackage: "@mutagent/langchain",
4901
4786
  website: "https://langchain-ai.github.io/langgraphjs/"
4902
4787
  },
4903
4788
  {
@@ -4915,20 +4800,6 @@ var frameworkMetadata = [
4915
4800
  npmPackage: "openai",
4916
4801
  mutagentPackage: "@mutagent/openai",
4917
4802
  website: "https://platform.openai.com"
4918
- },
4919
- {
4920
- name: "claude-code",
4921
- displayName: "Claude Code SDK",
4922
- description: "Native Anthropic integration for Claude",
4923
- npmPackage: "@anthropic-ai/claude-code",
4924
- website: "https://docs.anthropic.com"
4925
- },
4926
- {
4927
- name: "generic",
4928
- displayName: "Generic (OpenAI-compatible)",
4929
- description: "Any OpenAI-compatible API endpoint",
4930
- npmPackage: "openai",
4931
- website: "https://platform.openai.com"
4932
4803
  }
4933
4804
  ];
4934
4805
  function getFramework(name) {
@@ -4940,16 +4811,16 @@ function getFrameworkMetadata(name) {
4940
4811
 
4941
4812
  // src/commands/integrate.ts
4942
4813
  function detectPackageManager() {
4943
- if (existsSync11("bun.lockb") || existsSync11("bun.lock")) {
4814
+ if (existsSync9("bun.lockb") || existsSync9("bun.lock")) {
4944
4815
  return "bun";
4945
4816
  }
4946
- if (existsSync11("pnpm-lock.yaml")) {
4817
+ if (existsSync9("pnpm-lock.yaml")) {
4947
4818
  return "pnpm";
4948
4819
  }
4949
- if (existsSync11("yarn.lock")) {
4820
+ if (existsSync9("yarn.lock")) {
4950
4821
  return "yarn";
4951
4822
  }
4952
- if (existsSync11("package-lock.json")) {
4823
+ if (existsSync9("package-lock.json")) {
4953
4824
  return "npm";
4954
4825
  }
4955
4826
  try {
@@ -4978,8 +4849,6 @@ AI AGENT: Please determine the framework by:
4978
4849
  - \`@langchain/langgraph\` -> use \`mutagent integrate langgraph\`
4979
4850
  - \`ai\` (vercel) -> use \`mutagent integrate vercel-ai\`
4980
4851
  - \`openai\` -> use \`mutagent integrate openai\`
4981
- - \`@mastra/core\` -> use \`mutagent integrate mastra\`
4982
- - \`@anthropic-ai/claude-code\` -> use \`mutagent integrate claude-code\`
4983
4852
 
4984
4853
  ### Option 2: Ask the user
4985
4854
  "Which AI framework are you using?"
@@ -4987,9 +4856,6 @@ AI AGENT: Please determine the framework by:
4987
4856
  - LangGraph
4988
4857
  - Vercel AI SDK
4989
4858
  - OpenAI SDK
4990
- - Mastra
4991
- - Claude Code
4992
- - Generic (OpenAI-compatible)
4993
4859
 
4994
4860
  ### Next Steps
4995
4861
  1. Once determined, run: \`mutagent integrate <framework>\`
@@ -5006,12 +4872,12 @@ Examples:
5006
4872
  ${chalk9.dim("$")} mutagent integrate langchain --output ./INTEGRATION.md
5007
4873
  ${chalk9.dim("$")} mutagent integrate --list # List available frameworks
5008
4874
 
5009
- Frameworks: mastra, langchain, langgraph, vercel-ai, openai, claude-code, generic
4875
+ Frameworks: langchain, langgraph, vercel-ai, openai
5010
4876
 
5011
4877
  This is a SKILL LOADER for AI coding agents (Claude Code, Cursor, etc.).
5012
4878
  It returns INSTRUCTIONS that AI agents execute - it does NOT auto-install packages.
5013
4879
  `);
5014
- integrate.argument("[framework]", "Framework name (mastra, langchain, langgraph, vercel-ai, openai, claude-code, generic)").option("-o, --output <path>", "Save instructions to file").option("--raw", "Output raw markdown (no formatting)").option("--verify", "Verify integration after setup").action(async (frameworkArg, options) => {
4880
+ integrate.argument("[framework]", "Framework name (langchain, langgraph, vercel-ai, openai)").option("-o, --output <path>", "Save instructions to file").option("--raw", "Output raw markdown (no formatting)").option("--verify", "Verify integration after setup").action(async (frameworkArg, options) => {
5015
4881
  const isJson = getJsonFlag(integrate);
5016
4882
  const output = new OutputFormatter(isJson ? "json" : "table");
5017
4883
  try {
@@ -5109,7 +4975,7 @@ AI AGENT: Execute these steps using Bash for install, Write/Edit for code change
5109
4975
  init_sdk_client();
5110
4976
  import { Command as Command6 } from "commander";
5111
4977
  import chalk10 from "chalk";
5112
- import { readFileSync as readFileSync11 } from "fs";
4978
+ import { readFileSync as readFileSync9 } from "fs";
5113
4979
  init_errors();
5114
4980
  function createAgentsCommand() {
5115
4981
  const agents = new Command6("agents").description("Manage AI agents").addHelpText("after", `
@@ -5220,20 +5086,48 @@ System Prompt:`));
5220
5086
  handleError(error, isJson);
5221
5087
  }
5222
5088
  });
5223
- agents.command("create").description("Create a new agent").option("-f, --file <path>", "Create from JSON file").option("-n, --name <name>", "Agent name").option("-s, --slug <slug>", "Agent slug (URL-friendly identifier)").option("-p, --system-prompt <prompt>", "System prompt").option("-m, --model <model>", "Model (claude-sonnet-4-5, claude-opus-4-5, claude-haiku-4-5)").option("-d, --description <description>", "Agent description").addHelpText("after", `
5089
+ agents.command("create").description("Create a new agent").option("-d, --data <json>", "Agent as JSON string (recommended for CI/scripts/agents)").option("-f, --file <path>", "Create from JSON file").option("-n, --name <name>", "Agent name").option("-s, --slug <slug>", "Agent slug (URL-friendly identifier)").option("-p, --system-prompt <prompt>", "System prompt").option("-m, --model <model>", "Model (claude-sonnet-4-5, claude-opus-4-5, claude-haiku-4-5)").option("--description <desc>", "Agent description").addHelpText("after", `
5224
5090
  Examples:
5225
- ${chalk10.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "Review code for bugs"
5226
- ${chalk10.dim("$")} mutagent agents create --file agent.json
5227
- ${chalk10.dim("$")} mutagent agents create --name "Helper" --slug helper --system-prompt "..." --model claude-sonnet-4-5 --json
5091
+ ${chalk10.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "You are a code reviewer..."
5092
+ ${chalk10.dim("$")} mutagent agents create -d '{"name":"Code Reviewer","slug":"code-reviewer","systemPrompt":"You are a code reviewer..."}'
5093
+ ${chalk10.dim("$")} mutagent agents create --file agent.json ${chalk10.dim("# full agent object as JSON file")}
5228
5094
 
5229
- ${chalk10.dim("Required: --name, --slug, and --system-prompt (or --file).")}
5095
+ Expected JSON (--data):
5096
+ ${chalk10.dim('{"name":"<name>","slug":"<slug>","systemPrompt":"<system prompt>","model":"<model-id>","description":"<description>"}')}
5097
+
5098
+ Input Methods (pick one, priority order):
5099
+ --name/--slug/... Individual flags ${chalk10.green("(recommended)")}
5100
+ -d, --data Inline JSON object (CI/scripts/agents)
5101
+ --file Load from JSON file
5102
+
5103
+ ${chalk10.yellow("Note: Prefer individual flags or --data over --file to avoid stale files living in your repo.")}
5104
+ ${chalk10.red("Required: name, slug, systemPrompt.")} ${chalk10.dim("--data and --file are mutually exclusive. CLI flags override --data fields.")}
5230
5105
  `).action(async (options) => {
5231
5106
  const isJson = getJsonFlag(agents);
5232
5107
  const output = new OutputFormatter(isJson ? "json" : "table");
5233
5108
  try {
5234
5109
  let data;
5235
- if (options.file) {
5236
- const content = readFileSync11(options.file, "utf-8");
5110
+ if (options.data && options.file) {
5111
+ throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --data and --file together", "Use --data for inline JSON or --file for file-based input, not both");
5112
+ }
5113
+ if (options.data) {
5114
+ try {
5115
+ data = JSON.parse(options.data);
5116
+ } catch {
5117
+ throw new MutagentError("INVALID_JSON", "Invalid JSON in --data flag", `Provide a valid JSON object, e.g., '{"name":"my-agent","slug":"my-agent","systemPrompt":"You are..."}'`);
5118
+ }
5119
+ if (options.name)
5120
+ data.name = options.name;
5121
+ if (options.slug)
5122
+ data.slug = options.slug;
5123
+ if (options.systemPrompt)
5124
+ data.systemPrompt = options.systemPrompt;
5125
+ if (options.model)
5126
+ data.model = options.model;
5127
+ if (options.description)
5128
+ data.description = options.description;
5129
+ } else if (options.file) {
5130
+ const content = readFileSync9(options.file, "utf-8");
5237
5131
  data = JSON.parse(content);
5238
5132
  } else if (options.name && options.slug && options.systemPrompt) {
5239
5133
  data = {
@@ -5246,7 +5140,7 @@ ${chalk10.dim("Required: --name, --slug, and --system-prompt (or --file).")}
5246
5140
  if (options.description)
5247
5141
  data.description = options.description;
5248
5142
  } else {
5249
- throw new MutagentError("MISSING_ARGUMENTS", "Either --file or (--name, --slug, and --system-prompt) are required", "Use --file to load from JSON or provide --name, --slug, and --system-prompt");
5143
+ throw new MutagentError("MISSING_ARGUMENTS", "Either --data, --file, or (--name, --slug, and --system-prompt) are required", "Use --data for inline JSON, --file to load from JSON, or provide --name, --slug, and --system-prompt");
5250
5144
  }
5251
5145
  const client = getSDKClient();
5252
5146
  const agent = await client.createAgent(data);
@@ -5256,18 +5150,46 @@ ${chalk10.dim("Required: --name, --slug, and --system-prompt (or --file).")}
5256
5150
  handleError(error, isJson);
5257
5151
  }
5258
5152
  });
5259
- agents.command("update").description("Update an agent").argument("<id>", "Agent ID").option("-f, --file <path>", "Update from JSON file").option("-n, --name <name>", "New name").option("-p, --system-prompt <prompt>", "New system prompt").option("-m, --model <model>", "New model").option("-d, --description <description>", "New description").option("-s, --status <status>", "New status (active, paused, archived)").addHelpText("after", `
5153
+ agents.command("update").description("Update an agent").argument("<id>", "Agent ID").option("-d, --data <json>", "Agent updates as JSON string (CI/scripts/agents)").option("-f, --file <path>", "Update from JSON file").option("-n, --name <name>", "New name").option("-p, --system-prompt <prompt>", "New system prompt").option("-m, --model <model>", "New model").option("--description <desc>", "New description").option("-s, --status <status>", "New status (active, paused, archived)").addHelpText("after", `
5260
5154
  Examples:
5261
5155
  ${chalk10.dim("$")} mutagent agents update <id> --name "New Name"
5262
- ${chalk10.dim("$")} mutagent agents update <id> --status paused
5263
- ${chalk10.dim("$")} mutagent agents update <id> --file updated-agent.json --json
5156
+ ${chalk10.dim("$")} mutagent agents update <id> --system-prompt "Updated prompt" --status active
5157
+ ${chalk10.dim("$")} mutagent agents update <id> -d '{"name":"New Name","systemPrompt":"Updated prompt"}'
5158
+ ${chalk10.dim("$")} mutagent agents update <id> --file updated-agent.json ${chalk10.dim("# full agent object")}
5159
+
5160
+ Input Methods (pick one, priority order):
5161
+ --name/--system-prompt/... Individual flags ${chalk10.green("(recommended)")}
5162
+ -d, --data Inline JSON object (CI/scripts/agents)
5163
+ --file Load from JSON file
5164
+
5165
+ ${chalk10.yellow("Note: Prefer individual flags or --data over --file to avoid stale files living in your repo.")}
5166
+ ${chalk10.dim("--data and --file are mutually exclusive. CLI flags override --data fields.")}
5264
5167
  `).action(async (id, options) => {
5265
5168
  const isJson = getJsonFlag(agents);
5266
5169
  const output = new OutputFormatter(isJson ? "json" : "table");
5267
5170
  try {
5268
5171
  let data = {};
5269
- if (options.file) {
5270
- const content = readFileSync11(options.file, "utf-8");
5172
+ if (options.data && options.file) {
5173
+ throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --data and --file together", "Use --data for inline JSON or --file for file-based input, not both");
5174
+ }
5175
+ if (options.data) {
5176
+ try {
5177
+ data = JSON.parse(options.data);
5178
+ } catch {
5179
+ throw new MutagentError("INVALID_JSON", "Invalid JSON in --data flag", `Provide a valid JSON object, e.g., '{"name":"new-name","systemPrompt":"Updated prompt"}'`);
5180
+ }
5181
+ if (options.name)
5182
+ data.name = options.name;
5183
+ if (options.systemPrompt)
5184
+ data.systemPrompt = options.systemPrompt;
5185
+ if (options.model)
5186
+ data.model = options.model;
5187
+ if (options.description)
5188
+ data.description = options.description;
5189
+ if (options.status)
5190
+ data.status = options.status;
5191
+ } else if (options.file) {
5192
+ const content = readFileSync9(options.file, "utf-8");
5271
5193
  data = JSON.parse(content);
5272
5194
  } else {
5273
5195
  if (options.name)
@@ -5282,7 +5204,7 @@ Examples:
5282
5204
  data.status = options.status;
5283
5205
  }
5284
5206
  if (Object.keys(data).length === 0) {
5285
- throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --file, --name, --system-prompt, --model, --description, or --status");
5207
+ throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --data, --file, --name, --system-prompt, --model, --description, or --status");
5286
5208
  }
5287
5209
  const client = getSDKClient();
5288
5210
  const agent = await client.updateAgent(id, data);
@@ -5323,12 +5245,15 @@ ${chalk10.dim("Tip: Use --force to skip confirmation (required for non-interacti
5323
5245
  handleError(error, isJson);
5324
5246
  }
5325
5247
  });
5326
- const conversations = agents.command("conversations").description("Manage conversations for agents").addHelpText("after", `
5248
+ const conversations = new Command6("conversations").description("Manage conversations for agents").addHelpText("after", `
5327
5249
  Examples:
5328
5250
  ${chalk10.dim("$")} mutagent agents conversations list <agent-id>
5329
5251
  ${chalk10.dim("$")} mutagent agents conversations create <agent-id>
5330
5252
  ${chalk10.dim("$")} mutagent agents conversations messages <agent-id> <conversation-id>
5331
- `);
5253
+ `).action(() => {
5254
+ conversations.help();
5255
+ });
5256
+ agents.addCommand(conversations);
5332
5257
  conversations.command("list").description("List conversations for an agent").argument("<agent-id>", "Agent ID").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").addHelpText("after", `
5333
5258
  Examples:
5334
5259
  ${chalk10.dim("$")} mutagent agents conversations list <agent-id>
@@ -5477,6 +5402,8 @@ Examples:
5477
5402
  init_config();
5478
5403
  import { Command as Command7 } from "commander";
5479
5404
  import chalk11 from "chalk";
5405
+ init_errors();
5406
+ var VALID_CONFIG_KEYS = ["apiKey", "endpoint", "format", "timeout", "defaultWorkspace", "defaultOrganization"];
5480
5407
  function createConfigCommand() {
5481
5408
  const config = new Command7("config").description("Manage CLI configuration").addHelpText("after", `
5482
5409
  Examples:
@@ -5509,21 +5436,27 @@ ${chalk11.dim("Keys: apiKey, endpoint, format, timeout, defaultWorkspace, defaul
5509
5436
  `).action((key) => {
5510
5437
  const isJson = getJsonFlag(config);
5511
5438
  const output = new OutputFormatter(isJson ? "json" : "table");
5512
- const cfg = loadConfig();
5513
- if (!(key in cfg)) {
5514
- output.error(`Unknown config key: ${key}`);
5515
- process.exit(1);
5516
- }
5517
- const value = cfg[key];
5518
- const stringValue = typeof value === "object" ? JSON.stringify(value) : String(value ?? "");
5519
- const displayValue = key === "apiKey" && value ? `${stringValue.slice(0, 8)}...` : stringValue;
5520
- if (isJson) {
5521
- output.output({ [key]: displayValue });
5522
- } else {
5523
- output.info(displayValue);
5439
+ try {
5440
+ const cfg = loadConfig();
5441
+ if (!(key in cfg)) {
5442
+ throw new MutagentError("INVALID_ARGUMENTS", `Unknown config key: "${key}"`, `Valid keys: ${VALID_CONFIG_KEYS.join(", ")}
5443
+ ` + "Example: mutagent config get endpoint");
5444
+ }
5445
+ const value = cfg[key];
5446
+ const stringValue = typeof value === "object" ? JSON.stringify(value) : String(value ?? "");
5447
+ const displayValue = key === "apiKey" && value ? `${stringValue.slice(0, 8)}...` : stringValue;
5448
+ if (isJson) {
5449
+ output.output({ [key]: displayValue });
5450
+ } else {
5451
+ output.info(displayValue);
5452
+ }
5453
+ } catch (error) {
5454
+ handleError(error, isJson);
5524
5455
  }
5525
5456
  });
5526
- const set = new Command7("set").description("Set configuration value");
5457
+ const set = new Command7("set").description("Set configuration value").action(() => {
5458
+ set.help();
5459
+ });
5527
5460
  set.command("workspace").description("Set default workspace ID").argument("<id>", "Workspace ID to set as default").addHelpText("after", `
5528
5461
  Examples:
5529
5462
  ${chalk11.dim("$")} mutagent config set workspace <workspace-id>
@@ -5562,7 +5495,7 @@ ${chalk11.dim("Persists organization ID for org-scoped API keys.")}
5562
5495
  init_sdk_client();
5563
5496
  import { Command as Command8 } from "commander";
5564
5497
  import chalk12 from "chalk";
5565
- import { readFileSync as readFileSync12 } from "fs";
5498
+ import { readFileSync as readFileSync10 } from "fs";
5566
5499
  init_errors();
5567
5500
  function parseSSELine(line) {
5568
5501
  if (!line || line.startsWith(":")) {
@@ -5615,12 +5548,13 @@ Examples:
5615
5548
  ${chalk12.dim("$")} mutagent playground run <prompt-id> --system "You are helpful" --human "Hello"
5616
5549
  ${chalk12.dim("$")} mutagent playground run <prompt-id> --input '{}' --model gpt-4-turbo --json
5617
5550
 
5618
- Input Methods (pick one):
5551
+ Input Methods (pick one, priority order):
5552
+ --system/--human Quick system + user message ${chalk12.green("(recommended)")}
5619
5553
  --input '{"key":"value"}' Inline JSON variables
5620
- --file input.json Load from JSON file
5621
- --system/--human Quick system + user message
5622
5554
  --messages '[...]' Full messages array
5555
+ --file input.json Load from JSON file
5623
5556
 
5557
+ ${chalk12.yellow("Note: Prefer --system/--human or --input over --file to avoid stale files living in your repo.")}
5624
5558
  ${chalk12.dim(`Hint: Test before evaluating: mutagent playground run <id> --input '{"key":"value"}'`)}
5625
5559
  `).action(async (promptId, options) => {
5626
5560
  const isJson = getJsonFlag(playground);
@@ -5724,7 +5658,7 @@ function parsePromptStyleInput(options) {
5724
5658
  function getInputJson(options) {
5725
5659
  if (options.file) {
5726
5660
  try {
5727
- return readFileSync12(options.file, "utf-8");
5661
+ return readFileSync10(options.file, "utf-8");
5728
5662
  } catch (err) {
5729
5663
  throw new MutagentError("FILE_READ_ERROR", `Failed to read input file: ${options.file}`, err instanceof Error ? err.message : "Check the file path and permissions");
5730
5664
  }
@@ -5952,6 +5886,10 @@ Subcommands:
5952
5886
  list, get, test
5953
5887
 
5954
5888
  Note: Provider management (create, update, delete) is available in the Admin Panel only.
5889
+
5890
+ ${chalk14.yellow("Note:")} The providers module is not yet active. This is a placeholder
5891
+ for future external provider configuration. The server currently uses
5892
+ built-in provider settings.
5955
5893
  `);
5956
5894
  providers.command("list").description("List all providers").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").option("-t, --type <type>", "Filter by provider type").addHelpText("after", `
5957
5895
  Examples:
@@ -5980,6 +5918,8 @@ Examples:
5980
5918
  }));
5981
5919
  output.output({ ...result, data: withLinks });
5982
5920
  } else {
5921
+ console.log(chalk14.yellow("Note: The providers module is not yet active. This is a placeholder for future external provider configuration."));
5922
+ console.log("");
5983
5923
  if (result.data.length === 0) {
5984
5924
  output.info("No providers configured.");
5985
5925
  output.info(`Configure at: ${providerSettingsLink()}`);
@@ -6013,6 +5953,8 @@ Examples:
6013
5953
  if (isJson) {
6014
5954
  output.output({ ...provider, _links: providerLinks(provider.id) });
6015
5955
  } else {
5956
+ console.log(chalk14.yellow("Note: The providers module is not yet active. This is a placeholder for future external provider configuration."));
5957
+ console.log("");
6016
5958
  const formatted = {
6017
5959
  id: provider.id,
6018
5960
  name: provider.name,
@@ -6042,6 +5984,8 @@ ${chalk14.dim("Tests connectivity and lists available models for the provider.")
6042
5984
  try {
6043
5985
  const client = getSDKClient();
6044
5986
  if (!isJson) {
5987
+ console.log(chalk14.yellow("Note: The providers module is not yet active. This is a placeholder for future external provider configuration."));
5988
+ console.log("");
6045
5989
  output.info(`Testing provider ${id}...`);
6046
5990
  }
6047
5991
  const result = await client.testProvider(id);
@@ -6077,7 +6021,7 @@ init_config();
6077
6021
  import { Command as Command11 } from "commander";
6078
6022
  import inquirer3 from "inquirer";
6079
6023
  import chalk15 from "chalk";
6080
- import { existsSync as existsSync12, readFileSync as readFileSync13, writeFileSync as writeFileSync4 } from "fs";
6024
+ import { existsSync as existsSync10, mkdirSync as mkdirSync3, readFileSync as readFileSync11, writeFileSync as writeFileSync4 } from "fs";
6081
6025
  import { execSync as execSync2 } from "child_process";
6082
6026
  import { join as join5 } from "path";
6083
6027
  init_errors();
@@ -6130,16 +6074,16 @@ var FRAMEWORK_DETECTION_MAP = {
6130
6074
  }
6131
6075
  };
6132
6076
  function detectPackageManager2(cwd = process.cwd()) {
6133
- if (existsSync12(join5(cwd, "bun.lockb")) || existsSync12(join5(cwd, "bun.lock"))) {
6077
+ if (existsSync10(join5(cwd, "bun.lockb")) || existsSync10(join5(cwd, "bun.lock"))) {
6134
6078
  return "bun";
6135
6079
  }
6136
- if (existsSync12(join5(cwd, "pnpm-lock.yaml"))) {
6080
+ if (existsSync10(join5(cwd, "pnpm-lock.yaml"))) {
6137
6081
  return "pnpm";
6138
6082
  }
6139
- if (existsSync12(join5(cwd, "yarn.lock"))) {
6083
+ if (existsSync10(join5(cwd, "yarn.lock"))) {
6140
6084
  return "yarn";
6141
6085
  }
6142
- if (existsSync12(join5(cwd, "package-lock.json"))) {
6086
+ if (existsSync10(join5(cwd, "package-lock.json"))) {
6143
6087
  return "npm";
6144
6088
  }
6145
6089
  try {
@@ -6161,12 +6105,12 @@ function getInstallCommand2(pm, packages) {
6161
6105
  }
6162
6106
  function detectFrameworkFromPackageJson(cwd = process.cwd()) {
6163
6107
  const pkgPath = join5(cwd, "package.json");
6164
- if (!existsSync12(pkgPath)) {
6108
+ if (!existsSync10(pkgPath)) {
6165
6109
  return null;
6166
6110
  }
6167
6111
  let pkg;
6168
6112
  try {
6169
- pkg = JSON.parse(readFileSync13(pkgPath, "utf-8"));
6113
+ pkg = JSON.parse(readFileSync11(pkgPath, "utf-8"));
6170
6114
  } catch {
6171
6115
  return null;
6172
6116
  }
@@ -6182,7 +6126,7 @@ function detectFrameworkFromPackageJson(cwd = process.cwd()) {
6182
6126
  return null;
6183
6127
  }
6184
6128
  function hasRcConfig(cwd = process.cwd()) {
6185
- return existsSync12(join5(cwd, ".mutagentrc.json"));
6129
+ return existsSync10(join5(cwd, ".mutagentrc.json"));
6186
6130
  }
6187
6131
  function writeRcConfig(config, cwd = process.cwd()) {
6188
6132
  const rcPath = join5(cwd, ".mutagentrc.json");
@@ -6350,6 +6294,32 @@ Modes:
6350
6294
  output.info('Ready! Run "mutagent --help" to see available commands.');
6351
6295
  }
6352
6296
  }
6297
+ if (!isNonInteractive) {
6298
+ const skillPath = join5(cwd, ".claude/skills/mutagent-cli/SKILL.md");
6299
+ if (!existsSync10(skillPath)) {
6300
+ const { installSkill } = await inquirer3.prompt([{
6301
+ type: "confirm",
6302
+ name: "installSkill",
6303
+ message: "Install MutagenT skill for Claude Code? (Teaches AI agents how to use the CLI)",
6304
+ default: true
6305
+ }]);
6306
+ if (installSkill) {
6307
+ try {
6308
+ const skillDir = join5(cwd, ".claude/skills/mutagent-cli");
6309
+ if (!existsSync10(skillDir)) {
6310
+ mkdirSync3(skillDir, { recursive: true });
6311
+ }
6312
+ execSync2("node " + join5(cwd, "node_modules/.bin/mutagent") + " skills install", {
6313
+ cwd,
6314
+ stdio: "ignore"
6315
+ });
6316
+ output.success("Installed MutagenT CLI skill for Claude Code");
6317
+ } catch {
6318
+ output.info("Install skill manually: mutagent skills install");
6319
+ }
6320
+ }
6321
+ }
6322
+ }
6353
6323
  if (isJson) {
6354
6324
  output.output({
6355
6325
  success: true,
@@ -6478,7 +6448,7 @@ Scanning ${scanPath}...
6478
6448
  // src/commands/skills.ts
6479
6449
  import { Command as Command13 } from "commander";
6480
6450
  import chalk17 from "chalk";
6481
- import { existsSync as existsSync13, mkdirSync as mkdirSync3, writeFileSync as writeFileSync5 } from "fs";
6451
+ import { existsSync as existsSync11, mkdirSync as mkdirSync4, writeFileSync as writeFileSync5 } from "fs";
6482
6452
  import { join as join6 } from "path";
6483
6453
  var SKILL_FRONTMATTER = `---
6484
6454
  name: mutagent-cli
@@ -6493,6 +6463,22 @@ var SKILL_BODY = `# MutagenT CLI Skill
6493
6463
 
6494
6464
  ## Quick Reference
6495
6465
  Run \`mutagent --help\` for full command list and examples.
6466
+ Run \`mutagent <command> --help\` for on-the-fly instructions before using any command.
6467
+
6468
+ Key commands:
6469
+ - \`mutagent auth login\` — Authenticate
6470
+ - \`mutagent prompts list\` — List prompts
6471
+ - \`mutagent usage\` — Check plan limits and remaining optimization runs
6472
+ - \`mutagent explore\` — Discover prompts in codebase
6473
+ - \`mutagent integrate <framework>\` — Framework integration instructions
6474
+
6475
+ ## Agent Usage Guidelines
6476
+
6477
+ - **ALWAYS** run \`mutagent prompts list --json\` and \`mutagent usage --json\` BEFORE creating or modifying resources
6478
+ - **ALWAYS** use the \`--json\` flag for machine-readable output
6479
+ - **ALWAYS** confirm with the user before write operations (create, update, delete, optimize)
6480
+ - **CHECK** \`mutagent usage\` before running \`optimize start\` to verify remaining optimization runs
6481
+ - Format dashboard links as markdown: \`[View on Dashboard](url)\`
6496
6482
 
6497
6483
  ## Post-Onboarding Decision Tree
6498
6484
 
@@ -6511,8 +6497,9 @@ After \`mutagent auth login\`, the user lands in one of 3 paths:
6511
6497
  3. \`mutagent prompts create --data '{...}'\` — Upload with REQUIRED outputSchema
6512
6498
  4. Guided eval creator OR \`mutagent prompts evaluation create\`
6513
6499
  5. \`mutagent prompts dataset add\` — Upload/curate datasets (named)
6514
- 6. \`mutagent prompts optimize start\` — Run optimization
6515
- 7. Review scorecard Apply/Reject optimized prompt
6500
+ 6. \`mutagent usage --json\` — Verify remaining optimization runs
6501
+ 7. \`mutagent prompts optimize start\` Run optimization
6502
+ 8. Review scorecard → Apply/Reject optimized prompt
6516
6503
 
6517
6504
  ### Path C: Manual
6518
6505
  User uses CLI commands directly. Run \`mutagent --help\`.
@@ -6585,8 +6572,8 @@ that teaches coding agents how to use the MutagenT CLI effectively.
6585
6572
  const output = new OutputFormatter(isJson ? "json" : "table");
6586
6573
  const skillDir = join6(process.cwd(), SKILL_DIR);
6587
6574
  const skillPath = join6(skillDir, SKILL_FILE);
6588
- if (!existsSync13(skillDir)) {
6589
- mkdirSync3(skillDir, { recursive: true });
6575
+ if (!existsSync11(skillDir)) {
6576
+ mkdirSync4(skillDir, { recursive: true });
6590
6577
  }
6591
6578
  const content = `${SKILL_FRONTMATTER}
6592
6579
 
@@ -6610,16 +6597,120 @@ ${SKILL_BODY}
6610
6597
  return skills;
6611
6598
  }
6612
6599
 
6600
+ // src/commands/usage.ts
6601
+ init_config();
6602
+ import { Command as Command14 } from "commander";
6603
+ import chalk18 from "chalk";
6604
+ init_errors();
6605
+ init_sdk_client();
6606
+ var TRIAL_OPTIMIZATION_LIMIT = 5;
6607
+ var BILLING_URL = "https://app.mutagent.io/settings/billing";
6608
+ function renderProgressBar(used, limit, width = 30) {
6609
+ const ratio = Math.min(used / limit, 1);
6610
+ const filled = Math.round(ratio * width);
6611
+ const empty = width - filled;
6612
+ const bar = "█".repeat(filled) + "░".repeat(empty);
6613
+ const percent = Math.round(ratio * 100);
6614
+ return `${bar} ${String(percent)}%`;
6615
+ }
6616
+ function createUsageCommand() {
6617
+ const usage = new Command14("usage").description("Show resource counts and optimization run limits").addHelpText("after", `
6618
+ Examples:
6619
+ ${chalk18.dim("$")} mutagent usage
6620
+ ${chalk18.dim("$")} mutagent usage --json
6621
+ `);
6622
+ usage.action(async () => {
6623
+ const isJson = getJsonFlag(usage);
6624
+ const output = new OutputFormatter(isJson ? "json" : "table");
6625
+ try {
6626
+ const apiKey = getApiKey();
6627
+ if (!apiKey) {
6628
+ throw new AuthenticationError;
6629
+ }
6630
+ const spinner = await createSpinner("Fetching usage data...", isJson).start();
6631
+ const client = getSDKClient();
6632
+ const prompts = await client.listPrompts();
6633
+ const promptCount = prompts.length;
6634
+ let datasetCount = 0;
6635
+ let evaluationCount = 0;
6636
+ if (promptCount > 0) {
6637
+ const results = await Promise.all(prompts.map(async (prompt) => {
6638
+ const id = String(prompt.id);
6639
+ const [datasets, evaluations] = await Promise.all([
6640
+ client.listDatasets(id).catch(() => []),
6641
+ client.listEvaluations(id).catch(() => [])
6642
+ ]);
6643
+ return { datasets: datasets.length, evaluations: evaluations.length };
6644
+ }));
6645
+ for (const r of results) {
6646
+ datasetCount += r.datasets;
6647
+ evaluationCount += r.evaluations;
6648
+ }
6649
+ }
6650
+ const optimizationLimit = TRIAL_OPTIMIZATION_LIMIT;
6651
+ const optimizationUsed = 0;
6652
+ const optimizationRemaining = optimizationLimit - optimizationUsed;
6653
+ if (spinner && typeof spinner.stop === "function") {
6654
+ spinner.stop();
6655
+ }
6656
+ if (isJson) {
6657
+ output.output({
6658
+ resources: {
6659
+ prompts: promptCount,
6660
+ datasets: datasetCount,
6661
+ evaluations: evaluationCount
6662
+ },
6663
+ optimizationRuns: {
6664
+ used: optimizationUsed,
6665
+ limit: optimizationLimit,
6666
+ remaining: optimizationRemaining,
6667
+ plan: "trial"
6668
+ },
6669
+ _links: {
6670
+ billing: BILLING_URL
6671
+ }
6672
+ });
6673
+ } else {
6674
+ console.log("");
6675
+ console.log(chalk18.bold("\uD83D\uDCCA MutagenT Usage"));
6676
+ console.log(chalk18.dim("─".repeat(45)));
6677
+ console.log("");
6678
+ console.log(chalk18.bold("Resources:"));
6679
+ console.log(` Prompts: ${chalk18.cyan(String(promptCount))}`);
6680
+ console.log(` Datasets: ${chalk18.cyan(String(datasetCount))}`);
6681
+ console.log(` Evaluations: ${chalk18.cyan(String(evaluationCount))}`);
6682
+ console.log("");
6683
+ console.log(chalk18.bold(`Optimization Runs (${chalk18.yellow("trial")} plan):`));
6684
+ console.log(` Remaining: ${chalk18.cyan(String(optimizationRemaining))} / ${String(optimizationLimit)}`);
6685
+ console.log(` ${renderProgressBar(optimizationUsed, optimizationLimit)}`);
6686
+ console.log("");
6687
+ console.log(chalk18.yellow(` ⚠ ${String(optimizationRemaining)} optimization runs remaining`));
6688
+ console.log(chalk18.dim(` ℹ Optimization run counts are approximate`));
6689
+ console.log(` Upgrade: ${chalk18.underline(BILLING_URL)}`);
6690
+ console.log("");
6691
+ }
6692
+ } catch (error) {
6693
+ handleError(error, isJson);
6694
+ }
6695
+ });
6696
+ return usage;
6697
+ }
6698
+
6613
6699
  // src/bin/cli.ts
6614
6700
  init_config();
6701
+ import { existsSync as existsSync12 } from "fs";
6615
6702
  var cliVersion = "0.1.1";
6616
- try {
6617
- const __dirname2 = dirname(fileURLToPath(import.meta.url));
6618
- const pkgPath = join7(__dirname2, "..", "..", "package.json");
6619
- const pkg = JSON.parse(readFileSync14(pkgPath, "utf-8"));
6620
- cliVersion = pkg.version ?? cliVersion;
6621
- } catch {}
6622
- var program = new Command14;
6703
+ if (process.env.CLI_VERSION) {
6704
+ cliVersion = process.env.CLI_VERSION;
6705
+ } else {
6706
+ try {
6707
+ const __dirname2 = dirname(fileURLToPath(import.meta.url));
6708
+ const pkgPath = join7(__dirname2, "..", "..", "package.json");
6709
+ const pkg = JSON.parse(readFileSync12(pkgPath, "utf-8"));
6710
+ cliVersion = pkg.version ?? cliVersion;
6711
+ } catch {}
6712
+ }
6713
+ var program = new Command15;
6623
6714
  program.name("mutagent").description(`MutagenT CLI - AI-native prompt optimization platform
6624
6715
 
6625
6716
  Documentation: https://docs.mutagent.io/cli
@@ -6628,45 +6719,47 @@ program.name("mutagent").description(`MutagenT CLI - AI-native prompt optimizati
6628
6719
  showGlobalOptions: true
6629
6720
  });
6630
6721
  program.addHelpText("after", `
6631
- ${chalk18.yellow("Non-Interactive Mode (CI/CD & Coding Agents):")}
6632
- Export your API key: ${chalk18.green("export MUTAGENT_API_KEY=mt_your_key_here")}
6633
- Or pass inline: ${chalk18.green("mutagent prompts list --api-key mt_your_key")}
6634
- Machine-readable output: ${chalk18.green("mutagent prompts list --json")}
6635
- Disable prompts: ${chalk18.green("mutagent prompts list --non-interactive")}
6636
- Set default workspace: ${chalk18.green("mutagent config set workspace <workspace-id>")}
6637
- Set default org: ${chalk18.green("mutagent config set org <org-id>")}
6722
+ ${chalk19.yellow("Non-Interactive Mode (CI/CD & Coding Agents):")}
6723
+ Export your API key: ${chalk19.green("export MUTAGENT_API_KEY=mt_your_key_here")}
6724
+ Or pass inline: ${chalk19.green("mutagent prompts list --api-key mt_your_key")}
6725
+ Machine-readable output: ${chalk19.green("mutagent prompts list --json")}
6726
+ Disable prompts: ${chalk19.green("mutagent prompts list --non-interactive")}
6727
+ Set default workspace: ${chalk19.green("mutagent config set workspace <workspace-id>")}
6728
+ Set default org: ${chalk19.green("mutagent config set org <org-id>")}
6638
6729
  `);
6639
6730
  program.addHelpText("after", `
6640
- ${chalk18.yellow("Workflows:")}
6641
- ${chalk18.bold("Evaluate → Optimize Loop:")}
6731
+ ${chalk19.yellow("Workflows:")}
6732
+ ${chalk19.bold("Evaluate → Optimize Loop:")}
6642
6733
  1. mutagent prompts create --name "..." --raw-file prompt.txt
6643
6734
  2. mutagent prompts dataset add <prompt-id> --name "..." --file data.json
6644
6735
  3. mutagent prompts evaluation create <prompt-id> --name "..." --file criteria.json
6645
6736
  4. mutagent prompts optimize start <prompt-id> --dataset <id> --max-iterations 3
6646
6737
 
6647
- ${chalk18.bold("Quick Test:")}
6738
+ ${chalk19.bold("Quick Test:")}
6648
6739
  mutagent playground run <prompt-id> --input '{"key":"value"}'
6649
6740
 
6650
- ${chalk18.bold("Prerequisites for Optimization:")}
6651
- ${chalk18.green("✓")} Prompt with input/output parameters
6652
- ${chalk18.green("✓")} Dataset with items (input + expectedOutput pairs)
6653
- ${chalk18.green("✓")} Evaluation with criteria (field-level, input/output focused)
6654
- ${chalk18.dim("•")} LLM provider ${chalk18.dim("(only when server uses external providers)")}
6741
+ ${chalk19.bold("Prerequisites for Optimization:")}
6742
+ ${chalk19.green("✓")} Prompt with input/output parameters
6743
+ ${chalk19.green("✓")} Dataset with items (input + expectedOutput pairs)
6744
+ ${chalk19.green("✓")} Evaluation with criteria (field-level, input/output focused)
6745
+ ${chalk19.dim("•")} LLM provider ${chalk19.dim("(only when server uses external providers)")}
6655
6746
  `);
6656
6747
  program.addHelpText("after", `
6657
- ${chalk18.cyan("┌─ AI AGENT INTEGRATION HINT ────────────────────────────────────────────────┐")}
6658
- ${chalk18.cyan("│")} ${chalk18.cyan("│")}
6659
- ${chalk18.cyan("│")} Frameworks: mastra, langchain, langgraph, vercel-ai, claude-code, generic ${chalk18.cyan("│")}
6660
- ${chalk18.cyan("│")} ${chalk18.cyan("│")}
6661
- ${chalk18.cyan("│")} Get integration guide: mutagent integrate <framework> ${chalk18.cyan("│")}
6662
- ${chalk18.cyan("│")} Verify setup: mutagent integrate <framework> --verify ${chalk18.cyan("│")}
6663
- ${chalk18.cyan("│")} Use --json for AI parsing: mutagent <command> --json ${chalk18.cyan("│")}
6664
- ${chalk18.cyan("│")} ${chalk18.cyan("│")}
6665
- ${chalk18.cyan("└────────────────────────────────────────────────────────────────────────────┘")}
6666
- ${!hasCredentials() ? chalk18.yellow(`
6748
+ ${chalk19.cyan("┌─ AI AGENT INTEGRATION HINT ────────────────────────────────────────────────┐")}
6749
+ ${chalk19.cyan("│")} ${chalk19.cyan("│")}
6750
+ ${chalk19.cyan("│")} Frameworks: langchain, langgraph, vercel-ai, openai ${chalk19.cyan("│")}
6751
+ ${chalk19.cyan("│")} ${chalk19.cyan("│")}
6752
+ ${chalk19.cyan("│")} Get integration guide: mutagent integrate <framework> ${chalk19.cyan("│")}
6753
+ ${chalk19.cyan("│")} Verify setup: mutagent integrate <framework> --verify ${chalk19.cyan("│")}
6754
+ ${chalk19.cyan("│")} Use --json for AI parsing: mutagent <command> --json ${chalk19.cyan("│")}
6755
+ ${chalk19.cyan("│")} ${chalk19.cyan("│")}
6756
+ ${chalk19.cyan("└────────────────────────────────────────────────────────────────────────────┘")}
6757
+ ${!hasCredentials() ? chalk19.yellow(`
6667
6758
  Warning: Not authenticated. Run: mutagent auth login --browser
6668
- `) : ""}${!hasRcConfig() ? chalk18.green(`
6759
+ `) : ""}${!hasRcConfig() ? chalk19.green(`
6669
6760
  Get started: mutagent init
6761
+ `) : ""}${!existsSync12(join7(process.cwd(), ".claude/skills/mutagent-cli/SKILL.md")) ? chalk19.magenta(`
6762
+ Using Claude Code? Install the MutagenT skill: mutagent skills install
6670
6763
  `) : ""}`);
6671
6764
  program.hook("preAction", (thisCommand) => {
6672
6765
  const globalOpts = thisCommand.optsWithGlobals();
@@ -6693,7 +6786,8 @@ program.addCommand(createWorkspacesCommand());
6693
6786
  program.addCommand(createProvidersCommand());
6694
6787
  program.addCommand(createExploreCommand());
6695
6788
  program.addCommand(createSkillsCommand());
6789
+ program.addCommand(createUsageCommand());
6696
6790
  program.parse();
6697
6791
 
6698
- //# debugId=A683D38662CECE3C64756E2164756E21
6792
+ //# debugId=88B4F3961CA6D43E64756E2164756E21
6699
6793
  //# sourceMappingURL=cli.js.map