@mutagent/cli 0.1.123 → 0.1.125

package/dist/bin/cli.js

@@ -1242,7 +1242,7 @@ var init_sdk_client = __esm(() => {
 // src/bin/cli.ts
 import { Command as Command21 } from "commander";
 import chalk39 from "chalk";
-import { readFileSync as readFileSync12 } from "fs";
+import { readFileSync as readFileSync12, existsSync as existsSync14 } from "fs";
 import { join as join10, dirname as dirname3 } from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
 
@@ -1691,6 +1691,33 @@ var SCHEMA_PATTERN = /["']?(?:inputSchema|outputSchema|properties|required)["']?
 var ZOD_PROMPT_SCHEMA_PATTERN = /(?:const|let|var|export)\s+\w*(?:prompt|output|input|response|schema|message)\w*\s*=\s*z\.object\s*\(/i;
 var PYDANTIC_PATTERN = /class\s+\w+\s*\(\s*(?:BaseModel|BaseSettings)\s*\)/;
 var MARKER_START_PATTERN = /MutagenT:START\s+(\w+)(?:\s+id=(\S+))?/;
+var AI_FRAMEWORK_IMPORT_PATTERNS = [
+  { pattern: /from\s+['"]@langchain\/core(?:\/[^'"]*)?['"]/, reason: "langchain-core-import" },
+  { pattern: /from\s+['"]langchain(?:\/[^'"]*)?['"]/, reason: "langchain-import" },
+  { pattern: /from\s+['"]@langchain\/langgraph(?:\/[^'"]*)?['"]/, reason: "langgraph-import" },
+  { pattern: /from\s+['"]langgraph(?:\/[^'"]*)?['"]/, reason: "langgraph-import" },
+  { pattern: /from\s+['"]openai(?:\/[^'"]*)?['"]/, reason: "openai-sdk-import" },
+  { pattern: /from\s+['"]@anthropic-ai\/sdk(?:\/[^'"]*)?['"]/, reason: "anthropic-sdk-import" },
+  { pattern: /from\s+['"]anthropic(?:\/[^'"]*)?['"]/, reason: "anthropic-import" },
+  { pattern: /from\s+['"]@ai-sdk\/[^'"]+['"]/, reason: "ai-sdk-import" },
+  { pattern: /from\s+['"]ai['"]/, reason: "vercel-ai-import" },
+  { pattern: /from\s+['"]@mastra\/[^'"]+['"]/, reason: "mastra-import" },
+  { pattern: /from\s+['"]crewai(?:\/[^'"]*)?['"]/, reason: "crewai-import" },
+  { pattern: /from\s+['"]@openai\/agents(?:\/[^'"]*)?['"]/, reason: "openai-agents-sdk" },
+  { pattern: /from\s+['"]autogen(?:_agentchat)?(?:\/[^'"]*)?['"]/, reason: "autogen-import" }
+];
+var LOW_CONFIDENCE_PATH_SEGMENTS = ["scripts/", "tools/", "config/", "__tests__/", "tests/", "fixtures/", "__mocks__/"];
+function isLowConfidencePath(relativePath) {
+  const normalized = relativePath.replace(/\\/g, "/");
+  return LOW_CONFIDENCE_PATH_SEGMENTS.some((seg) => normalized.includes(seg));
+}
+function detectAIFrameworkImport(content) {
+  for (const { pattern, reason } of AI_FRAMEWORK_IMPORT_PATTERNS) {
+    if (pattern.test(content))
+      return reason;
+  }
+  return null;
+}
 var AGENT_HIGH_PATTERNS = [
   { pattern: /\bAgentExecutor\b/, reason: "langchain-agent-executor" },
   { pattern: /\bcreate(?:React|OpenAIFunctions|Tool(?:Calling)?|Structured(?:Chat)?)Agent\b/, reason: "langchain-create-agent" },
@@ -1757,6 +1784,8 @@ function walkDir(dir, extensions, excludeDirs, maxDepth, currentDepth = 0) {
       continue;
     }
     if (stat.isDirectory()) {
+      if (entry === ".claude")
+        continue;
       if (excludeDirs.includes(entry))
         continue;
       files.push(...walkDir(fullPath, extensions, excludeDirs, maxDepth, currentDepth + 1));
@@ -1778,6 +1807,8 @@ function scanForPrompts(filePath, relativePath) {
     return results;
   }
   const fileDelimiter = inferPromptVariables(content).delimiter;
+  const aiFrameworkReason = detectAIFrameworkImport(content);
+  const pathPenalty = isLowConfidencePath(relativePath);
   const lines = content.split(`
 `);
   for (let i = 0;i < lines.length; i++) {
@@ -1787,24 +1818,26 @@ function scanForPrompts(filePath, relativePath) {
     const inferred = inferPromptVariables(line);
     if (inferred.variables.length > 0) {
       const preview = line.trim().substring(0, 80);
+      const confidence = pathPenalty ? "low" : aiFrameworkReason ? "high" : "high";
       results.push({
         file: relativePath,
         line: i + 1,
         preview,
-        reason: "template-variable",
-        confidence: "high",
+        reason: aiFrameworkReason ? `template-variable+${aiFrameworkReason}` : "template-variable",
+        confidence,
         delimiter: fileDelimiter
       });
     }
     if (PROMPT_NAME_PATTERN.test(line)) {
       const preview = line.trim().substring(0, 80);
       if (!results.some((r) => r.file === relativePath && r.line === i + 1)) {
+        const confidence = pathPenalty ? "low" : "medium";
         results.push({
           file: relativePath,
           line: i + 1,
           preview,
           reason: "prompt-constant",
-          confidence: "medium",
+          confidence,
           delimiter: fileDelimiter
         });
       }
@@ -1825,12 +1858,13 @@ function scanForPrompts(filePath, relativePath) {
     if (PYDANTIC_PATTERN.test(line)) {
       const preview = line.trim().substring(0, 80);
       if (!results.some((r) => r.file === relativePath && r.line === i + 1)) {
+        const confidence = pathPenalty ? "low" : "medium";
         results.push({
           file: relativePath,
           line: i + 1,
           preview,
           reason: "pydantic-model",
-          confidence: "medium",
+          confidence,
           delimiter: fileDelimiter
         });
       }
@@ -1846,6 +1880,8 @@ function scanForAgents(filePath, relativePath) {
   } catch {
     return results;
   }
+  const pathPenalty = isLowConfidencePath(relativePath);
+  const aiFrameworkReason = detectAIFrameworkImport(content);
   const lines = content.split(`
 `);
   const seen = new Set;
@@ -1860,7 +1896,7 @@ function scanForAgents(filePath, relativePath) {
           line: i + 1,
           preview: line.trim().substring(0, 80),
           reason,
-          confidence: "high"
+          confidence: pathPenalty ? "low" : "high"
         });
         seen.add(i);
         break;
@@ -1875,7 +1911,7 @@ function scanForAgents(filePath, relativePath) {
           line: i + 1,
           preview: line.trim().substring(0, 80),
           reason,
-          confidence: "medium"
+          confidence: pathPenalty ? "low" : aiFrameworkReason ? "high" : "medium"
         });
         seen.add(i);
         break;
@@ -5929,6 +5965,41 @@ ${chalk15.dim('Jobs with status "completed" or "cancelled" cannot be cancelled a
 }
 
 // src/commands/prompts/optimize.ts
+var MODEL_FAMILIES = [
+  {
+    providerTypes: ["openai", "azure"],
+    prefixes: ["gpt-"],
+    examples: ["gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano", "gpt-5.2"]
+  },
+  {
+    providerTypes: ["anthropic"],
+    prefixes: ["claude-"],
+    examples: ["claude-sonnet-4-6", "claude-opus-4-6", "claude-sonnet-4-5", "claude-opus-4-5", "claude-haiku-4-5"]
+  },
+  {
+    providerTypes: ["bedrock"],
+    prefixes: ["anthropic.claude-"],
+    examples: ["anthropic.claude-sonnet-4-5-20250929-v1:0", "anthropic.claude-opus-4-6-v1", "anthropic.claude-haiku-4-5-20251001-v1:0"]
+  },
+  {
+    providerTypes: ["google"],
+    prefixes: ["gemini-"],
+    examples: ["gemini-3.1-pro-preview", "gemini-3-flash-preview", "gemini-3.1-flash-lite-preview"]
+  },
+  {
+    providerTypes: ["deepseek"],
+    prefixes: ["deepseek-"],
+    examples: ["deepseek-chat", "deepseek-reasoner"]
+  },
+  {
+    providerTypes: ["xai"],
+    prefixes: ["grok-"],
+    examples: ["grok-4.20-0309-reasoning", "grok-4-1-fast-reasoning", "grok-4-1-fast-non-reasoning"]
+  }
+];
+function getModelFamiliesForProviderTypes(providerTypes) {
+  return MODEL_FAMILIES.filter((family) => family.providerTypes.some((pt) => providerTypes.includes(pt)));
+}
 function registerOptimizeCommands(prompts) {
   const optimize = new Command5("optimize").description("Manage prompt optimization jobs").addHelpText("after", `
 Examples:
@@ -5953,6 +6024,10 @@ ${chalk16.yellow("Pre-Optimization Checklist (auto-validated by preflight):")}
   □ Dataset items         ${chalk16.dim("(warns if expectedOutput missing)")}
   □ Criteria ↔ Schema     ${chalk16.dim("(warns if criteria reference unknown fields)")}
 
+${chalk16.yellow("Required: at least one BYOK provider must be configured.")}
+  Run ${chalk16.cyan("mutagent providers list")} to verify your configured providers.
+  Add one: ${chalk16.cyan("mutagent providers add --help")}
+
 ${PREREQUISITES_TEXT}
 
 ${chalk16.dim("Monitor progress with: mutagent prompts optimize status <job-id>")}
@@ -5969,6 +6044,31 @@ ${chalk16.yellow("AI Agent: ALWAYS append --json to this command.")}
     try {
       await checkProviderConfigured();
       const client = await getSDKClient();
+      if (!options.model && !isJson) {
+        output.warn("No --model specified. Server will use default. Pass --model <id> to control costs.");
+      }
+      if (options.model) {
+        try {
+          const providers = await client.listProviders();
+          if (providers.data && providers.data.length > 0) {
+            const configuredTypes = [...new Set(providers.data.map((p) => p.type))];
+            const modelFamilies = getModelFamiliesForProviderTypes(configuredTypes);
+            const modelLower = options.model.toLowerCase();
+            const matchedFamily = modelFamilies.find(({ prefixes }) => prefixes.some((prefix) => modelLower.startsWith(prefix)));
+            if (!matchedFamily) {
+              const supportedModels = modelFamilies.flatMap((f) => f.examples).join(", ");
+              const errorMsg = `Model '${options.model}' is not supported by any configured provider. ` + `Your providers: [${configuredTypes.join(", ")}]. ` + `Supported models: ${supportedModels || "(unknown — check mutagent providers list --json)"}. ` + `Run: mutagent providers list --json`;
+              if (isJson) {
+                output.output({ success: false, error: errorMsg, code: "MODEL_NOT_SUPPORTED" });
+                return;
+              }
+              output.error(errorMsg);
+              process.exitCode = 1;
+              return;
+            }
+          }
+        } catch {}
+      }
       if (!isJson) {
         output.info("Running pre-flight checks...");
       }
@@ -9150,8 +9250,7 @@ Modes:
                 "",
                 "If the user picks option 1, run BOTH commands:",
                 "  mutagent skills install --json",
-                "Then add this hooks config to .claude/settings.local.json:",
-                '  {"hooks":{"SessionStart":[{"matcher":"startup","hooks":[{"type":"command","command":"mutagent hooks claude-code session-start"}]}],"Stop":[{"hooks":[{"type":"command","command":"mutagent hooks claude-code session-end"}]}],"PreToolUse":[{"hooks":[{"type":"command","command":"mutagent hooks claude-code pre-tool-use"}]}],"PostToolUse":[{"hooks":[{"type":"command","command":"mutagent hooks claude-code post-tool-use"}]}]}}',
+                "  mutagent hooks install --json",
                 "",
                 "If the user picks option 2, run only: mutagent skills install --json"
               ].join(`
@@ -9174,8 +9273,9 @@ import { Command as Command16 } from "commander";
 import chalk35 from "chalk";
 import { resolve as resolve3 } from "path";
 init_errors();
+var DEFAULT_EXCLUDE_DIRS = "node_modules,dist,.git,build,.next,__pycache__,venv,.venv,.claude,scripts,__tests__,tests,fixtures,__mocks__,.github";
 function createExploreCommand() {
-  const explore = new Command16("explore").description("Scan codebase for prompts, datasets, and MutagenT markers").option("-p, --path <dir>", "Directory to scan", ".").option("--depth <n>", "Max directory depth", "10").option("--include <glob>", "Include file pattern", "**/*.{ts,js,py,tsx,jsx}").option("--exclude <dirs>", "Comma-separated directories to exclude", "node_modules,dist,.git,build,.next,__pycache__,venv,.venv").option("--markers-only", "Only find existing MutagenT markers").addHelpText("after", `
+  const explore = new Command16("explore").description("Scan codebase for prompts, datasets, and MutagenT markers").option("-p, --path <dir>", "Directory to scan", ".").option("--depth <n>", "Max directory depth", "10").option("--include <glob>", "Include file pattern", "**/*.{ts,js,py,tsx,jsx}").option("--exclude <dirs>", "Comma-separated directories to exclude", DEFAULT_EXCLUDE_DIRS).option("--markers-only", "Only find existing MutagenT markers").option("--min-confidence <level>", "Minimum confidence level to include in output (high|medium|low)", "medium").addHelpText("after", `
 Examples:
   ${chalk35.dim("$")} mutagent explore
   ${chalk35.dim("$")} mutagent explore --path ./src
@@ -9208,8 +9308,13 @@ ${chalk35.dim("Results are saved to .mutagent/mutation-context.md for use by oth
       const scanPath = resolve3(options.path ?? ".");
       const depth = parseInt(options.depth ?? "10", 10);
       const extensions = parseExtensions(options.include ?? "**/*.{ts,js,py,tsx,jsx}");
-      const excludeDirs = parseExcludeDirs(options.exclude ?? "node_modules,dist,.git,build,.next,__pycache__,venv,.venv");
+      const excludeDirs = parseExcludeDirs(options.exclude ?? DEFAULT_EXCLUDE_DIRS);
       const markersOnly = options.markersOnly ?? false;
+      const minConfidenceRaw = options.minConfidence ?? "medium";
+      const validConfidenceLevels = ["high", "medium", "low"];
+      const minConfidence = validConfidenceLevels.includes(minConfidenceRaw) ? minConfidenceRaw : "medium";
+      const confidenceRank = { high: 2, medium: 1, low: 0 };
+      const filterByConfidence = (items) => items.filter((item) => confidenceRank[item.confidence] >= confidenceRank[minConfidence]);
       const exploreOpts = {
         path: scanPath,
         depth,
@@ -9222,7 +9327,12 @@ ${chalk35.dim("Results are saved to .mutagent/mutation-context.md for use by oth
 Scanning ${scanPath}...
 `));
       }
-      const result = exploreCodebase(exploreOpts);
+      const rawResult = exploreCodebase(exploreOpts);
+      const result = {
+        ...rawResult,
+        prompts: filterByConfidence(rawResult.prompts),
+        agents: filterByConfidence(rawResult.agents)
+      };
       const ctx = MutationContext.load(scanPath);
       for (const prompt of result.prompts) {
         ctx.addDiscoveredPrompt(prompt.file, prompt.line, prompt.preview);
@@ -9333,7 +9443,9 @@ description: |
   Guides coding agents through prompt upload, evaluation creation,
   dataset curation, optimization, and framework integration.
   Triggers: "mutagent", "optimize prompt", "upload prompt", "integrate tracing",
-  "create evaluation", "upload dataset", "explore prompts", "mutagent cli".
+  "create evaluation", "upload dataset", "explore prompts", "mutagent cli",
+  "eval", "dataset", "guided", "how do I optimize", "improve my prompt",
+  "set up tracing", "add observability".
 ---
 
 # MutagenT CLI Skill
@@ -9995,10 +10107,16 @@ Before running explore, understand what the CLI will return:
 
 \`\`\`
 1. mutagent explore --json
-   → surfaces: prompts[], agents[], datasets[], markers[]
-   → show command output to user
+   → quick regex scan: surfaces prompts[], agents[], datasets[], markers[]
+   → this is a STARTING POINT, not the final answer
+
+1b. Use your own Explore/Read/Grep tools to VERIFY each candidate:
+   → Read the actual file — is it really a prompt template or just a string?
+   → Check imports — does it use langchain, openai, @ai-sdk, mastra?
+   → Follow the call chain — single LLM call (prompt) or loop with tools (agent)?
+   → The CLI scan is fast but fragile (regex-based). Your tools understand code.
 
-2. Classify results:
+2. Classify verified results:
    - prompts[]  → optimization-eligible (single-shot, output schema)
    - agents[]   → WIP (multi-turn/tool-calling) — route to [workflows/agents.md](./agents.md)
    - datasets[] → existing local data (uploadable in optimization workflow)
@@ -10570,7 +10688,7 @@ import { Command as Command19 } from "commander";
 import { randomUUID } from "crypto";
 import { join as join8 } from "path";
 import { tmpdir } from "os";
-import { readFileSync as readFileSync10, writeFileSync as writeFileSync6, unlinkSync, existsSync as existsSync13 } from "fs";
+import { readFileSync as readFileSync10, writeFileSync as writeFileSync6, unlinkSync, existsSync as existsSync13, mkdirSync as mkdirSync5 } from "fs";
 async function safeExecute(fn) {
   try {
     await fn();
@@ -10806,6 +10924,59 @@ async function handlePostToolUse() {
     }
   ]);
 }
+var MUTAGENT_HOOKS = {
+  SessionStart: [{ matcher: "startup", hooks: [{ type: "command", command: "mutagent hooks claude-code session-start" }] }],
+  Stop: [{ hooks: [{ type: "command", command: "mutagent hooks claude-code session-end" }] }],
+  PreToolUse: [{ hooks: [{ type: "command", command: "mutagent hooks claude-code pre-tool-use" }] }],
+  PostToolUse: [{ hooks: [{ type: "command", command: "mutagent hooks claude-code post-tool-use" }] }]
+};
+function hasCommand(matchers, command) {
+  return matchers.some((m) => m.hooks.some((h) => h.command === command));
+}
+function installHooks(cwd) {
+  const claudeDir = join8(cwd, ".claude");
+  const settingsPath = join8(claudeDir, "settings.local.json");
+  const existed = existsSync13(settingsPath);
+  let settings = {};
+  if (existed) {
+    try {
+      settings = JSON.parse(readFileSync10(settingsPath, "utf-8"));
+    } catch {
+      settings = {};
+    }
+  }
+  const added = [];
+  const alreadyPresent = [];
+  for (const [event, newMatchers] of Object.entries(MUTAGENT_HOOKS)) {
+    if (!newMatchers)
+      continue;
+    settings.hooks ??= {};
+    settings.hooks[event] ??= [];
+    const existing = settings.hooks[event];
+    for (const matcher of newMatchers) {
+      for (const hook of matcher.hooks) {
+        if (hasCommand(existing, hook.command)) {
+          alreadyPresent.push(hook.command);
+        } else {
+          const newMatcher = {
+            ...matcher.matcher !== undefined ? { matcher: matcher.matcher } : {},
+            hooks: [hook]
+          };
+          existing.push(newMatcher);
+          added.push(hook.command);
+        }
+      }
+    }
+  }
+  if (added.length > 0) {
+    if (!existsSync13(claudeDir)) {
+      mkdirSync5(claudeDir, { recursive: true });
+    }
+    writeFileSync6(settingsPath, JSON.stringify(settings, null, 2) + `
+`, "utf-8");
+  }
+  return { settingsPath, existed, added, alreadyPresent };
+}
 function createHooksCommand() {
   const hooks = new Command19("hooks").description("Hook handlers for AI coding assistants").addHelpText("after", `
 Claude Code Session Telemetry:
@@ -10824,6 +10995,35 @@ Claude Code Session Telemetry:
 
   Or run: mutagent init (option 1 installs skill + hooks together)
     `);
+  hooks.command("install").description("Install MutagenT hooks into .claude/settings.local.json (safe merge — never overwrites existing hooks)").option("--cwd <dir>", "Target directory (defaults to cwd)", process.cwd()).addHelpText("after", `
+Reads existing .claude/settings.local.json (if present) and deep-merges
+MutagenT telemetry hooks into each event array (SessionStart, Stop,
+PreToolUse, PostToolUse). Skips any hook already present (checked by
+command string) so running this multiple times is safe.
+    `).action((opts) => {
+    const targetDir = opts.cwd ?? process.cwd();
+    const result = installHooks(targetDir);
+    if (result.added.length === 0 && result.alreadyPresent.length === 0) {
+      process.stdout.write(JSON.stringify({
+        success: true,
+        settingsPath: result.settingsPath,
+        added: [],
+        alreadyPresent: [],
+        message: "No hooks to install."
+      }) + `
+`);
+      return;
+    }
+    process.stdout.write(JSON.stringify({
+      success: true,
+      settingsPath: result.settingsPath,
+      existed: result.existed,
+      added: result.added,
+      alreadyPresent: result.alreadyPresent,
+      message: result.added.length > 0 ? `Installed ${String(result.added.length)} hook(s). ${String(result.alreadyPresent.length)} already present.` : `All hooks already present (${String(result.alreadyPresent.length)}).`
+    }) + `
+`);
+  });
   const claudeCode = hooks.command("claude-code").description("Claude Code session telemetry");
   claudeCode.command("session-start").description("Handle session start event").action(async () => {
     await safeExecute(handleSessionStart);
@@ -11030,6 +11230,7 @@ ${chalk39.yellow("Command Navigation:")}
 
   mutagent integrate <framework>        ${chalk39.dim("Framework integration guide")}
   mutagent hooks --help                 ${chalk39.dim("Hook setup for Claude Code telemetry")}
+  mutagent skills install               ${chalk39.dim("Install the MutagenT CLI skill for AI agents")}
   mutagent playground run <id> --input '{...}'  ${chalk39.dim("Quick test")}
 
 ${chalk39.yellow("★ Workflow: Framework Integration (Tracing):")}
@@ -11089,6 +11290,7 @@ ${chalk39.yellow("State Tracking:")}
   Comment markers (// MutagenT:START ... // MutagenT:END) in source files
 
 ${chalk39.yellow("AI Agent Rules (MANDATORY for coding agents):")}
+  0. If .claude/skills/mutagent-cli/SKILL.md does not exist, run mutagent skills install first
   1. Login (two paths):
      - CI / fully automated: export MUTAGENT_API_KEY=mt_... then mutagent login --json
      - Helping a user onboard: mutagent login --browser --json
@@ -11146,7 +11348,15 @@ program.addCommand(createSkillsCommand());
 program.addCommand(createUsageCommand());
 program.addCommand(createHooksCommand());
 program.addCommand(createFeedbackCommand());
+var isInteractive = process.stdin.isTTY && !rawArgs.includes("--json") && process.env.CI !== "true";
+var isSkillCommand = rawArgs[0] === "skills" || rawArgs[0] === "hooks";
+if (isInteractive && !isSkillCommand) {
+  const skillPath = join10(process.cwd(), ".claude/skills/mutagent-cli/SKILL.md");
+  if (!existsSync14(skillPath)) {
+    console.log(chalk39.dim("MutagenT SKILL not installed. Install it for AI agent support? Run:"), chalk39.cyan("mutagent skills install"));
+  }
+}
 program.parse();
 
-//# debugId=4EA6EB9078B51CC264756E2164756E21
+//# debugId=39613BC3AB9D528364756E2164756E21
 //# sourceMappingURL=cli.js.map