@memberjunction/cli 2.119.0 → 2.120.0

package/README.md

@@ -209,6 +209,7 @@ Available AI commands:
 - `actions run` - Execute an AI action with parameters
 - `prompts list` - List available AI models for direct prompt execution
 - `prompts run` - Execute a direct prompt with an AI model
+- `audit agent-run` - Audit and analyze AI agent execution runs for debugging
 
 **📚 For detailed documentation:** See the [AI-CLI README](../AI/AICLI/README.md)
 
@@ -248,6 +249,21 @@ mj ai prompts run -p "Explain quantum computing in simple terms"
 # Use a specific model
 mj ai prompts run -p "Write a Python function to sort a list" --model "gpt-4"
 
+# Audit recent agent runs
+mj ai audit agent-run --list --status failed --days 7
+
+# Audit specific run with summary
+mj ai audit agent-run <run-id>
+
+# Examine specific step in detail
+mj ai audit agent-run <run-id> --step 3 --detail full
+
+# Analyze errors in a run
+mj ai audit agent-run <run-id> --errors
+
+# Export full audit data
+mj ai audit agent-run <run-id> --export full --file audit.json
+
 # Use system prompt and temperature
 mj ai prompts run -p "Generate a haiku" --system "You are a poet" --temperature 0.3
 ```
@@ -281,6 +297,22 @@ mj ai prompts run -p "Generate a haiku" --system "You are a poet" --temperature
 - `-v, --verbose`: Show detailed execution information
 - `--timeout <ms>`: Execution timeout in milliseconds (default: 300000)
 
+**Audit Commands:**
+- `<run-id>`: Agent Run ID (UUID) to audit (optional for --list mode)
+- `-l, --list`: List recent agent runs (filter with other options)
+- `-a, --agent <name>`: Filter by agent name (requires --list)
+- `--status <status>`: Filter by status: success, failed, running, all (default: all)
+- `--days <number>`: Number of days to look back (default: 7)
+- `--limit <number>`: Maximum runs to return (default: 50)
+- `-s, --step <number>`: Show details for specific step (1-based index)
+- `-d, --detail <level>`: Detail level for step: minimal, standard, detailed, full (default: standard)
+- `-e, --errors`: Show only error details and context
+- `--export <type>`: Export data: full, summary, steps
+- `-f, --file <path>`: Output file path for export
+- `--max-tokens <number>`: Max tokens per field (default: 5000, 0 = no limit)
+- `-o, --output <format>`: Output format (compact, json, table, markdown)
+- `-v, --verbose`: Show detailed diagnostic information
+
 #### AI Features:
 
 **Progress Tracking**: Real-time visual progress indicators during agent execution
@@ -299,6 +331,15 @@ mj ai prompts run -p "Generate a haiku" --system "You are a poet" --temperature
 - Natural back-and-forth dialogue
 - Exit with "exit", "quit", or Ctrl+C
 
+**Agent Run Auditing**: Comprehensive debugging and analysis tools
+- List recent runs with filtering by agent, status, and date
+- View run summaries with performance metrics and step breakdowns
+- Deep dive into specific steps with smart truncation for large payloads
+- Automatic error pattern detection with suggested fixes
+- Multiple output formats including markdown for AI assistants
+- Export capabilities for offline analysis
+- Optimized for read-only performance with `simple` result type
+
 #### AI Configuration:
 
 Add AI-specific settings to your `mj.config.cjs`:

package/dist/commands/ai/audit/agent-run.d.ts

@@ -0,0 +1,24 @@
+import { Command } from '@oclif/core';
+export default class AgentRun extends Command {
+    static description: string;
+    static examples: string[];
+    static flags: {
+        output: import("@oclif/core/lib/interfaces").OptionFlag<string, import("@oclif/core/lib/interfaces").CustomOptions>;
+        step: import("@oclif/core/lib/interfaces").OptionFlag<number | undefined, import("@oclif/core/lib/interfaces").CustomOptions>;
+        detail: import("@oclif/core/lib/interfaces").OptionFlag<string | undefined, import("@oclif/core/lib/interfaces").CustomOptions>;
+        errors: import("@oclif/core/lib/interfaces").BooleanFlag<boolean>;
+        list: import("@oclif/core/lib/interfaces").BooleanFlag<boolean>;
+        agent: import("@oclif/core/lib/interfaces").OptionFlag<string | undefined, import("@oclif/core/lib/interfaces").CustomOptions>;
+        status: import("@oclif/core/lib/interfaces").OptionFlag<string, import("@oclif/core/lib/interfaces").CustomOptions>;
+        days: import("@oclif/core/lib/interfaces").OptionFlag<number, import("@oclif/core/lib/interfaces").CustomOptions>;
+        limit: import("@oclif/core/lib/interfaces").OptionFlag<number, import("@oclif/core/lib/interfaces").CustomOptions>;
+        export: import("@oclif/core/lib/interfaces").OptionFlag<string | undefined, import("@oclif/core/lib/interfaces").CustomOptions>;
+        file: import("@oclif/core/lib/interfaces").OptionFlag<string | undefined, import("@oclif/core/lib/interfaces").CustomOptions>;
+        'max-tokens': import("@oclif/core/lib/interfaces").OptionFlag<number, import("@oclif/core/lib/interfaces").CustomOptions>;
+        verbose: import("@oclif/core/lib/interfaces").BooleanFlag<boolean>;
+    };
+    static args: {
+        runId: import("@oclif/core/lib/interfaces").Arg<string | undefined, Record<string, unknown>>;
+    };
+    run(): Promise<void>;
+}

package/dist/commands/ai/audit/agent-run.js

@@ -0,0 +1,213 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const core_1 = require("@oclif/core");
+const ai_cli_1 = require("@memberjunction/ai-cli");
+const ora_classic_1 = __importDefault(require("ora-classic"));
+const chalk_1 = __importDefault(require("chalk"));
+const fs = __importStar(require("fs/promises"));
+class AgentRun extends core_1.Command {
+    static description = 'Audit and analyze AI agent execution runs for debugging and performance analysis';
+    static examples = [
+        // Summary mode
+        '<%= config.bin %> <%= command.id %> abc-123-def-456',
+        '<%= config.bin %> <%= command.id %> abc-123-def-456 --output=markdown',
+        // Step detail mode
+        '<%= config.bin %> <%= command.id %> abc-123-def-456 --step 3',
+        '<%= config.bin %> <%= command.id %> abc-123-def-456 --step 3 --detail full',
+        // Error analysis
+        '<%= config.bin %> <%= command.id %> abc-123-def-456 --errors',
+        // List recent runs
+        '<%= config.bin %> <%= command.id %> --list --agent "Skip: Requirements Expert"',
+        '<%= config.bin %> <%= command.id %> --list --status failed --days 7',
+        // Export mode
+        '<%= config.bin %> <%= command.id %> abc-123-def-456 --export full --file report.json',
+    ];
+    static flags = {
+        // Output format
+        output: core_1.Flags.string({
+            char: 'o',
+            description: 'Output format',
+            options: ['compact', 'json', 'table', 'markdown'],
+            default: 'compact',
+        }),
+        // Step detail mode
+        step: core_1.Flags.integer({
+            char: 's',
+            description: 'Show details for specific step number (1-based index)',
+            exclusive: ['list', 'errors'],
+        }),
+        detail: core_1.Flags.string({
+            char: 'd',
+            description: 'Detail level for step output (default: standard)',
+            options: ['minimal', 'standard', 'detailed', 'full'],
+            dependsOn: ['step'],
+        }),
+        // Error analysis mode
+        errors: core_1.Flags.boolean({
+            char: 'e',
+            description: 'Show only error details and context',
+            default: false,
+            exclusive: ['step', 'list'],
+        }),
+        // List mode
+        list: core_1.Flags.boolean({
+            char: 'l',
+            description: 'List recent agent runs',
+            default: false,
+            exclusive: ['step', 'errors'],
+        }),
+        agent: core_1.Flags.string({
+            char: 'a',
+            description: 'Filter by agent name',
+            dependsOn: ['list'],
+        }),
+        status: core_1.Flags.string({
+            description: 'Filter by run status',
+            options: ['success', 'failed', 'running', 'all'],
+            default: 'all',
+            dependsOn: ['list'],
+        }),
+        days: core_1.Flags.integer({
+            description: 'Number of days to look back',
+            default: 7,
+            dependsOn: ['list'],
+        }),
+        limit: core_1.Flags.integer({
+            description: 'Maximum number of runs to return',
+            default: 50,
+            dependsOn: ['list'],
+        }),
+        // Export mode
+        export: core_1.Flags.string({
+            description: 'Export full data to file',
+            options: ['full', 'summary', 'steps'],
+            dependsOn: ['file'],
+        }),
+        file: core_1.Flags.string({
+            char: 'f',
+            description: 'Output file path for export',
+        }),
+        // Truncation control
+        'max-tokens': core_1.Flags.integer({
+            description: 'Maximum tokens per field (0 = no limit)',
+            default: 5000,
+        }),
+        // Verbose mode
+        verbose: core_1.Flags.boolean({
+            char: 'v',
+            description: 'Show detailed diagnostic information',
+            default: false,
+        }),
+    };
+    static args = {
+        runId: core_1.Args.string({
+            description: 'AI Agent Run ID to audit (UUID)',
+            required: false, // Optional because --list mode doesn't need it
+        }),
+    };
+    async run() {
+        const { args, flags } = await this.parse(AgentRun);
+        const spinner = (0, ora_classic_1.default)();
+        try {
+            const service = new ai_cli_1.AgentAuditService();
+            // MODE 1: List recent runs
+            if (flags.list) {
+                spinner.start('Loading recent agent runs...');
+                const runs = await service.listRecentRuns({
+                    agentName: flags.agent,
+                    status: flags.status,
+                    days: flags.days,
+                    limit: flags.limit,
+                });
+                spinner.stop();
+                this.log(service.formatRunList(runs, flags.output));
+                process.exit(0);
+                return;
+            }
+            // Validate runId for non-list modes
+            if (!args.runId) {
+                this.error('Run ID is required (use --list to see recent runs)');
+            }
+            // MODE 2: Step detail mode
+            if (flags.step != null) {
+                spinner.start(`Loading step ${flags.step} details...`);
+                const stepDetail = await service.getStepDetail(args.runId, flags.step, {
+                    detailLevel: (flags.detail || 'standard'),
+                    maxTokens: flags['max-tokens'],
+                });
+                spinner.stop();
+                this.log(service.formatStepDetail(stepDetail, flags.output));
+                process.exit(0);
+                return;
+            }
+            // MODE 3: Error analysis mode
+            if (flags.errors) {
+                spinner.start('Analyzing errors...');
+                const errorAnalysis = await service.analyzeErrors(args.runId);
+                spinner.stop();
+                this.log(service.formatErrorAnalysis(errorAnalysis, flags.output));
+                process.exit(0);
+                return;
+            }
+            // MODE 4: Export mode
+            if (flags.export && flags.file) {
+                spinner.start('Exporting full audit data...');
+                const exportData = await service.exportRun(args.runId, flags.export);
+                await fs.writeFile(flags.file, JSON.stringify(exportData, null, 2), 'utf8');
+                spinner.stop();
+                this.log(chalk_1.default.green(`✓ Full audit data exported to ${flags.file}`));
+                this.log(chalk_1.default.dim(`  File size: ${(JSON.stringify(exportData).length / 1024).toFixed(1)} KB`));
+                process.exit(0);
+                return;
+            }
+            // MODE 5: Summary mode (default)
+            spinner.start('Loading agent run summary...');
+            const summary = await service.getRunSummary(args.runId, {
+                includeStepList: true,
+                maxTokens: flags['max-tokens'],
+            });
+            spinner.stop();
+            this.log(service.formatRunSummary(summary, flags.output));
+            if (flags.verbose) {
+                this.log(chalk_1.default.dim('\n💡 Tip: Use --step <N> to see details for a specific step'));
+                this.log(chalk_1.default.dim('💡 Tip: Use --errors to see only error information'));
+            }
+            process.exit(0);
+        }
+        catch (error) {
+            spinner.fail('Audit failed');
+            if (flags.verbose && error instanceof Error) {
+                this.log(chalk_1.default.red('\nError Details:'));
+                this.log(error.stack || error.message);
+            }
+            this.error(error);
+        }
+    }
+}
+exports.default = AgentRun;

package/dist/commands/ai/audit/index.d.ts

@@ -0,0 +1,6 @@
+import { Command } from '@oclif/core';
+export default class Audit extends Command {
+    static description: string;
+    static hidden: boolean;
+    run(): Promise<void>;
+}

package/dist/commands/ai/audit/index.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const core_1 = require("@oclif/core");
+class Audit extends core_1.Command {
+    static description = 'Analyze and audit AI agent runs, prompts, and actions for debugging and performance analysis';
+    static hidden = false;
+    async run() {
+        // This command just displays help for the audit topic
+        await this.config.runCommand('help', ['ai', 'audit']);
+    }
+}
+exports.default = Audit;

package/oclif.manifest.json

@@ -1821,6 +1821,222 @@
         "run.js"
       ]
     },
+    "ai:audit:agent-run": {
+      "aliases": [],
+      "args": {
+        "runId": {
+          "description": "AI Agent Run ID to audit (UUID)",
+          "name": "runId",
+          "required": false
+        }
+      },
+      "description": "Audit and analyze AI agent execution runs for debugging and performance analysis",
+      "examples": [
+        "<%= config.bin %> <%= command.id %> abc-123-def-456",
+        "<%= config.bin %> <%= command.id %> abc-123-def-456 --output=markdown",
+        "<%= config.bin %> <%= command.id %> abc-123-def-456 --step 3",
+        "<%= config.bin %> <%= command.id %> abc-123-def-456 --step 3 --detail full",
+        "<%= config.bin %> <%= command.id %> abc-123-def-456 --errors",
+        "<%= config.bin %> <%= command.id %> --list --agent \"Skip: Requirements Expert\"",
+        "<%= config.bin %> <%= command.id %> --list --status failed --days 7",
+        "<%= config.bin %> <%= command.id %> abc-123-def-456 --export full --file report.json"
+      ],
+      "flags": {
+        "output": {
+          "char": "o",
+          "description": "Output format",
+          "name": "output",
+          "default": "compact",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "options": [
+            "compact",
+            "json",
+            "table",
+            "markdown"
+          ],
+          "type": "option"
+        },
+        "step": {
+          "char": "s",
+          "description": "Show details for specific step number (1-based index)",
+          "exclusive": [
+            "list",
+            "errors"
+          ],
+          "name": "step",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "detail": {
+          "char": "d",
+          "dependsOn": [
+            "step"
+          ],
+          "description": "Detail level for step output (default: standard)",
+          "name": "detail",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "options": [
+            "minimal",
+            "standard",
+            "detailed",
+            "full"
+          ],
+          "type": "option"
+        },
+        "errors": {
+          "char": "e",
+          "description": "Show only error details and context",
+          "exclusive": [
+            "step",
+            "list"
+          ],
+          "name": "errors",
+          "allowNo": false,
+          "type": "boolean"
+        },
+        "list": {
+          "char": "l",
+          "description": "List recent agent runs",
+          "exclusive": [
+            "step",
+            "errors"
+          ],
+          "name": "list",
+          "allowNo": false,
+          "type": "boolean"
+        },
+        "agent": {
+          "char": "a",
+          "dependsOn": [
+            "list"
+          ],
+          "description": "Filter by agent name",
+          "name": "agent",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "status": {
+          "dependsOn": [
+            "list"
+          ],
+          "description": "Filter by run status",
+          "name": "status",
+          "default": "all",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "options": [
+            "success",
+            "failed",
+            "running",
+            "all"
+          ],
+          "type": "option"
+        },
+        "days": {
+          "dependsOn": [
+            "list"
+          ],
+          "description": "Number of days to look back",
+          "name": "days",
+          "default": 7,
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "limit": {
+          "dependsOn": [
+            "list"
+          ],
+          "description": "Maximum number of runs to return",
+          "name": "limit",
+          "default": 50,
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "export": {
+          "dependsOn": [
+            "file"
+          ],
+          "description": "Export full data to file",
+          "name": "export",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "options": [
+            "full",
+            "summary",
+            "steps"
+          ],
+          "type": "option"
+        },
+        "file": {
+          "char": "f",
+          "description": "Output file path for export",
+          "name": "file",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "max-tokens": {
+          "description": "Maximum tokens per field (0 = no limit)",
+          "name": "max-tokens",
+          "default": 5000,
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "verbose": {
+          "char": "v",
+          "description": "Show detailed diagnostic information",
+          "name": "verbose",
+          "allowNo": false,
+          "type": "boolean"
+        }
+      },
+      "hasDynamicHelp": false,
+      "hiddenAliases": [],
+      "id": "ai:audit:agent-run",
+      "pluginAlias": "@memberjunction/cli",
+      "pluginName": "@memberjunction/cli",
+      "pluginType": "core",
+      "strict": true,
+      "enableJsonFlag": false,
+      "isESM": false,
+      "relativePath": [
+        "dist",
+        "commands",
+        "ai",
+        "audit",
+        "agent-run.js"
+      ]
+    },
+    "ai:audit": {
+      "aliases": [],
+      "args": {},
+      "description": "Analyze and audit AI agent runs, prompts, and actions for debugging and performance analysis",
+      "flags": {},
+      "hasDynamicHelp": false,
+      "hidden": false,
+      "hiddenAliases": [],
+      "id": "ai:audit",
+      "pluginAlias": "@memberjunction/cli",
+      "pluginName": "@memberjunction/cli",
+      "pluginType": "core",
+      "strict": true,
+      "enableJsonFlag": false,
+      "isESM": false,
+      "relativePath": [
+        "dist",
+        "commands",
+        "ai",
+        "audit",
+        "index.js"
+      ]
+    },
     "ai:prompts:list": {
       "aliases": [],
       "args": {},
@@ -1961,5 +2177,5 @@
       ]
     }
   },
-  "version": "2.119.0"
+  "version": "2.120.0"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@memberjunction/cli",
-  "version": "2.119.0",
+  "version": "2.120.0",
   "description": "MemberJunction command line tools",
   "keywords": [
     "oclif"
@@ -51,13 +51,13 @@
   },
   "dependencies": {
     "@inquirer/prompts": "^5.0.1",
-    "@memberjunction/ai-cli": "2.119.0",
-    "@memberjunction/codegen-lib": "2.119.0",
-    "@memberjunction/core": "2.119.0",
-    "@memberjunction/db-auto-doc": "2.119.0",
-    "@memberjunction/metadata-sync": "2.119.0",
-    "@memberjunction/sqlserver-dataprovider": "2.119.0",
-    "@memberjunction/testing-cli": "2.119.0",
+    "@memberjunction/ai-cli": "2.120.0",
+    "@memberjunction/codegen-lib": "2.120.0",
+    "@memberjunction/core": "2.120.0",
+    "@memberjunction/db-auto-doc": "2.120.0",
+    "@memberjunction/metadata-sync": "2.120.0",
+    "@memberjunction/sqlserver-dataprovider": "2.120.0",
+    "@memberjunction/testing-cli": "2.120.0",
     "@oclif/core": "^3",
     "@oclif/plugin-help": "^6",
     "@oclif/plugin-version": "^2.0.17",