@rockcarver/frodo-cli 4.0.0-50 → 4.0.0-51

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rockcarver/frodo-cli",
-  "version": "4.0.0-50",
+  "version": "4.0.0-51",
   "type": "module",
   "description": "A command line interface to manage ForgeRock Identity Cloud tenants, ForgeOps deployments, and classic deployments.",
   "keywords": [
@@ -37,6 +37,14 @@
     "test:serial": "FRODO_TEST=1 NODE_OPTIONS='--no-warnings --experimental-vm-modules' npx jest --silent --runInBand",
     "test:debug": "FRODO_TEST=1 NODE_OPTIONS='--no-warnings --experimental-vm-modules' npx jest --silent=false --json --outputFile=./testResults.json",
     "test:update": "FRODO_TEST=1 NODE_OPTIONS='--no-warnings --experimental-vm-modules' npx jest --silent=false --updateSnapshot",
+    "mcp:batch1": "node tools/mcp-agentic-batch-run.mjs --output docs/mcp-agentic-run-log.batch1.json",
+    "mcp:batch2": "node tools/mcp-agentic-batch-run.mjs --batch batch2 --output docs/mcp-agentic-run-log.batch2.json",
+    "mcp:batch3": "node tools/mcp-agentic-batch-run.mjs --batch batch3 --output docs/mcp-agentic-run-log.batch3.json",
+    "mcp:batch4": "node tools/mcp-agentic-batch-run.mjs --batch batch4 --output docs/mcp-agentic-run-log.batch4.json",
+    "mcp:score": "node tools/mcp-agentic-score.mjs",
+    "mcp:assess": "node tools/mcp-agentic-assess.mjs",
+    "mcp:test-aiagents": "node tools/mcp-aiagent-introspection-test.mjs",
+    "mcp:test-oauth2-mayact": "node tools/mcp-oauth2-mayact-update-test.mjs",
     "lint": "eslint --cache --cache-location .eslintcache \"src/**/*.ts\"",
     "lint:fix": "eslint --cache --cache-location .eslintcache --fix \"src/**/*.ts\"",
     "build": "npm run build:binary",
@@ -96,7 +104,7 @@
   },
   "devDependencies": {
     "@modelcontextprotocol/sdk": "^1.29.0",
-    "@rockcarver/frodo-lib": "4.0.0-39",
+    "@rockcarver/frodo-lib": "4.0.0-46",
     "@types/colors": "^1.2.1",
     "@types/fs-extra": "^11.0.1",
     "@types/jest": "^29.2.3",

package/tools/agent-e2e-audit.mjs

@@ -0,0 +1,219 @@
+#!/usr/bin/env node
+
+import fs from 'fs';
+import path from 'path';
+
+const ROOT = process.cwd();
+const TEST_DIR = path.join(ROOT, 'test/e2e');
+const SNAP_DIR = path.join(ROOT, 'test/e2e/__snapshots__');
+const MOCKS_DIR = path.join(ROOT, 'test/e2e/mocks');
+
+const TEST_FILES = fs
+    .readdirSync(TEST_DIR)
+    .filter((f) => (f.includes('agent') || f === 'config-manager-export-oauth2-agents.e2e.test.js') && f.endsWith('.e2e.test.js'))
+    .sort();
+
+const ERROR_PATTERNS = [
+    /\[Polly\].*Recording for the following request is not found/i,
+    /Error getting tokens/i,
+    /Command failed:/i,
+    /TypeError:/,
+    /ReferenceError:/,
+    /SyntaxError:/,
+    /ENOENT:/,
+    /Error exporting/i,
+];
+
+function walkDirs(dir, predicate) {
+    const out = [];
+    if (!fs.existsSync(dir)) return out;
+    const stack = [dir];
+    while (stack.length) {
+        const cur = stack.pop();
+        for (const entry of fs.readdirSync(cur)) {
+            const full = path.join(cur, entry);
+            const st = fs.statSync(full);
+            if (st.isDirectory()) {
+                if (predicate(full, entry)) out.push(full);
+                stack.push(full);
+            }
+        }
+    }
+    return out;
+}
+
+function countHarFiles(dir) {
+    let count = 0;
+    const stack = [dir];
+    while (stack.length) {
+        const cur = stack.pop();
+        for (const entry of fs.readdirSync(cur)) {
+            const full = path.join(cur, entry);
+            const st = fs.statSync(full);
+            if (st.isDirectory()) stack.push(full);
+            else if (entry.endsWith('.har')) count += 1;
+        }
+    }
+    return count;
+}
+
+function parseCommands(testText) {
+    return [...testText.matchAll(/const\s+CMD\s*=\s*`([^`]+)`/g)].map((m) => m[1]);
+}
+
+function getSuiteStem(testFile) {
+    return testFile.replace('.e2e.test.js', '');
+}
+
+function findMatchingMockDirs(stem, commands) {
+    if (stem.startsWith('agent-')) {
+        const agentRoot = walkDirs(MOCKS_DIR, (_, entry) => /^agent_\d+$/.test(entry))[0];
+        if (!agentRoot) return [];
+        const entries = fs.readdirSync(agentRoot).filter((d) => fs.statSync(path.join(agentRoot, d)).isDirectory());
+
+        const op = stem.replace('agent-', '');
+        if (!op.includes('-')) {
+            return entries.filter((d) => d.startsWith(`${op}_`)).map((d) => path.join(agentRoot, d));
+        }
+        if (op.startsWith('web-')) return entries.filter((d) => d.startsWith(`web-${op.replace('web-', '')}`)).map((d) => path.join(agentRoot, d));
+        if (op.startsWith('java-')) return entries.filter((d) => d.startsWith(`java-${op.replace('java-', '')}`)).map((d) => path.join(agentRoot, d));
+        if (op.startsWith('gateway-')) return entries.filter((d) => d.startsWith(`gateway-${op.replace('gateway-', '')}`)).map((d) => path.join(agentRoot, d));
+        if (op.startsWith('ai-')) return entries.filter((d) => d.startsWith(`ai-${op.replace('ai-', '')}`)).map((d) => path.join(agentRoot, d));
+        return [];
+    }
+
+    const cmdLine = commands.join('\n');
+    if (cmdLine.includes('config-manager pull oauth2-agents')) {
+        return walkDirs(MOCKS_DIR, (_, entry) => /^oauth2-agents_\d+$/.test(entry));
+    }
+
+    return [];
+}
+
+function snapshotHealth(snapshotText) {
+    const hits = [];
+    for (const re of ERROR_PATTERNS) {
+        if (re.test(snapshotText)) hits.push(re.source);
+    }
+    return hits;
+}
+
+function parseSnapshotEntries(snapshotText) {
+    const entries = [];
+    const re = /exports\[`([^`]+)`\]\s*=\s*`([\s\S]*?)`;/g;
+    for (const m of snapshotText.matchAll(re)) {
+        entries.push({
+            name: m[1],
+            body: m[2],
+        });
+    }
+    return entries;
+}
+
+function findSuspiciousSnapshotTests(snapshotText) {
+    const suspiciousTests = [];
+    const entries = parseSnapshotEntries(snapshotText);
+    for (const entry of entries) {
+        const matchedPatterns = [];
+        for (const pattern of ERROR_PATTERNS) {
+            if (pattern.test(entry.body)) matchedPatterns.push(pattern.source);
+        }
+        if (matchedPatterns.length > 0) {
+            suspiciousTests.push({
+                testName: entry.name,
+                matchedPatterns,
+            });
+        }
+    }
+    return suspiciousTests;
+}
+
+function classifySuite({ testFile, commands, snapshotPath, mockDirs }) {
+    const reasons = [];
+    let verdict = 'GOOD';
+
+    if (!fs.existsSync(snapshotPath)) {
+        verdict = 'BAD';
+        reasons.push('missing snapshot file');
+    }
+
+    if (mockDirs.length === 0) {
+        verdict = 'BAD';
+        reasons.push('recording directory missing');
+    }
+
+    let snapshotCount = 0;
+    let suspicious = [];
+    let suspiciousTests = [];
+    if (fs.existsSync(snapshotPath)) {
+        const snapshotText = fs.readFileSync(snapshotPath, 'utf8');
+        snapshotCount = (snapshotText.match(/exports\[`/g) || []).length;
+        suspicious = snapshotHealth(snapshotText);
+        suspiciousTests = findSuspiciousSnapshotTests(snapshotText);
+        if (suspicious.length > 0) {
+            verdict = 'BAD';
+            reasons.push('snapshot contains explicit error signature');
+        }
+    }
+
+    const harCount = mockDirs.reduce((sum, dir) => sum + countHarFiles(dir), 0);
+    if (mockDirs.length > 0 && harCount === 0) {
+        verdict = 'BAD';
+        reasons.push('recording directory has no .har files');
+    }
+
+    return {
+        suite: testFile,
+        verdict,
+        reasons,
+        commands: commands.length,
+        snapshots: snapshotCount,
+        mockDirs: mockDirs.length,
+        harFiles: harCount,
+        suspicious,
+        suspiciousTests,
+    };
+}
+
+const results = [];
+for (const testFile of TEST_FILES) {
+    const testPath = path.join(TEST_DIR, testFile);
+    const testText = fs.readFileSync(testPath, 'utf8');
+    const commands = parseCommands(testText);
+    const stem = getSuiteStem(testFile);
+    const snapshotPath = path.join(SNAP_DIR, `${testFile}.snap`);
+    const mockDirs = findMatchingMockDirs(stem, commands);
+
+    results.push(
+        classifySuite({
+            testFile,
+            commands,
+            snapshotPath,
+            mockDirs,
+        }),
+    );
+}
+
+const bad = results.filter((r) => r.verdict === 'BAD');
+const good = results.length - bad.length;
+
+console.log(`Audited suites: ${results.length}`);
+console.log(`GOOD: ${good}`);
+console.log(`BAD: ${bad.length}`);
+console.log('');
+
+for (const r of results) {
+    const prefix = r.verdict === 'GOOD' ? '[GOOD]' : '[BAD] ';
+    console.log(`${prefix} ${r.suite}`);
+    if (r.reasons.length > 0) console.log(`  reasons: ${r.reasons.join('; ')}`);
+    if (r.suspicious.length > 0) console.log(`  suspicious: ${r.suspicious.join(', ')}`);
+    if (r.suspiciousTests.length > 0) {
+        for (const t of r.suspiciousTests) {
+            console.log(`  suspicious_test: ${t.testName}`);
+            console.log(`    matched: ${t.matchedPatterns.join(', ')}`);
+        }
+    }
+    console.log(`  cmds=${r.commands} snapshots=${r.snapshots} mockDirs=${r.mockDirs} harFiles=${r.harFiles}`);
+}
+
+process.exit(bad.length > 0 ? 1 : 0);

package/tools/mcp-agentic-assess.mjs

@@ -0,0 +1,204 @@
+#!/usr/bin/env node
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+function parseArgs(argv) {
+    const args = {
+        output: 'docs/mcp-agentic-assessment.json',
+        scoreboards: [
+            'docs/mcp-agentic-scoreboard.batch2.json',
+            'docs/mcp-agentic-scoreboard.batch3.json',
+            'docs/mcp-agentic-scoreboard.batch4.json',
+        ],
+    };
+
+    for (let i = 2; i < argv.length; i += 1) {
+        const token = argv[i];
+        if (token === '--output') {
+            args.output = argv[i + 1] || args.output;
+            i += 1;
+            continue;
+        }
+        if (token === '--scoreboard') {
+            args.scoreboards.push(argv[i + 1]);
+            i += 1;
+        }
+    }
+
+    return args;
+}
+
+function readJson(filePath) {
+    return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+}
+
+function round(value, decimals = 2) {
+    const scale = 10 ** decimals;
+    return Math.round(value * scale) / scale;
+}
+
+function normalizeMetrics(metrics) {
+    return {
+        runCount: metrics.runCount || 0,
+        completedCount: metrics.completedCount || 0,
+        completionRate: metrics.completionRate || 0,
+        meanAttemptsPerTask: metrics.meanAttemptsPerTask || 0,
+        firstAttemptSuccessRate: metrics.firstAttemptSuccessRate || 0,
+        duplicateRetryRate: metrics.duplicateRetryRate || 0,
+        intentionalFollowUpRate: metrics.intentionalFollowUpRate || 0,
+        temptationFailureRate:
+            typeof metrics.temptationFailureRate === 'number'
+                ? metrics.temptationFailureRate
+                : 0,
+    };
+}
+
+function computeBatchWeightedScore(metrics) {
+    const m = normalizeMetrics(metrics);
+
+    // Heavier penalty for temptation failures, light penalty for intentional follow-ups.
+    const weightedPenalty =
+        (100 - m.completionRate) * 1.5 +
+        (100 - m.firstAttemptSuccessRate) * 1.0 +
+        m.meanAttemptsPerTask * 12 +
+        m.duplicateRetryRate * 1.0 +
+        m.intentionalFollowUpRate * 0.15 +
+        m.temptationFailureRate * 1.8;
+
+    return round(1000 - weightedPenalty, 3);
+}
+
+function aggregateVariant(variantName, batchEntries) {
+    let runCount = 0;
+    let weightedCompletion = 0;
+    let weightedFirstAttempt = 0;
+    let weightedMeanAttempts = 0;
+    let weightedDuplicateRetries = 0;
+    let weightedIntentionalFollowUps = 0;
+    let weightedTemptationFailures = 0;
+    let weightedScoreTotal = 0;
+
+    const batchScores = {};
+
+    for (const entry of batchEntries) {
+        const metrics = normalizeMetrics(entry.metrics);
+        const weight = Math.max(1, metrics.runCount);
+        const batchScore = computeBatchWeightedScore(metrics);
+
+        batchScores[entry.batch] = {
+            ...metrics,
+            weightedScore: batchScore,
+        };
+
+        runCount += weight;
+        weightedCompletion += metrics.completionRate * weight;
+        weightedFirstAttempt += metrics.firstAttemptSuccessRate * weight;
+        weightedMeanAttempts += metrics.meanAttemptsPerTask * weight;
+        weightedDuplicateRetries += metrics.duplicateRetryRate * weight;
+        weightedIntentionalFollowUps += metrics.intentionalFollowUpRate * weight;
+        weightedTemptationFailures += metrics.temptationFailureRate * weight;
+        weightedScoreTotal += batchScore * weight;
+    }
+
+    const denominator = runCount || 1;
+
+    return {
+        variant: variantName,
+        batches: batchScores,
+        aggregate: {
+            weightedCompletionRate: round(weightedCompletion / denominator),
+            weightedFirstAttemptSuccessRate: round(weightedFirstAttempt / denominator),
+            weightedMeanAttemptsPerTask: round(weightedMeanAttempts / denominator),
+            weightedDuplicateRetryRate: round(weightedDuplicateRetries / denominator),
+            weightedIntentionalFollowUpRate: round(weightedIntentionalFollowUps / denominator),
+            weightedTemptationFailureRate: round(weightedTemptationFailures / denominator),
+            weightedScore: round(weightedScoreTotal / denominator, 3),
+        },
+    };
+}
+
+function buildAssessment(scoreboards) {
+    const byVariant = new Map();
+
+    for (const scoreboardEntry of scoreboards) {
+        const batch = scoreboardEntry.batch;
+        const variants = scoreboardEntry.scoreboard.variants || {};
+        for (const [variantName, metrics] of Object.entries(variants)) {
+            const entries = byVariant.get(variantName) || [];
+            entries.push({ batch, metrics });
+            byVariant.set(variantName, entries);
+        }
+    }
+
+    const variantAssessments = [...byVariant.entries()]
+        .map(([variantName, entries]) => aggregateVariant(variantName, entries))
+        .sort((left, right) => right.aggregate.weightedScore - left.aggregate.weightedScore);
+
+    const recommended = variantAssessments[0]?.variant || null;
+
+    return {
+        generatedAt: new Date().toISOString(),
+        batches: scoreboards.map((entry) => entry.batch),
+        formula: {
+            weightedScore:
+                '1000 - ((100-completionRate)*1.5 + (100-firstAttemptSuccessRate)*1.0 + meanAttemptsPerTask*12 + duplicateRetryRate*1.0 + intentionalFollowUpRate*0.15 + temptationFailureRate*1.8)',
+            notes: [
+                'Temptation failures carry a heavier penalty than intentional follow-ups.',
+                'Intentional pagination/chaining follow-ups are lightly penalized to avoid over-penalizing valid workflows.',
+                'Higher weightedScore is better.',
+            ],
+        },
+        ranking: variantAssessments,
+        recommendation: {
+            variant: recommended,
+            rationale:
+                recommended === 'agentic'
+                    ? 'Best weighted score with strong completion/first-attempt rates and no import/export temptation failures.'
+                    : 'Highest weighted score under the current batch evidence.',
+        },
+    };
+}
+
+function printAssessment(assessment) {
+    console.log('MCP Agentic Assessment');
+    console.log('======================');
+    console.log(`Batches: ${assessment.batches.join(', ')}`);
+    console.log(`Recommended variant: ${assessment.recommendation.variant}`);
+
+    for (const entry of assessment.ranking) {
+        console.log('');
+        console.log(`Variant: ${entry.variant}`);
+        console.log(`  Weighted score: ${entry.aggregate.weightedScore}`);
+        console.log(`  Completion: ${entry.aggregate.weightedCompletionRate}%`);
+        console.log(`  First-attempt success: ${entry.aggregate.weightedFirstAttemptSuccessRate}%`);
+        console.log(`  Mean attempts/task: ${entry.aggregate.weightedMeanAttemptsPerTask}`);
+        console.log(
+            `  Temptation failure rate: ${entry.aggregate.weightedTemptationFailureRate}%`
+        );
+        console.log(
+            `  Intentional follow-up rate: ${entry.aggregate.weightedIntentionalFollowUpRate}%`
+        );
+    }
+}
+
+function main() {
+    const args = parseArgs(process.argv);
+    const scoreboards = args.scoreboards.map((scoreboardPath) => {
+        const resolved = path.resolve(process.cwd(), scoreboardPath);
+        const scoreboard = readJson(resolved);
+        const batchMatch = scoreboardPath.match(/batch\d+/i);
+        const batch = batchMatch ? batchMatch[0].toLowerCase() : path.basename(scoreboardPath);
+        return { batch, path: scoreboardPath, scoreboard };
+    });
+
+    const assessment = buildAssessment(scoreboards);
+    printAssessment(assessment);
+
+    const outputPath = path.resolve(process.cwd(), args.output);
+    fs.writeFileSync(outputPath, JSON.stringify(assessment, null, 2), 'utf8');
+    console.log('');
+    console.log(`Wrote assessment JSON to: ${outputPath}`);
+}
+
+main();