erosolar-cli 1.7.22 → 1.7.24

package/dist/core/responseVerifier.js

@@ -9,11 +9,248 @@
  *
  * @license MIT
  */
-import { exec } from 'node:child_process';
+import { exec, spawn } from 'node:child_process';
 import { promisify } from 'node:util';
 import * as fs from 'node:fs/promises';
 import * as path from 'node:path';
 const execAsync = promisify(exec);
+/**
+ * Spawns a fresh isolated erosolar-cli instance for testing
+ */
+async function spawnIsolatedCLI(cwd, timeout = 60000) {
+    const cliPath = path.join(cwd, 'dist/bin/erosolar.js');
+    // Verify CLI exists
+    try {
+        await fs.access(cliPath);
+    }
+    catch {
+        throw new Error(`CLI not found at ${cliPath}. Run build first.`);
+    }
+    let output = '';
+    let errors = '';
+    let exitResolve;
+    const exitPromise = new Promise(resolve => { exitResolve = resolve; });
+    const child = spawn('node', [cliPath, '--plain'], {
+        cwd,
+        env: { ...process.env, EROSOLAR_TEST_MODE: '1', NO_COLOR: '1' },
+        stdio: ['pipe', 'pipe', 'pipe']
+    });
+    child.stdout.on('data', (data) => { output += data.toString(); });
+    child.stderr.on('data', (data) => { errors += data.toString(); });
+    child.on('close', (code) => { exitResolve(code); });
+    child.on('error', (err) => { errors += err.message; exitResolve(1); });
+    // Set timeout
+    const timeoutId = setTimeout(() => {
+        child.kill('SIGTERM');
+        errors += `\nTimeout after ${timeout}ms`;
+    }, timeout);
+    child.on('close', () => clearTimeout(timeoutId));
+    // Wait for startup (look for prompt or give it 2 seconds)
+    await new Promise(resolve => {
+        const checkStartup = setInterval(() => {
+            if (output.includes('erosolar') || output.includes('>') || output.length > 100) {
+                clearInterval(checkStartup);
+                resolve();
+            }
+        }, 100);
+        setTimeout(() => { clearInterval(checkStartup); resolve(); }, 2000);
+    });
+    return {
+        process: child,
+        stdin: child.stdin,
+        output,
+        errors,
+        exitPromise
+    };
+}
+/**
+ * Sends a command to the spawned CLI and waits for response
+ */
+async function sendCommand(cli, command, waitMs = 5000) {
+    const outputBefore = cli.output.length;
+    cli.stdin.write(command + '\n');
+    // Wait for output to stabilize
+    await new Promise(resolve => {
+        let lastLength = cli.output.length;
+        const checkInterval = setInterval(() => {
+            if (cli.output.length > lastLength) {
+                lastLength = cli.output.length;
+            }
+            else if (cli.output.length > outputBefore) {
+                clearInterval(checkInterval);
+                resolve();
+            }
+        }, 200);
+        setTimeout(() => { clearInterval(checkInterval); resolve(); }, waitMs);
+    });
+    return cli.output.slice(outputBefore);
+}
+/**
+ * Runs an isolated runtime test in a fresh CLI instance
+ */
+export async function runIsolatedTest(test, cwd, llmVerifier) {
+    const startTime = Date.now();
+    const result = {
+        test,
+        success: false,
+        output: '',
+        errors: '',
+        exitCode: null,
+        duration: 0,
+        matchedPatterns: [],
+        unmatchedPatterns: []
+    };
+    try {
+        // Rebuild if required
+        if (test.requiresBuild) {
+            try {
+                await execAsync('npm run build', { cwd, timeout: 120000 });
+            }
+            catch (buildErr) {
+                result.errors = `Build failed: ${buildErr instanceof Error ? buildErr.message : 'unknown'}`;
+                result.duration = Date.now() - startTime;
+                return result;
+            }
+        }
+        // Spawn fresh CLI
+        const cli = await spawnIsolatedCLI(cwd, test.timeout || 60000);
+        // Execute each command
+        for (const cmd of test.commands) {
+            const cmdOutput = await sendCommand(cli, cmd);
+            result.output += `> ${cmd}\n${cmdOutput}\n`;
+        }
+        // Gracefully exit
+        cli.stdin.write('/quit\n');
+        await new Promise(resolve => setTimeout(resolve, 500));
+        cli.process.kill('SIGTERM');
+        result.exitCode = await cli.exitPromise;
+        result.errors = cli.errors;
+        // Check expected output patterns
+        if (test.expectedOutputs) {
+            for (const pattern of test.expectedOutputs) {
+                if (result.output.includes(pattern) || new RegExp(pattern, 'i').test(result.output)) {
+                    result.matchedPatterns.push(pattern);
+                }
+                else {
+                    result.unmatchedPatterns.push(pattern);
+                }
+            }
+        }
+        // LLM assessment of behavior if specified
+        if (test.expectedBehavior && llmVerifier) {
+            const assessPrompt = `Assess if this CLI output demonstrates the expected behavior.
+
+EXPECTED BEHAVIOR: ${test.expectedBehavior}
+
+CLI OUTPUT:
+---
+${result.output.slice(0, 4000)}
+---
+
+Return JSON: {"matches": true/false, "confidence": 0-100, "reasoning": "explanation"}`;
+            try {
+                const assessment = await llmVerifier(assessPrompt);
+                const match = assessment.match(/\{[\s\S]*\}/);
+                if (match) {
+                    const parsed = JSON.parse(match[0]);
+                    result.llmAssessment = `${parsed.matches ? '✅' : '❌'} [${parsed.confidence}%] ${parsed.reasoning}`;
+                    if (!parsed.matches || parsed.confidence < 70) {
+                        result.unmatchedPatterns.push(`behavior: ${test.expectedBehavior}`);
+                    }
+                    else {
+                        result.matchedPatterns.push(`behavior: ${test.expectedBehavior}`);
+                    }
+                }
+            }
+            catch {
+                result.llmAssessment = 'LLM assessment failed';
+            }
+        }
+        // Determine success
+        result.success = result.unmatchedPatterns.length === 0 &&
+            (result.matchedPatterns.length > 0 || !test.expectedOutputs?.length);
+    }
+    catch (err) {
+        result.errors = err instanceof Error ? err.message : 'Unknown error';
+    }
+    result.duration = Date.now() - startTime;
+    return result;
+}
+/**
+ * Generates isolated runtime tests for self-referential claims
+ * (claims about erosolar-cli's own behavior/features)
+ */
+export async function generateIsolatedTests(claims, llmVerifier) {
+    const selfClaims = claims.filter(c => c.statement.toLowerCase().includes('erosolar') ||
+        c.statement.toLowerCase().includes('cli') ||
+        c.statement.toLowerCase().includes('command') ||
+        c.statement.toLowerCase().includes('feature') ||
+        c.category === 'behavior' ||
+        c.category === 'feature');
+    if (selfClaims.length === 0)
+        return [];
+    const prompt = `Generate isolated CLI tests for these claims about erosolar-cli behavior.
+
+CLAIMS:
+${selfClaims.map((c, i) => `${i + 1}. ${c.statement}`).join('\n')}
+
+For each claim, generate a test that:
+1. Spawns a fresh CLI instance
+2. Sends commands to test the claimed behavior
+3. Checks expected output patterns
+
+Return JSON array:
+[{
+  "id": "test-1",
+  "description": "what we're testing",
+  "commands": ["command1", "command2"],
+  "expectedOutputs": ["pattern1", "pattern2"],
+  "expectedBehavior": "description of expected behavior",
+  "requiresBuild": false,
+  "timeout": 30000
+}]
+
+Output ONLY valid JSON array.`;
+    try {
+        const response = await llmVerifier(prompt);
+        const match = response.match(/\[[\s\S]*\]/);
+        if (match) {
+            return JSON.parse(match[0]);
+        }
+    }
+    catch {
+        // Fall through to manual generation
+    }
+    // Fallback: generate basic tests for self-claims
+    return selfClaims.map((c, i) => ({
+        id: `iso-${i}`,
+        description: c.statement,
+        commands: ['/help'], // Basic smoke test
+        expectedBehavior: c.statement,
+        timeout: 30000
+    }));
+}
+/**
+ * Runs all isolated tests and returns aggregated results
+ */
+export async function runIsolatedVerification(claims, cwd, llmVerifier) {
+    if (!llmVerifier) {
+        return { tests: [], summary: { total: 0, passed: 0, failed: 0 }, allPassed: true };
+    }
+    const tests = await generateIsolatedTests(claims, llmVerifier);
+    const results = [];
+    for (const test of tests) {
+        const result = await runIsolatedTest(test, cwd, llmVerifier);
+        results.push(result);
+    }
+    const passed = results.filter(r => r.success).length;
+    const failed = results.filter(r => !r.success).length;
+    return {
+        tests: results,
+        summary: { total: tests.length, passed, failed },
+        allPassed: failed === 0
+    };
+}
 /**
  * LLM-based claim extraction prompt.
  * Used when pattern matching isn't sufficient.
@@ -764,57 +1001,12 @@ export function generateVerificationTest(claim) {
     }
 }
 /**
- * Verify all claims in an assistant response
+ * Verify all claims in an assistant response using LLM-based semantic analysis.
+ * Requires a VerificationContext with an llmVerifier function.
+ * All claim extraction and verification is done via LLM.
  */
-export async function verifyResponse(response, responseId) {
-    const claims = extractClaims(response);
-    const results = [];
-    for (const claim of claims) {
-        const test = generateVerificationTest(claim);
-        try {
-            const result = await test();
-            results.push(result);
-        }
-        catch (err) {
-            results.push({
-                claim,
-                verified: false,
-                confidence: 'low',
-                evidence: 'Verification test failed to execute',
-                error: err instanceof Error ? err.message : 'Unknown error',
-                timestamp: new Date().toISOString()
-            });
-        }
-    }
-    const verified = results.filter(r => r.verified).length;
-    const failed = results.filter(r => !r.verified && r.confidence === 'high').length;
-    const inconclusive = results.filter(r => !r.verified && r.confidence !== 'high').length;
-    let overallVerdict;
-    if (failed > 0) {
-        overallVerdict = 'contradicted';
-    }
-    else if (verified === claims.length && claims.length > 0) {
-        overallVerdict = 'verified';
-    }
-    else if (verified > 0) {
-        overallVerdict = 'partially_verified';
-    }
-    else {
-        overallVerdict = 'unverified';
-    }
-    return {
-        responseId: responseId || `response-${Date.now()}`,
-        timestamp: new Date().toISOString(),
-        claims,
-        results,
-        summary: {
-            total: claims.length,
-            verified,
-            failed,
-            inconclusive
-        },
-        overallVerdict
-    };
+export async function verifyResponse(response, context, responseId) {
+    return verifyResponseComprehensive(response, context, responseId);
 }
 /**
  * Format a verification report for display
@@ -854,10 +1046,11 @@ export function formatVerificationReport(report) {
     return lines.join('\n');
 }
 /**
- * Quick verification - returns true if response claims are valid
+ * Quick verification - returns true if response claims are valid.
+ * Requires a VerificationContext with llmVerifier for LLM-based semantic analysis.
  */
-export async function quickVerify(response) {
-    const report = await verifyResponse(response);
+export async function quickVerify(response, context) {
+    const report = await verifyResponse(response, context);
     return report.overallVerdict === 'verified' || report.overallVerdict === 'partially_verified';
 }
 /**
@@ -1120,203 +1313,59 @@ export function generateExtendedVerificationTest(claim, context) {
         case 'error_fixed':
         case 'feature_implemented':
         case 'refactor_complete':
-            // These require semantic verification - use LLM if available
+            // These require semantic verification - LLM is required
             return async () => {
-                if (context.llmVerifier) {
-                    return verifyClaimWithLLM(claim, context);
-                }
-                // Fall back to checking if related files were modified
-                const relatedPath = claim.params.path;
-                if (relatedPath) {
-                    try {
-                        const resolvedPath = path.isAbsolute(relatedPath)
-                            ? relatedPath
-                            : path.resolve(context.workingDirectory, relatedPath);
-                        const stats = await fs.stat(resolvedPath);
-                        const recentlyModified = (Date.now() - stats.mtimeMs) < 10 * 60 * 1000;
-                        return {
-                            ...baseResult,
-                            verified: recentlyModified,
-                            confidence: 'low',
-                            evidence: recentlyModified
-                                ? `Related file ${relatedPath} was recently modified`
-                                : `Related file ${relatedPath} exists but wasn't recently modified`
-                        };
-                    }
-                    catch {
-                        return {
-                            ...baseResult,
-                            verified: false,
-                            confidence: 'low',
-                            evidence: 'Could not verify - no LLM available and related file not found'
-                        };
-                    }
+                if (!context.llmVerifier) {
+                    return {
+                        ...baseResult,
+                        verified: false,
+                        confidence: 'low',
+                        evidence: 'Semantic verification requires LLM verifier'
+                    };
                 }
-                return {
-                    ...baseResult,
-                    verified: false,
-                    confidence: 'low',
-                    evidence: 'Semantic verification required but no LLM verifier available'
-                };
+                return verifyClaimWithLLM(claim, context);
             };
         case 'data_transformed':
-            return async () => {
-                // Check if we have before/after state to compare
-                if (context.previousState && context.currentState) {
-                    const inputKey = claim.params.input;
-                    const outputKey = claim.params.output;
-                    if (inputKey && outputKey) {
-                        const inputExists = context.previousState[inputKey] !== undefined;
-                        const outputExists = context.currentState[outputKey] !== undefined;
-                        return {
-                            ...baseResult,
-                            verified: inputExists && outputExists,
-                            confidence: inputExists && outputExists ? 'medium' : 'low',
-                            evidence: `Input "${inputKey}" ${inputExists ? 'found' : 'missing'}, Output "${outputKey}" ${outputExists ? 'found' : 'missing'}`
-                        };
-                    }
-                }
-                // Fall back to LLM verification
-                if (context.llmVerifier) {
-                    return verifyClaimWithLLM(claim, context);
-                }
-                return {
-                    ...baseResult,
-                    verified: false,
-                    confidence: 'low',
-                    evidence: 'Cannot verify data transformation without state comparison or LLM'
-                };
-            };
         case 'database_updated':
-            return async () => {
-                // Can't directly verify database changes without connection info
-                // Check if there's a command we can run
-                const checkCommand = claim.params.checkCommand;
-                if (checkCommand) {
-                    try {
-                        const { stdout } = await execAsync(checkCommand, {
-                            timeout: 10000,
-                            cwd: context.workingDirectory
-                        });
-                        return {
-                            ...baseResult,
-                            verified: true,
-                            confidence: 'medium',
-                            evidence: `Check command output: ${stdout.slice(0, 200)}`
-                        };
-                    }
-                    catch (err) {
-                        return {
-                            ...baseResult,
-                            verified: false,
-                            confidence: 'medium',
-                            evidence: 'Database check command failed',
-                            error: err instanceof Error ? err.message : 'Unknown error'
-                        };
-                    }
-                }
-                // Fall back to LLM
-                if (context.llmVerifier) {
-                    return verifyClaimWithLLM(claim, context);
-                }
-                return {
-                    ...baseResult,
-                    verified: false,
-                    confidence: 'low',
-                    evidence: 'Cannot verify database changes without check command or LLM'
-                };
-            };
         case 'permission_granted':
-            return async () => {
-                const targetPath = claim.params.path;
-                const expectedMode = claim.params.mode;
-                if (targetPath) {
-                    try {
-                        const resolvedPath = path.isAbsolute(targetPath)
-                            ? targetPath
-                            : path.resolve(context.workingDirectory, targetPath);
-                        const stats = await fs.stat(resolvedPath);
-                        const mode = (stats.mode & 0o777).toString(8);
-                        if (expectedMode) {
-                            const matches = mode === expectedMode;
-                            return {
-                                ...baseResult,
-                                verified: matches,
-                                confidence: 'high',
-                                evidence: matches
-                                    ? `File has expected permissions: ${mode}`
-                                    : `Expected mode ${expectedMode}, got ${mode}`
-                            };
-                        }
-                        return {
-                            ...baseResult,
-                            verified: true,
-                            confidence: 'medium',
-                            evidence: `File permissions: ${mode}`
-                        };
-                    }
-                    catch (err) {
-                        return {
-                            ...baseResult,
-                            verified: false,
-                            confidence: 'high',
-                            evidence: 'Could not check file permissions',
-                            error: err instanceof Error ? err.message : 'Unknown error'
-                        };
-                    }
-                }
-                // Fall back to LLM
-                if (context.llmVerifier) {
-                    return verifyClaimWithLLM(claim, context);
-                }
-                return {
-                    ...baseResult,
-                    verified: false,
-                    confidence: 'low',
-                    evidence: 'Cannot verify permission without file path or LLM'
-                };
-            };
         case 'generic':
         default:
-            // For generic claims, always try LLM verification first
+            // All these claim types require LLM verification
             return async () => {
-                if (context.llmVerifier) {
-                    return verifyClaimWithLLM(claim, context);
+                if (!context.llmVerifier) {
+                    return {
+                        ...baseResult,
+                        verified: false,
+                        confidence: 'low',
+                        evidence: `${claim.type} verification requires LLM verifier`
+                    };
                 }
-                return {
-                    ...baseResult,
-                    verified: false,
-                    confidence: 'low',
-                    evidence: 'Generic claim requires LLM verification which is not available'
-                };
+                return verifyClaimWithLLM(claim, context);
             };
     }
 }
 /**
- * Comprehensive verification using both runtime and LLM-based strategies
+ * Comprehensive verification using LLM-based semantic analysis.
+ * Requires an LLM verifier - all claims are verified through LLM semantic analysis.
  */
 export async function verifyResponseComprehensive(response, context, responseId) {
-    // First extract claims using LLM if available, otherwise pattern matching
-    const claims = context.llmVerifier
-        ? await extractClaimsWithLLM(response, context.llmVerifier)
-        : extractClaims(response);
+    if (!context.llmVerifier) {
+        return {
+            responseId: responseId || `response-${Date.now()}`,
+            timestamp: new Date().toISOString(),
+            claims: [],
+            results: [],
+            summary: { total: 0, verified: 0, failed: 0, inconclusive: 0 },
+            overallVerdict: 'unverified'
+        };
+    }
+    // Extract ALL claims using LLM (required)
+    const claims = await extractClaimsWithLLM(response, context.llmVerifier);
     const results = [];
     for (const claim of claims) {
-        // Check if this is a standard claim type that can be runtime-verified
-        const standardTypes = [
-            'file_created', 'file_modified', 'file_deleted', 'code_compiles',
-            'tests_pass', 'git_committed', 'package_published', 'command_executed',
-            'dependency_installed', 'service_running', 'url_accessible', 'content_contains'
-        ];
-        let test;
-        if (standardTypes.includes(claim.type)) {
-            test = generateVerificationTest(claim);
-        }
-        else {
-            test = generateExtendedVerificationTest(claim, context);
-        }
+        // ALL claims are verified via LLM semantic analysis
         try {
-            const result = await test();
+            const result = await verifyClaimWithLLM(claim, context);
             results.push(result);
         }
         catch (err) {
@@ -1324,7 +1373,7 @@ export async function verifyResponseComprehensive(response, context, responseId)
                 claim,
                 verified: false,
                 confidence: 'low',
-                evidence: 'Verification test failed to execute',
+                evidence: 'LLM verification failed',
                 error: err instanceof Error ? err.message : 'Unknown error',
                 timestamp: new Date().toISOString()
             });
@@ -1814,4 +1863,205 @@ export async function verifyResponseHybrid(response, context, responseId) {
         overallVerdict
     };
 }
+const UNIVERSAL_EXTRACT = `Extract ALL verifiable claims from this AI response. Include explicit claims, implicit claims, state changes, results, assertions.
+
+RESPONSE:
+---
+{RESPONSE}
+---
+CONTEXT: {CONTEXT}
+DIR: {WORKING_DIR}
+
+Return JSON array: [{"id":"c1","statement":"claim","category":"file_op|code|state|data|behavior|fact|other","verifiable":true/false,"verificationApproach":"how","priority":"critical|high|medium|low","context":{}}]
+Output ONLY valid JSON.`;
+const UNIVERSAL_GEN = `Generate verification code for: {STATEMENT}
+Category: {CATEGORY} | Approach: {APPROACH} | Context: {CONTEXT} | Dir: {WORKING_DIR} | Platform: {PLATFORM}
+
+Use shell/javascript/python. READ-ONLY only.
+Return JSON: {"steps":[{"type":"shell|javascript|python","code":"code","desc":"what"}],"success":"success criteria","failure":"failure criteria","confPass":0-100,"confFail":0-100,"safe":{"ok":true/false,"why":"reason"}}
+Output ONLY valid JSON.`;
+const UNIVERSAL_ASSESS = `Assess: RESPONSE:{RESPONSE} CLAIMS:{CLAIMS} RESULTS:{RESULTS}
+Return JSON: {"trust":0-100,"summary":"text","concerns":[]}
+Output ONLY valid JSON.`;
+const UNSAFE = [/\brm\s/i, /rmdir/i, /sudo/i, /chmod\s*7/i, /eval\s*\(/i, /exec\s*\(/i, /child_process/i, /os\.system/i, /subprocess/i, /curl.*\|.*sh/i, /DROP\s+TABLE/i, /DELETE\s+FROM/i, /kill/i];
+export function validateUniversalCode(c) {
+    for (const p of UNSAFE)
+        if (p.test(c))
+            return { safe: false, reason: p.source };
+    return c.length > 5000 ? { safe: false, reason: 'too long' } : { safe: true, reason: 'ok' };
+}
+async function runUniversalStep(s, cwd) {
+    const v = validateUniversalCode(s.code);
+    if (!v.safe)
+        return { ok: false, out: v.reason };
+    try {
+        if (s.type === 'shell') {
+            const { stdout, stderr } = await execAsync(s.code, { cwd, timeout: 30000, maxBuffer: 5 * 1024 * 1024 });
+            return { ok: true, out: stdout + stderr };
+        }
+        if (s.type === 'javascript') {
+            const w = `(async()=>{try{const fs=require('fs').promises;const r=await(async()=>{${s.code}})();console.log(JSON.stringify({ok:1,r}))}catch(e){console.log(JSON.stringify({ok:0,e:e.message}))}})()`;
+            const { stdout } = await execAsync(`node -e ${JSON.stringify(w)}`, { cwd, timeout: 30000 });
+            return { ok: true, out: stdout };
+        }
+        if (s.type === 'python') {
+            const { stdout, stderr } = await execAsync(`python3 -c ${JSON.stringify(s.code)}`, { cwd, timeout: 30000 });
+            return { ok: true, out: stdout + stderr };
+        }
+        return { ok: false, out: 'unknown type' };
+    }
+    catch (e) {
+        return { ok: false, out: e instanceof Error ? e.message : 'err' };
+    }
+}
+export async function extractUniversalClaims(r, ctx) {
+    if (!ctx.llmVerifier)
+        return extractClaims(r).map((c, i) => ({ id: `c${i}`, statement: c.description, category: c.type, verifiable: true, verificationApproach: 'runtime', priority: 'medium', context: c.params }));
+    try {
+        const p = UNIVERSAL_EXTRACT.replace('{RESPONSE}', r.slice(0, 8000)).replace('{CONTEXT}', ctx.conversationHistory?.slice(-3).join('\n') || '').replace('{WORKING_DIR}', ctx.workingDirectory);
+        const res = await ctx.llmVerifier(p);
+        const m = res.match(/\[[\s\S]*\]/);
+        if (m)
+            return JSON.parse(m[0]);
+    }
+    catch { /* fall through */ }
+    return extractClaims(r).map((c, i) => ({ id: `c${i}`, statement: c.description, category: c.type, verifiable: true, verificationApproach: 'runtime', priority: 'medium', context: c.params }));
+}
+export async function verifyUniversalClaim(claim, ctx) {
+    const base = { claim, timestamp: new Date().toISOString() };
+    if (!claim.verifiable)
+        return { ...base, verified: false, confidence: 0, method: 'skip', evidence: 'Not verifiable', reasoning: 'Cannot verify' };
+    if (!ctx.llmVerifier)
+        return { ...base, verified: false, confidence: 0, method: 'skip', evidence: 'No LLM', reasoning: 'Needs LLM' };
+    try {
+        const p = UNIVERSAL_GEN.replace('{STATEMENT}', claim.statement).replace('{CATEGORY}', claim.category).replace('{APPROACH}', claim.verificationApproach).replace('{CONTEXT}', JSON.stringify(claim.context)).replace('{WORKING_DIR}', ctx.workingDirectory).replace('{PLATFORM}', process.platform);
+        const res = await ctx.llmVerifier(p);
+        const m = res.match(/\{[\s\S]*\}/);
+        if (!m)
+            throw new Error('bad');
+        const plan = JSON.parse(m[0]);
+        if (!plan.safe.ok)
+            return { ...base, verified: false, confidence: 0, method: 'blocked', evidence: plan.safe.why, reasoning: 'Unsafe' };
+        let allOk = true, out = '', code = '';
+        for (const s of plan.steps) {
+            code += s.code + '\n';
+            const r = await runUniversalStep(s, ctx.workingDirectory);
+            out += r.out + '\n';
+            if (!r.ok)
+                allOk = false;
+        }
+        return { ...base, verified: allOk, confidence: allOk ? plan.confPass : plan.confFail, method: plan.steps.map(s => s.type).join('+'), evidence: allOk ? plan.success : plan.failure, reasoning: allOk ? 'All passed' : 'Some failed', executedCode: code, rawOutput: out.slice(0, 2000) };
+    }
+    catch (e) {
+        return { ...base, verified: false, confidence: 10, method: 'error', evidence: 'Failed', reasoning: e instanceof Error ? e.message : 'err' };
+    }
+}
+export async function verifyResponseUniversal(response, ctx, id) {
+    const claims = await extractUniversalClaims(response, ctx);
+    const results = [];
+    // Identify self-referential claims (about erosolar-cli itself)
+    const selfClaims = claims.filter(c => c.statement.toLowerCase().includes('erosolar') ||
+        c.statement.toLowerCase().includes('cli') ||
+        c.category === 'behavior' ||
+        c.category === 'feature');
+    const regularClaims = claims.filter(c => !selfClaims.includes(c));
+    // Run isolated runtime tests for self-referential claims
+    if (selfClaims.length > 0 && ctx.llmVerifier) {
+        const isoResults = await runIsolatedVerification(selfClaims.map(c => ({ statement: c.statement, category: c.category, context: c.context })), ctx.workingDirectory, ctx.llmVerifier);
+        // Convert isolated results to UniversalVerificationResult
+        for (let i = 0; i < selfClaims.length && i < isoResults.tests.length; i++) {
+            const claim = selfClaims[i];
+            const isoTest = isoResults.tests[i];
+            results.push({
+                claim,
+                verified: isoTest.success,
+                confidence: isoTest.success ? 90 : (isoTest.matchedPatterns.length > 0 ? 50 : 20),
+                method: 'isolated-runtime',
+                evidence: isoTest.success ? `Verified in fresh CLI instance` : `Failed: ${isoTest.unmatchedPatterns.join(', ')}`,
+                reasoning: isoTest.llmAssessment || (isoTest.success ? 'All patterns matched in isolated runtime' : 'Patterns not matched'),
+                executedCode: isoTest.test.commands.join('\n'),
+                rawOutput: isoTest.output.slice(0, 2000),
+                timestamp: new Date().toISOString()
+            });
+        }
+    }
+    // Verify regular claims with standard approach
+    for (const c of regularClaims) {
+        results.push(c.verifiable || c.priority === 'critical' || c.priority === 'high'
+            ? await verifyUniversalClaim(c, ctx)
+            : { claim: c, verified: false, confidence: 0, method: 'skip', evidence: 'Low priority', reasoning: 'Skipped', timestamp: new Date().toISOString() });
+    }
+    const vClaims = claims.filter(c => c.verifiable).length;
+    const verified = results.filter(r => r.verified).length;
+    const failed = results.filter(r => !r.verified && r.confidence > 50).length;
+    const inconclusive = results.filter(r => !r.verified && r.confidence <= 50 && r.method !== 'skip').length;
+    const avgConf = results.length ? results.reduce((s, r) => s + r.confidence, 0) / results.length : 0;
+    // Count isolated tests for assessment
+    const isoCount = results.filter(r => r.method === 'isolated-runtime').length;
+    const isoVerified = results.filter(r => r.method === 'isolated-runtime' && r.verified).length;
+    let assessment = '', trust = 0;
+    if (ctx.llmVerifier)
+        try {
+            const isoSummary = isoCount > 0 ? ` Isolated runtime tests: ${isoVerified}/${isoCount} passed.` : '';
+            const p = UNIVERSAL_ASSESS.replace('{RESPONSE}', response.slice(0, 4000)).replace('{CLAIMS}', JSON.stringify(claims.slice(0, 15))).replace('{RESULTS}', JSON.stringify(results.slice(0, 15)));
+            const r = await ctx.llmVerifier(p);
+            const m = r.match(/\{[\s\S]*\}/);
+            if (m) {
+                const a = JSON.parse(m[0]);
+                trust = a.trust;
+                assessment = a.summary + isoSummary + (a.concerns?.length ? ` Concerns: ${a.concerns.join('; ')}` : '');
+            }
+        }
+        catch {
+            trust = Math.round(avgConf * verified / Math.max(vClaims, 1));
+            assessment = `${verified}/${vClaims} verified`;
+        }
+    else {
+        trust = Math.round(avgConf * verified / Math.max(vClaims, 1));
+        assessment = `${verified}/${vClaims} verified`;
+    }
+    return { responseId: id || `u-${Date.now()}`, originalResponse: response, timestamp: new Date().toISOString(), claims, results, summary: { totalClaims: claims.length, verifiableClaims: vClaims, verified, failed, inconclusive, averageConfidence: Math.round(avgConf) }, overallAssessment: assessment, trustScore: trust };
+}
+export async function quickUniversalVerify(r, ctx) {
+    const claims = await extractUniversalClaims(r, ctx);
+    const crit = claims.filter(c => c.verifiable && (c.priority === 'critical' || c.priority === 'high')).slice(0, 5);
+    if (!crit.length)
+        return { trustScore: 50, summary: 'No critical claims' };
+    let v = 0;
+    for (const c of crit)
+        if ((await verifyUniversalClaim(c, ctx)).verified)
+            v++;
+    return { trustScore: Math.round(v / crit.length * 100), summary: `${v}/${crit.length} critical verified` };
+}
+export function formatUniversalReport(r) {
+    const bar = '█'.repeat(Math.round(r.trustScore / 10)) + '░'.repeat(10 - Math.round(r.trustScore / 10));
+    const icon = r.trustScore >= 80 ? '✅' : r.trustScore >= 50 ? '⚠️' : '❌';
+    let out = `╔════════════════════════════════════════════════════════════╗\n║           UNIVERSAL VERIFICATION REPORT                    ║\n╚════════════════════════════════════════════════════════════╝\n\n`;
+    out += `Trust: ${icon} ${r.trustScore}/100 [${bar}]\n${r.overallAssessment}\n\nClaims: ${r.summary.totalClaims} | ✅ ${r.summary.verified} | ❌ ${r.summary.failed} | ❓ ${r.summary.inconclusive}\n\n`;
+    // Group results by method
+    const isoResults = r.results.filter(x => x.method === 'isolated-runtime');
+    const otherResults = r.results.filter(x => x.method !== 'isolated-runtime');
+    // Show isolated runtime tests first (most robust verification)
+    if (isoResults.length > 0) {
+        out += `🔬 ISOLATED RUNTIME TESTS (fresh CLI instance):\n`;
+        for (const x of isoResults.slice(0, 4)) {
+            out += `  ${x.verified ? '✅' : '❌'} [${x.confidence}%] ${x.claim.statement.slice(0, 50)}...\n`;
+            if (x.reasoning)
+                out += `     └─ ${x.reasoning.slice(0, 60)}\n`;
+        }
+        if (isoResults.length > 4)
+            out += `  ... +${isoResults.length - 4} more isolated tests\n`;
+        out += '\n';
+    }
+    // Show other verification results
+    if (otherResults.length > 0) {
+        out += `📋 STANDARD VERIFICATION:\n`;
+        for (const x of otherResults.slice(0, 6)) {
+            out += `  ${x.verified ? '✅' : x.confidence > 50 ? '❌' : '❓'} [${x.confidence}%] ${x.claim.statement.slice(0, 50)}...\n`;
+        }
+        if (otherResults.length > 6)
+            out += `  ... +${otherResults.length - 6} more\n`;
+    }
+    return out;
+}
 //# sourceMappingURL=responseVerifier.js.map