erosolar-cli 1.7.23 → 1.7.25

package/dist/core/responseVerifier.js

@@ -1,1604 +1,398 @@
 /**
- * AI Response Verification System
+ * AI Response Verification System - Isolated Runtime Only
  *
- * Automatically verifies assistant claims by:
- * 1. Extracting verifiable claims from responses
- * 2. Generating runtime verification tests
- * 3. Executing tests to verify claims
- * 4. Reporting verification results
+ * Verifies assistant claims by spawning fresh CLI instances and running
+ * actual runtime tests. All verification happens in isolation.
  *
  * @license MIT
  */
-import { exec } from 'node:child_process';
+import { exec, spawn } from 'node:child_process';
 import { promisify } from 'node:util';
 import * as fs from 'node:fs/promises';
 import * as path from 'node:path';
 const execAsync = promisify(exec);
+// ============================================================================
+// ISOLATED RUNTIME - Core Functions
+// ============================================================================
 /**
- * LLM-based claim extraction prompt.
- * Used when pattern matching isn't sufficient.
+ * Spawns a fresh isolated erosolar-cli instance for testing
  */
-const CLAIM_EXTRACTION_PROMPT = `Analyze this assistant response and extract ALL verifiable claims - anything the assistant claims to have done or accomplished.
-
-For each claim, identify:
-1. Type: One of these claim types:
-
-   FILE OPERATIONS:
-   - file_created: A new file was created
-   - file_modified: An existing file was changed
-   - file_deleted: A file was removed
-   - content_contains: A file contains specific content
-
-   BUILD/TEST:
-   - code_compiles: Code builds/compiles without errors
-   - tests_pass: Tests run successfully
-
-   VERSION CONTROL:
-   - git_committed: Changes were committed to git
-   - package_published: Package was published to npm
-
-   SYSTEM:
-   - command_executed: A shell command was run
-   - dependency_installed: A package/dependency was installed
-   - service_running: A service/server is running (on a port)
-   - url_accessible: A URL is accessible/working
-   - env_var_set: Environment variable was set
-
-   CONFIGURATION:
-   - config_changed: Configuration file was updated
-   - permission_granted: File permissions were changed
-
-   API/DATA:
-   - api_response: API returned expected response
-   - database_updated: Database record was modified
-   - data_transformed: Data was transformed correctly
-
-   SEMANTIC (require deeper analysis):
-   - error_fixed: An error/bug was fixed
-   - feature_implemented: A feature was implemented
-   - refactor_complete: Code was refactored
-
-   CATCH-ALL:
-   - generic: Any other verifiable claim not covered above
-
-2. The specific details needed to verify the claim
-
-Return a JSON array of claims. Each claim should have:
-- type: one of the types above
-- description: human readable description of what was claimed
-- params: Object with relevant fields:
-  - path: file path (for file operations)
-  - command: shell command (for command_executed)
-  - version: version number (for package_published)
-  - hash: git commit hash (for git_committed)
-  - package: package name (for dependency_installed)
-  - port: port number (for service_running)
-  - name: process/service name (for service_running, env_var_set)
-  - url: URL (for url_accessible, api_response)
-  - content: text to search for (for content_contains)
-  - key: config key path like "server.port" (for config_changed)
-  - value: expected value (for config_changed, env_var_set)
-  - status: HTTP status code (for api_response)
-  - body: expected response body (for api_response)
-  - mode: file permission mode like "755" (for permission_granted)
-  - checkCommand: command that can verify the claim (for database_updated)
-
-IMPORTANT: Extract ALL claims, including semantic ones like "I fixed the bug" or "The feature is now working". Use the 'generic' type for claims that don't fit other categories.
-
-Only skip opinions ("I think..."), explanations of what code does, or future plans ("will do", "should work").
-
-Response to analyze:
----
-{RESPONSE}
----
-
-Return ONLY valid JSON array, no other text.`;
-/**
- * Extract claims using LLM (for complex responses).
- * Falls back to pattern matching if LLM extraction fails.
- */
-export async function extractClaimsWithLLM(response, llmCall) {
-    if (!llmCall) {
-        // No LLM available, use pattern matching
-        return extractClaims(response);
-    }
+async function spawnIsolatedCLI(cwd, timeout = 60000) {
+    const cliPath = path.join(cwd, 'dist/bin/erosolar.js');
+    // Verify CLI exists
     try {
-        const prompt = CLAIM_EXTRACTION_PROMPT.replace('{RESPONSE}', response.slice(0, 4000));
-        const result = await llmCall(prompt);
-        // Parse JSON response
-        const jsonMatch = result.match(/\[[\s\S]*\]/);
-        if (!jsonMatch) {
-            return extractClaims(response);
-        }
-        const parsed = JSON.parse(jsonMatch[0]);
-        return parsed.map(claim => ({
-            type: claim.type,
-            description: claim.description,
-            evidence: 'Extracted by LLM',
-            params: claim.params
-        }));
+        await fs.access(cliPath);
     }
     catch {
-        // LLM extraction failed, fall back to patterns
-        return extractClaims(response);
-    }
+        throw new Error(`CLI not found at ${cliPath}. Run build first.`);
+    }
+    let output = '';
+    let errors = '';
+    let exitResolve;
+    const exitPromise = new Promise(resolve => { exitResolve = resolve; });
+    const child = spawn('node', [cliPath, '--plain'], {
+        cwd,
+        env: { ...process.env, EROSOLAR_TEST_MODE: '1', NO_COLOR: '1' },
+        stdio: ['pipe', 'pipe', 'pipe']
+    });
+    child.stdout.on('data', (data) => { output += data.toString(); });
+    child.stderr.on('data', (data) => { errors += data.toString(); });
+    child.on('close', (code) => { exitResolve(code); });
+    child.on('error', (err) => { errors += err.message; exitResolve(1); });
+    // Set timeout
+    const timeoutId = setTimeout(() => {
+        child.kill('SIGTERM');
+        errors += `\nTimeout after ${timeout}ms`;
+    }, timeout);
+    child.on('close', () => clearTimeout(timeoutId));
+    // Wait for startup
+    await new Promise(resolve => {
+        const checkStartup = setInterval(() => {
+            if (output.includes('erosolar') || output.includes('>') || output.length > 100) {
+                clearInterval(checkStartup);
+                resolve();
+            }
+        }, 100);
+        setTimeout(() => { clearInterval(checkStartup); resolve(); }, 2000);
+    });
+    return {
+        process: child,
+        stdin: child.stdin,
+        output,
+        errors,
+        exitPromise
+    };
 }
 /**
- * Extract verifiable claims from an assistant response.
- * Covers common patterns for file operations, builds, tests, git, and npm.
+ * Sends a command to the spawned CLI and waits for response
  */
-export function extractClaims(response) {
-    const claims = [];
-    const seenPaths = new Set();
-    // Helper to add file claim if not duplicate
-    const addFileClaim = (type, path, evidence) => {
-        if (path && !seenPaths.has(path)) {
-            seenPaths.add(path);
-            claims.push({
-                type,
-                description: `File ${path} was ${type === 'file_created' ? 'created' : 'modified'}`,
-                evidence,
-                params: { path }
-            });
-        }
-    };
-    // Pattern: File creation claims - comprehensive patterns
-    const fileCreationPatterns = [
-        // "I created file X", "Created X", "I've created X"
-        /(?:I(?:'ve)?\s+)?(?:created|wrote|written|generated|added)\s+(?:a\s+)?(?:new\s+)?(?:file\s+)?[`"']?([^\s`"',]+\.[a-zA-Z0-9]+)[`"']?/gi,
-        // "File X created", "File created at X"
-        /(?:File\s+)?[`"']?([^\s`"',]+\.[a-zA-Z0-9]+)[`"']?\s+(?:was\s+)?(?:created|written|generated)/gi,
-        // "Created file at X", "Wrote file to X"
-        /(?:created|wrote)\s+(?:a\s+)?(?:new\s+)?file\s+(?:at|to|in)\s+[`"']?([^\s`"',]+\.[a-zA-Z0-9]+)[`"']?/gi,
-        // "File created successfully" with path nearby
-        /[`"']([^\s`"',]+\.[a-zA-Z0-9]+)[`"']\s+(?:has been\s+)?(?:created|written)/gi,
-        // "successfully created X"
-        /successfully\s+(?:created|wrote|generated)\s+[`"']?([^\s`"',]+\.[a-zA-Z0-9]+)[`"']?/gi,
-        // "The file X now exists" or "X now contains"
-        /(?:the\s+)?(?:file\s+)?[`"']?([^\s`"',]+\.[a-zA-Z0-9]+)[`"']?\s+(?:now\s+)?(?:exists|contains)/gi,
-    ];
-    for (const pattern of fileCreationPatterns) {
-        pattern.lastIndex = 0; // Reset regex state
-        let match;
-        while ((match = pattern.exec(response)) !== null) {
-            if (match[1]) {
-                addFileClaim('file_created', match[1], match[0]);
+async function sendCommand(cli, command, waitMs = 5000) {
+    const outputBefore = cli.output.length;
+    cli.stdin.write(command + '\n');
+    await new Promise(resolve => {
+        let lastLength = cli.output.length;
+        const checkInterval = setInterval(() => {
+            if (cli.output.length > lastLength) {
+                lastLength = cli.output.length;
             }
-        }
-    }
-    // Pattern: File modification claims - comprehensive patterns
-    const fileModPatterns = [
-        // "I modified X", "Updated X", "I've edited X"
-        /(?:I(?:'ve)?\s+)?(?:modified|updated|changed|edited|fixed|patched|amended)\s+[`"']?([^\s`"',]+\.[a-zA-Z0-9]+)[`"']?/gi,
-        // "File X was updated"
-        /(?:File\s+)?[`"']?([^\s`"',]+\.[a-zA-Z0-9]+)[`"']?\s+(?:was\s+)?(?:modified|updated|changed|edited|fixed)/gi,
-        // "Made changes to X"
-        /(?:made\s+)?changes?\s+to\s+[`"']?([^\s`"',]+\.[a-zA-Z0-9]+)[`"']?/gi,
-        // "X has been updated"
-        /[`"']?([^\s`"',]+\.[a-zA-Z0-9]+)[`"']?\s+has\s+been\s+(?:updated|modified|changed|edited)/gi,
-        // "successfully updated X"
-        /successfully\s+(?:updated|modified|edited|fixed)\s+[`"']?([^\s`"',]+\.[a-zA-Z0-9]+)[`"']?/gi,
-    ];
-    for (const pattern of fileModPatterns) {
-        pattern.lastIndex = 0;
-        let match;
-        while ((match = pattern.exec(response)) !== null) {
-            if (match[1]) {
-                addFileClaim('file_modified', match[1], match[0]);
-            }
-        }
-    }
-    // Pattern: Command execution claims
-    const cmdPatterns = [
-        /(?:I(?:'ve)?\s+)?(?:ran|executed|run|running)\s+`([^`]+)`/gi,
-        /(?:Running|Executed|Ran)\s+`([^`]+)`/gi,
-        /`([^`]+)`\s+(?:completed|succeeded|finished|passed)/gi,
-        /executed\s+(?:the\s+)?command[:\s]+`([^`]+)`/gi,
-    ];
-    const seenCommands = new Set();
-    for (const pattern of cmdPatterns) {
-        pattern.lastIndex = 0;
-        let match;
-        while ((match = pattern.exec(response)) !== null) {
-            const command = match[1];
-            if (command && !seenCommands.has(command)) {
-                seenCommands.add(command);
-                claims.push({
-                    type: 'command_executed',
-                    description: `Command "${command.slice(0, 50)}${command.length > 50 ? '...' : ''}" was executed`,
-                    evidence: match[0],
-                    params: { command }
-                });
+            else if (cli.output.length > outputBefore) {
+                clearInterval(checkInterval);
+                resolve();
             }
-        }
-    }
-    // Pattern: Build/compile success claims
-    const buildPatterns = [
-        /(?:build|compilation|type[- ]?check)\s+(?:passed|succeeded|completed|successful|success)/gi,
-        /(?:successfully|passed)\s+(?:the\s+)?(?:build|compilation|type[- ]?check)/gi,
-        /no\s+(?:type\s+)?errors/gi,
-        /type[- ]?check(?:ing)?\s+(?:passed|succeeded|completed)/gi,
-        /(?:built|compiled)\s+successfully/gi,
-        /build\s+(?:is\s+)?(?:complete|successful)/gi,
-    ];
-    let hasBuildClaim = false;
-    for (const pattern of buildPatterns) {
-        pattern.lastIndex = 0;
-        if (!hasBuildClaim && pattern.test(response)) {
-            claims.push({
-                type: 'code_compiles',
-                description: 'Code compiles without errors',
-                evidence: response.match(pattern)?.[0] || '',
-                params: {}
-            });
-            hasBuildClaim = true;
-            break;
-        }
-    }
-    // Pattern: Test pass claims
-    const testPatterns = [
-        /(?:all\s+)?tests?\s+(?:pass|passed|passing|succeeded|successful)/gi,
-        /(?:passed|passing)\s+(?:all\s+)?tests?/gi,
-        /(\d+)\s+tests?\s+passed/gi,
-        /tests?\s+(?:completed|finished)\s+successfully/gi,
-        /(?:test|tests)\s+suite\s+(?:passed|succeeded)/gi,
-        /all\s+(\d+)\s+tests?\s+(?:pass|passed)/gi,
-    ];
-    let hasTestClaim = false;
-    for (const pattern of testPatterns) {
-        pattern.lastIndex = 0;
-        const match = pattern.exec(response);
-        if (!hasTestClaim && match) {
-            claims.push({
-                type: 'tests_pass',
-                description: 'Tests pass',
-                evidence: match[0],
-                params: { count: match[1] ? parseInt(match[1], 10) : undefined }
-            });
-            hasTestClaim = true;
-            break;
-        }
+        }, 200);
+        setTimeout(() => { clearInterval(checkInterval); resolve(); }, waitMs);
+    });
+    return cli.output.slice(outputBefore);
+}
+/**
+ * Run a shell command for verification (file checks, etc.)
+ */
+async function runShellVerification(cmd, cwd) {
+    // Safety check - block dangerous commands
+    const dangerous = [/\brm\s/i, /rmdir/i, /sudo/i, /chmod\s*7/i, /eval\s*\(/i, /DROP\s+TABLE/i, /DELETE\s+FROM/i];
+    for (const p of dangerous) {
+        if (p.test(cmd))
+            return { ok: false, out: `Blocked dangerous command: ${p.source}` };
     }
-    // Pattern: Git commit claims
-    const gitPatterns = [
-        /committed\s+(?:the\s+)?(?:changes?\s+)?(?:with\s+message\s+)?["']?([^"'\n]+)["']?/gi,
-        /\[(?:main|master|[a-zA-Z0-9/_-]+)\s+([a-f0-9]{7,})\]/gi,
-        /git\s+commit.*-m\s+["']([^"']+)["']/gi,
-        /created\s+(?:a\s+)?commit/gi,
-        /changes?\s+(?:have\s+been\s+)?committed/gi,
-        /commit\s+([a-f0-9]{7,})/gi,
-    ];
-    let hasGitClaim = false;
-    for (const pattern of gitPatterns) {
-        pattern.lastIndex = 0;
-        const match = pattern.exec(response);
-        if (!hasGitClaim && match) {
-            claims.push({
-                type: 'git_committed',
-                description: 'Changes were committed',
-                evidence: match[0],
-                params: { hash: match[1] }
-            });
-            hasGitClaim = true;
-            break;
-        }
+    try {
+        const { stdout, stderr } = await execAsync(cmd, { cwd, timeout: 30000 });
+        return { ok: true, out: stdout + stderr };
     }
-    // Pattern: Package publish claims
-    const publishPatterns = [
-        /published\s+(?:to\s+)?(?:npm|registry)/gi,
-        /\+\s+[a-z@/_-]+@(\d+\.\d+\.\d+)/gi,
-        /npm\s+publish/gi,
-        /package\s+(?:was\s+)?published/gi,
-        /published\s+(?:version\s+)?v?(\d+\.\d+\.\d+)/gi,
-        /successfully\s+published/gi,
-    ];
-    let hasPublishClaim = false;
-    for (const pattern of publishPatterns) {
-        pattern.lastIndex = 0;
-        const match = pattern.exec(response);
-        if (!hasPublishClaim && match) {
-            claims.push({
-                type: 'package_published',
-                description: 'Package was published',
-                evidence: match[0],
-                params: { version: match[1] }
-            });
-            hasPublishClaim = true;
-            break;
-        }
+    catch (e) {
+        return { ok: false, out: e instanceof Error ? e.message : 'Command failed' };
     }
-    // Pattern: File deletion claims
-    const deletionPatterns = [
-        /(?:I(?:'ve)?\s+)?(?:deleted|removed)\s+(?:the\s+)?(?:file\s+)?[`"']?([^\s`"',]+\.[a-zA-Z0-9]+)[`"']?/gi,
-        /[`"']?([^\s`"',]+\.[a-zA-Z0-9]+)[`"']?\s+(?:was\s+)?(?:deleted|removed)/gi,
-    ];
-    for (const pattern of deletionPatterns) {
-        pattern.lastIndex = 0;
-        let match;
-        while ((match = pattern.exec(response)) !== null) {
-            const filePath = match[1];
-            if (filePath && !seenPaths.has(filePath)) {
-                seenPaths.add(filePath);
-                claims.push({
-                    type: 'file_deleted',
-                    description: `File ${filePath} was deleted`,
-                    evidence: match[0],
-                    params: { path: filePath }
-                });
+}
+/**
+ * Runs an isolated runtime test
+ */
+export async function runIsolatedTest(test, cwd, llmVerifier) {
+    const startTime = Date.now();
+    const result = {
+        test,
+        success: false,
+        output: '',
+        errors: '',
+        exitCode: null,
+        duration: 0,
+        matchedPatterns: [],
+        unmatchedPatterns: []
+    };
+    try {
+        // Rebuild if required
+        if (test.requiresBuild) {
+            try {
+                await execAsync('npm run build', { cwd, timeout: 120000 });
             }
-        }
-    }
-    // Pattern: Dependency installation claims
-    const installPatterns = [
-        /(?:installed|added)\s+(?:the\s+)?(?:package|dependency)\s+[`"']?([^\s`"',]+)[`"']?/gi,
-        /npm\s+install(?:ed)?\s+[`"']?([^\s`"',]+)[`"']?/gi,
-        /(?:package|dependency)\s+[`"']?([^\s`"',]+)[`"']?\s+(?:was\s+)?installed/gi,
-    ];
-    for (const pattern of installPatterns) {
-        pattern.lastIndex = 0;
-        let match;
-        while ((match = pattern.exec(response)) !== null) {
-            const packageName = match[1];
-            if (packageName) {
-                claims.push({
-                    type: 'dependency_installed',
-                    description: `Package ${packageName} was installed`,
-                    evidence: match[0],
-                    params: { package: packageName }
-                });
+            catch (buildErr) {
+                result.errors = `Build failed: ${buildErr instanceof Error ? buildErr.message : 'unknown'}`;
+                result.duration = Date.now() - startTime;
+                return result;
             }
         }
-    }
-    // Pattern: Service running claims
-    const servicePatterns = [
-        /(?:server|service|app(?:lication)?)\s+(?:is\s+)?(?:running|started|listening)\s+(?:on\s+)?(?:port\s+)?(\d+)/gi,
-        /(?:listening|running)\s+(?:on\s+)?(?:port\s+)?(\d+)/gi,
-        /started\s+(?:the\s+)?(?:server|service)\s+(?:on\s+)?(?:port\s+)?(\d+)/gi,
-        /(?:port\s+)?(\d+)\s+is\s+(?:now\s+)?(?:open|listening)/gi,
-    ];
-    for (const pattern of servicePatterns) {
-        pattern.lastIndex = 0;
-        const match = pattern.exec(response);
-        if (match && match[1]) {
-            const port = parseInt(match[1], 10);
-            if (port > 0 && port < 65536) {
-                claims.push({
-                    type: 'service_running',
-                    description: `Service running on port ${port}`,
-                    evidence: match[0],
-                    params: { port }
-                });
-                break; // Only one service claim per response
+        // Run shell commands first if any (file checks, etc.)
+        if (test.shellCommands && test.shellCommands.length > 0) {
+            for (const cmd of test.shellCommands) {
+                const shellResult = await runShellVerification(cmd, cwd);
+                result.output += `$ ${cmd}\n${shellResult.out}\n`;
+                if (!shellResult.ok) {
+                    result.errors += shellResult.out + '\n';
+                }
             }
         }
-    }
-    // Pattern: URL accessible claims
-    const urlPatterns = [
-        /(?:accessible|available|live)\s+at\s+(https?:\/\/[^\s]+)/gi,
-        /(?:visit|open|access)\s+(https?:\/\/[^\s]+)/gi,
-        /(https?:\/\/[^\s]+)\s+(?:is\s+)?(?:now\s+)?(?:accessible|available|live)/gi,
-        /deployed\s+(?:to|at)\s+(https?:\/\/[^\s]+)/gi,
-    ];
-    for (const pattern of urlPatterns) {
-        pattern.lastIndex = 0;
-        const match = pattern.exec(response);
-        if (match && match[1]) {
-            const url = match[1].replace(/[.,;:!?)]+$/, ''); // Remove trailing punctuation
-            claims.push({
-                type: 'url_accessible',
-                description: `URL ${url} is accessible`,
-                evidence: match[0],
-                params: { url }
-            });
-            break; // Only one URL claim per response
-        }
-    }
-    // Pattern: Content contains claims
-    const contentPatterns = [
-        /(?:file\s+)?[`"']?([^\s`"',]+\.[a-zA-Z0-9]+)[`"']?\s+(?:now\s+)?contains?\s+[`"']([^`"']+)[`"']/gi,
-        /added\s+[`"']([^`"']+)[`"']\s+to\s+[`"']?([^\s`"',]+\.[a-zA-Z0-9]+)[`"']?/gi,
-    ];
-    for (const pattern of contentPatterns) {
-        pattern.lastIndex = 0;
-        const match = pattern.exec(response);
-        if (match) {
-            // Pattern 1: file contains "text"
-            // Pattern 2: added "text" to file
-            const isPattern2 = pattern.source.startsWith('added');
-            const filePath = isPattern2 ? match[2] : match[1];
-            const content = isPattern2 ? match[1] : match[2];
-            if (filePath && content) {
-                claims.push({
-                    type: 'content_contains',
-                    description: `File ${filePath} contains specified content`,
-                    evidence: match[0],
-                    params: { path: filePath, content }
-                });
+        // Run CLI commands if any
+        if (test.commands && test.commands.length > 0) {
+            const cli = await spawnIsolatedCLI(cwd, test.timeout || 60000);
+            for (const cmd of test.commands) {
+                const cmdOutput = await sendCommand(cli, cmd);
+                result.output += `> ${cmd}\n${cmdOutput}\n`;
             }
+            cli.stdin.write('/quit\n');
+            await new Promise(resolve => setTimeout(resolve, 500));
+            cli.process.kill('SIGTERM');
+            result.exitCode = await cli.exitPromise;
+            result.errors += cli.errors;
         }
-    }
-    return claims;
-}
-/**
- * Generate a verification test for a claim
- */
-export function generateVerificationTest(claim) {
-    const baseResult = {
-        claim,
-        timestamp: new Date().toISOString()
-    };
-    switch (claim.type) {
-        case 'file_created':
-        case 'file_modified':
-            return async () => {
-                const filePath = claim.params.path;
-                try {
-                    const resolvedPath = path.isAbsolute(filePath) ? filePath : path.resolve(process.cwd(), filePath);
-                    const stats = await fs.stat(resolvedPath);
-                    const recentlyModified = (Date.now() - stats.mtimeMs) < 5 * 60 * 1000; // Within 5 minutes
-                    return {
-                        ...baseResult,
-                        verified: stats.isFile(),
-                        confidence: recentlyModified ? 'high' : 'medium',
-                        evidence: `File exists. Size: ${stats.size} bytes. Modified: ${stats.mtime.toISOString()}`
-                    };
+        // Check expected output patterns
+        if (test.expectedOutputs) {
+            for (const pattern of test.expectedOutputs) {
+                if (result.output.includes(pattern) || new RegExp(pattern, 'i').test(result.output)) {
+                    result.matchedPatterns.push(pattern);
                 }
-                catch (err) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'high',
-                        evidence: 'File does not exist',
-                        error: err instanceof Error ? err.message : 'Unknown error'
-                    };
-                }
-            };
-        case 'file_deleted':
-            return async () => {
-                const filePath = claim.params.path;
-                try {
-                    const resolvedPath = path.isAbsolute(filePath) ? filePath : path.resolve(process.cwd(), filePath);
-                    await fs.stat(resolvedPath);
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'high',
-                        evidence: 'File still exists (deletion claim is false)'
-                    };
+                else {
+                    result.unmatchedPatterns.push(pattern);
                 }
-                catch {
-                    return {
-                        ...baseResult,
-                        verified: true,
-                        confidence: 'high',
-                        evidence: 'File does not exist (deletion verified)'
-                    };
-                }
-            };
-        case 'code_compiles':
-            return async () => {
-                try {
-                    const { stdout, stderr } = await execAsync('npm run type-check 2>&1 || npm run build 2>&1', {
-                        timeout: 60000,
-                        cwd: process.cwd()
-                    });
-                    const output = stdout + stderr;
-                    const hasErrors = /error/i.test(output) && !/0 errors/i.test(output);
-                    return {
-                        ...baseResult,
-                        verified: !hasErrors,
-                        confidence: 'high',
-                        evidence: hasErrors ? `Compilation errors found: ${output.slice(0, 500)}` : 'Code compiles successfully'
-                    };
-                }
-                catch (err) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'high',
-                        evidence: 'Compilation check failed',
-                        error: err instanceof Error ? err.message : 'Unknown error'
-                    };
-                }
-            };
-        case 'tests_pass':
-            return async () => {
-                try {
-                    const { stdout, stderr } = await execAsync('npm test 2>&1', {
-                        timeout: 120000,
-                        cwd: process.cwd()
-                    });
-                    const output = stdout + stderr;
-                    const hasFailed = /fail|error/i.test(output) && !/0 failed/i.test(output);
-                    return {
-                        ...baseResult,
-                        verified: !hasFailed,
-                        confidence: 'high',
-                        evidence: hasFailed ? `Test failures: ${output.slice(0, 500)}` : 'All tests pass'
-                    };
-                }
-                catch (err) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'high',
-                        evidence: 'Test execution failed',
-                        error: err instanceof Error ? err.message : 'Unknown error'
-                    };
-                }
-            };
-        case 'git_committed':
-            return async () => {
-                try {
-                    const { stdout } = await execAsync('git log -1 --oneline', {
-                        timeout: 5000,
-                        cwd: process.cwd()
-                    });
-                    const hash = claim.params.hash;
-                    if (hash && stdout.includes(hash.slice(0, 7))) {
-                        return {
-                            ...baseResult,
-                            verified: true,
-                            confidence: 'high',
-                            evidence: `Commit found: ${stdout.trim()}`
-                        };
-                    }
-                    // Check if there's a recent commit
-                    const { stdout: logOutput } = await execAsync('git log -1 --format="%H %s"', {
-                        timeout: 5000
-                    });
-                    return {
-                        ...baseResult,
-                        verified: true,
-                        confidence: 'medium',
-                        evidence: `Most recent commit: ${logOutput.trim()}`
-                    };
-                }
-                catch (err) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'high',
-                        evidence: 'Git check failed',
-                        error: err instanceof Error ? err.message : 'Unknown error'
-                    };
-                }
-            };
-        case 'package_published':
-            return async () => {
-                try {
-                    // Read package.json to get name and version
-                    const pkgPath = path.resolve(process.cwd(), 'package.json');
-                    const pkgContent = await fs.readFile(pkgPath, 'utf-8');
-                    const pkg = JSON.parse(pkgContent);
-                    const { stdout } = await execAsync(`npm view ${pkg.name}@${pkg.version} version 2>&1`, {
-                        timeout: 10000
-                    });
-                    const published = stdout.trim() === pkg.version;
-                    return {
-                        ...baseResult,
-                        verified: published,
-                        confidence: 'high',
-                        evidence: published ? `${pkg.name}@${pkg.version} found on npm` : 'Version not found on npm'
-                    };
-                }
-                catch (err) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'medium',
-                        evidence: 'Could not verify npm publication',
-                        error: err instanceof Error ? err.message : 'Unknown error'
-                    };
-                }
-            };
-        case 'command_executed':
-            // Can't really verify past command execution, just acknowledge
-            return async () => ({
-                ...baseResult,
-                verified: true, // Assume true since we can't replay
-                confidence: 'low',
-                evidence: 'Command execution cannot be retroactively verified'
-            });
-        case 'dependency_installed':
-            return async () => {
-                const packageName = claim.params.package;
-                if (!packageName) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'low',
-                        evidence: 'No package name provided'
-                    };
-                }
-                try {
-                    // Check if package exists in node_modules
-                    const modulePath = path.resolve(process.cwd(), 'node_modules', packageName);
-                    await fs.stat(modulePath);
-                    // Also verify in package.json
-                    const pkgPath = path.resolve(process.cwd(), 'package.json');
-                    const pkgContent = await fs.readFile(pkgPath, 'utf-8');
-                    const pkg = JSON.parse(pkgContent);
-                    const inDeps = pkg.dependencies?.[packageName] || pkg.devDependencies?.[packageName];
-                    return {
-                        ...baseResult,
-                        verified: true,
-                        confidence: inDeps ? 'high' : 'medium',
-                        evidence: inDeps
-                            ? `Package ${packageName} installed (${inDeps})`
-                            : `Package ${packageName} found in node_modules but not in package.json`
-                    };
-                }
-                catch {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'high',
-                        evidence: `Package ${packageName} not found in node_modules`
-                    };
-                }
-            };
-        case 'service_running':
-            return async () => {
-                const port = claim.params.port;
-                const name = claim.params.name;
-                try {
-                    if (port) {
-                        // Check if port is in use
-                        const { stdout } = await execAsync(`lsof -i :${port} 2>/dev/null || netstat -an | grep ${port}`, {
-                            timeout: 5000
-                        });
-                        const isRunning = stdout.trim().length > 0;
-                        return {
-                            ...baseResult,
-                            verified: isRunning,
-                            confidence: 'high',
-                            evidence: isRunning ? `Service running on port ${port}` : `No service found on port ${port}`
-                        };
-                    }
-                    else if (name) {
-                        // Check if process is running by name
-                        const { stdout } = await execAsync(`pgrep -f "${name}" 2>/dev/null || ps aux | grep "${name}" | grep -v grep`, {
-                            timeout: 5000
-                        });
-                        const isRunning = stdout.trim().length > 0;
-                        return {
-                            ...baseResult,
-                            verified: isRunning,
-                            confidence: 'medium',
-                            evidence: isRunning ? `Process "${name}" appears to be running` : `Process "${name}" not found`
-                        };
-                    }
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'low',
-                        evidence: 'No port or service name provided for verification'
-                    };
-                }
-                catch {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'medium',
-                        evidence: 'Could not verify service status'
-                    };
-                }
-            };
-        case 'url_accessible':
-            return async () => {
-                const url = claim.params.url;
-                if (!url) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'low',
-                        evidence: 'No URL provided'
-                    };
-                }
-                try {
-                    const { stdout } = await execAsync(`curl -s -o /dev/null -w "%{http_code}" "${url}" 2>&1`, {
-                        timeout: 10000
-                    });
-                    const statusCode = parseInt(stdout.trim(), 10);
-                    const isAccessible = statusCode >= 200 && statusCode < 400;
-                    return {
-                        ...baseResult,
-                        verified: isAccessible,
-                        confidence: 'high',
-                        evidence: `URL returned status ${statusCode}`
-                    };
-                }
-                catch (err) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'high',
-                        evidence: 'URL is not accessible',
-                        error: err instanceof Error ? err.message : 'Unknown error'
-                    };
-                }
-            };
-        case 'content_contains':
-            return async () => {
-                const filePath = claim.params.path;
-                const searchText = claim.params.content;
-                if (!filePath || !searchText) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'low',
-                        evidence: 'Missing file path or search content'
-                    };
-                }
-                try {
-                    const resolvedPath = path.isAbsolute(filePath) ? filePath : path.resolve(process.cwd(), filePath);
-                    const content = await fs.readFile(resolvedPath, 'utf-8');
-                    const contains = content.includes(searchText);
-                    return {
-                        ...baseResult,
-                        verified: contains,
-                        confidence: 'high',
-                        evidence: contains
-                            ? `File contains the expected content`
-                            : `File does not contain "${searchText.slice(0, 50)}..."`
-                    };
-                }
-                catch (err) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'high',
-                        evidence: 'Could not read file',
-                        error: err instanceof Error ? err.message : 'Unknown error'
-                    };
-                }
-            };
-        default:
-            return async () => ({
-                ...baseResult,
-                verified: false,
-                confidence: 'low',
-                evidence: `Unknown claim type: ${claim.type}`
-            });
-    }
-}
-/**
- * Verify all claims in an assistant response using LLM-based semantic analysis.
- * Requires a VerificationContext with an llmVerifier function.
- * All claim extraction and verification is done via LLM.
- */
-export async function verifyResponse(response, context, responseId) {
-    return verifyResponseComprehensive(response, context, responseId);
-}
-/**
- * Format a verification report for display
- */
-export function formatVerificationReport(report) {
-    const lines = [];
-    lines.push('═══════════════════════════════════════════════════════════');
-    lines.push('  RESPONSE VERIFICATION REPORT');
-    lines.push('═══════════════════════════════════════════════════════════');
-    lines.push('');
-    const verdictEmoji = {
-        verified: '✅',
-        partially_verified: '⚠️',
-        unverified: '❓',
-        contradicted: '❌'
-    };
-    lines.push(`Verdict: ${verdictEmoji[report.overallVerdict]} ${report.overallVerdict.toUpperCase()}`);
-    lines.push(`Claims: ${report.summary.total} total, ${report.summary.verified} verified, ${report.summary.failed} failed`);
-    lines.push('');
-    if (report.results.length > 0) {
-        lines.push('Verification Details:');
-        lines.push('─────────────────────');
-        for (const result of report.results) {
-            const icon = result.verified ? '✅' : (result.confidence === 'high' ? '❌' : '❓');
-            lines.push(`${icon} ${result.claim.description}`);
-            lines.push(`   Evidence: ${result.evidence.slice(0, 100)}`);
-            if (result.error) {
-                lines.push(`   Error: ${result.error}`);
             }
         }
-    }
-    else {
-        lines.push('No verifiable claims found in response.');
-    }
-    lines.push('');
-    lines.push('═══════════════════════════════════════════════════════════');
-    return lines.join('\n');
-}
-/**
- * Quick verification - returns true if response claims are valid.
- * Requires a VerificationContext with llmVerifier for LLM-based semantic analysis.
- */
-export async function quickVerify(response, context) {
-    const report = await verifyResponse(response, context);
-    return report.overallVerdict === 'verified' || report.overallVerdict === 'partially_verified';
-}
-/**
- * LLM-based verification prompt for claims that can't be programmatically verified
- */
-const LLM_VERIFICATION_PROMPT = `You are a verification assistant. Analyze whether the following claim is likely TRUE or FALSE based on the evidence provided.
-
-CLAIM: {CLAIM}
+        // LLM assessment of behavior
+        if (test.expectedBehavior && llmVerifier) {
+            const assessPrompt = `Assess if this output demonstrates the expected behavior.
 
-EVIDENCE/CONTEXT:
-{CONTEXT}
+EXPECTED: ${test.expectedBehavior}
 
-Respond with a JSON object:
-{
-  "verdict": "verified" | "unverified" | "inconclusive",
-  "confidence": "high" | "medium" | "low",
-  "reasoning": "Brief explanation of your analysis",
-  "suggested_test": "Optional: A command or check that could verify this claim"
-}
+OUTPUT:
+---
+${result.output.slice(0, 4000)}
+---
 
-Be conservative - only mark as "verified" if there's strong evidence. Mark as "inconclusive" if you can't determine the truth.`;
-/**
- * Verify a claim using LLM when runtime verification isn't possible
- */
-export async function verifyClaimWithLLM(claim, context) {
-    const baseResult = {
-        claim,
-        timestamp: new Date().toISOString()
-    };
-    if (!context.llmVerifier) {
-        return {
-            ...baseResult,
-            verified: false,
-            confidence: 'low',
-            evidence: 'No LLM verifier available for semantic verification'
-        };
-    }
-    try {
-        // Build context string
-        const contextParts = [];
-        if (context.previousState) {
-            contextParts.push(`Previous state: ${JSON.stringify(context.previousState, null, 2)}`);
-        }
-        if (context.currentState) {
-            contextParts.push(`Current state: ${JSON.stringify(context.currentState, null, 2)}`);
-        }
-        if (context.conversationHistory?.length) {
-            contextParts.push(`Recent conversation:\n${context.conversationHistory.slice(-5).join('\n')}`);
-        }
-        contextParts.push(`Claim evidence: ${claim.evidence}`);
-        contextParts.push(`Claim params: ${JSON.stringify(claim.params)}`);
-        const prompt = LLM_VERIFICATION_PROMPT
-            .replace('{CLAIM}', claim.description)
-            .replace('{CONTEXT}', contextParts.join('\n\n'));
-        const result = await context.llmVerifier(prompt);
-        // Parse LLM response
-        const jsonMatch = result.match(/\{[\s\S]*\}/);
-        if (!jsonMatch) {
-            return {
-                ...baseResult,
-                verified: false,
-                confidence: 'low',
-                evidence: 'LLM verification returned invalid response'
-            };
-        }
-        const parsed = JSON.parse(jsonMatch[0]);
-        return {
-            ...baseResult,
-            verified: parsed.verdict === 'verified',
-            confidence: parsed.confidence || 'medium',
-            evidence: `LLM Analysis: ${parsed.reasoning}${parsed.suggested_test ? ` (Suggested test: ${parsed.suggested_test})` : ''}`
-        };
-    }
-    catch (err) {
-        return {
-            ...baseResult,
-            verified: false,
-            confidence: 'low',
-            evidence: 'LLM verification failed',
-            error: err instanceof Error ? err.message : 'Unknown error'
-        };
-    }
-}
-/**
- * Generate verification test for extended claim types
- */
-export function generateExtendedVerificationTest(claim, context) {
-    const baseResult = {
-        claim,
-        timestamp: new Date().toISOString()
-    };
-    switch (claim.type) {
-        case 'api_response':
-            return async () => {
-                const url = claim.params.url;
-                const expectedStatus = claim.params.status;
-                const expectedBody = claim.params.body;
-                if (!url) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'low',
-                        evidence: 'No API URL provided'
-                    };
-                }
-                try {
-                    const { stdout } = await execAsync(`curl -s -w "\\n%{http_code}" "${url}" 2>&1`, { timeout: 15000 });
-                    const lines = stdout.trim().split('\n');
-                    const statusCode = parseInt(lines.pop() || '0', 10);
-                    const body = lines.join('\n');
-                    let verified = true;
-                    const evidenceParts = [];
-                    if (expectedStatus && statusCode !== expectedStatus) {
-                        verified = false;
-                        evidenceParts.push(`Expected status ${expectedStatus}, got ${statusCode}`);
+Return JSON: {"matches": true/false, "confidence": 0-100, "reasoning": "explanation"}`;
+            try {
+                const assessment = await llmVerifier(assessPrompt);
+                const match = assessment.match(/\{[\s\S]*\}/);
+                if (match) {
+                    const parsed = JSON.parse(match[0]);
+                    result.llmAssessment = `${parsed.matches ? '✅' : '❌'} [${parsed.confidence}%] ${parsed.reasoning}`;
+                    if (!parsed.matches || parsed.confidence < 70) {
+                        result.unmatchedPatterns.push(`behavior: ${test.expectedBehavior}`);
                     }
                     else {
-                        evidenceParts.push(`Status: ${statusCode}`);
-                    }
-                    if (expectedBody && !body.includes(expectedBody)) {
-                        verified = false;
-                        evidenceParts.push(`Expected body to contain "${expectedBody.slice(0, 50)}..."`);
+                        result.matchedPatterns.push(`behavior: ${test.expectedBehavior}`);
                     }
-                    return {
-                        ...baseResult,
-                        verified,
-                        confidence: 'high',
-                        evidence: evidenceParts.join('. ')
-                    };
                 }
-                catch (err) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'high',
-                        evidence: 'API request failed',
-                        error: err instanceof Error ? err.message : 'Unknown error'
-                    };
-                }
-            };
-        case 'env_var_set':
-            return async () => {
-                const varName = claim.params.name;
-                const expectedValue = claim.params.value;
-                if (!varName) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'low',
-                        evidence: 'No environment variable name provided'
-                    };
-                }
-                const actualValue = process.env[varName];
-                if (actualValue === undefined) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'high',
-                        evidence: `Environment variable ${varName} is not set`
-                    };
-                }
-                if (expectedValue && actualValue !== expectedValue) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'high',
-                        evidence: `Expected ${varName}="${expectedValue}", got "${actualValue}"`
-                    };
-                }
-                return {
-                    ...baseResult,
-                    verified: true,
-                    confidence: 'high',
-                    evidence: `${varName} is set${expectedValue ? ` to expected value` : `: ${actualValue.slice(0, 50)}`}`
-                };
-            };
-        case 'config_changed':
-            return async () => {
-                const configPath = claim.params.path;
-                const expectedKey = claim.params.key;
-                const expectedValue = claim.params.value;
-                if (!configPath) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'low',
-                        evidence: 'No config file path provided'
-                    };
-                }
-                try {
-                    const resolvedPath = path.isAbsolute(configPath)
-                        ? configPath
-                        : path.resolve(context.workingDirectory, configPath);
-                    const content = await fs.readFile(resolvedPath, 'utf-8');
-                    // Try to parse as JSON
-                    let config;
-                    try {
-                        config = JSON.parse(content);
-                    }
-                    catch {
-                        // Not JSON, check raw content
-                        if (expectedValue && content.includes(String(expectedValue))) {
-                            return {
-                                ...baseResult,
-                                verified: true,
-                                confidence: 'medium',
-                                evidence: `Config file contains expected value`
-                            };
-                        }
-                        return {
-                            ...baseResult,
-                            verified: true,
-                            confidence: 'low',
-                            evidence: 'Config file exists but format unknown'
-                        };
-                    }
-                    if (expectedKey) {
-                        const keys = expectedKey.split('.');
-                        let value = config;
-                        for (const key of keys) {
-                            value = value?.[key];
-                        }
-                        if (expectedValue !== undefined) {
-                            const matches = JSON.stringify(value) === JSON.stringify(expectedValue);
-                            return {
-                                ...baseResult,
-                                verified: matches,
-                                confidence: 'high',
-                                evidence: matches
-                                    ? `${expectedKey} has expected value`
-                                    : `${expectedKey} = ${JSON.stringify(value)}, expected ${JSON.stringify(expectedValue)}`
-                            };
-                        }
-                        return {
-                            ...baseResult,
-                            verified: value !== undefined,
-                            confidence: 'high',
-                            evidence: value !== undefined
-                                ? `${expectedKey} exists: ${JSON.stringify(value).slice(0, 100)}`
-                                : `${expectedKey} not found in config`
-                        };
-                    }
-                    return {
-                        ...baseResult,
-                        verified: true,
-                        confidence: 'medium',
-                        evidence: 'Config file exists and is valid JSON'
-                    };
-                }
-                catch (err) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'high',
-                        evidence: 'Could not read config file',
-                        error: err instanceof Error ? err.message : 'Unknown error'
-                    };
-                }
-            };
-        case 'error_fixed':
-        case 'feature_implemented':
-        case 'refactor_complete':
-            // These require semantic verification - LLM is required
-            return async () => {
-                if (!context.llmVerifier) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'low',
-                        evidence: 'Semantic verification requires LLM verifier'
-                    };
-                }
-                return verifyClaimWithLLM(claim, context);
-            };
-        case 'data_transformed':
-        case 'database_updated':
-        case 'permission_granted':
-        case 'generic':
-        default:
-            // All these claim types require LLM verification
-            return async () => {
-                if (!context.llmVerifier) {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'low',
-                        evidence: `${claim.type} verification requires LLM verifier`
-                    };
-                }
-                return verifyClaimWithLLM(claim, context);
-            };
-    }
-}
-/**
- * Comprehensive verification using LLM-based semantic analysis.
- * Requires an LLM verifier - all claims are verified through LLM semantic analysis.
- */
-export async function verifyResponseComprehensive(response, context, responseId) {
-    if (!context.llmVerifier) {
-        return {
-            responseId: responseId || `response-${Date.now()}`,
-            timestamp: new Date().toISOString(),
-            claims: [],
-            results: [],
-            summary: { total: 0, verified: 0, failed: 0, inconclusive: 0 },
-            overallVerdict: 'unverified'
-        };
-    }
-    // Extract ALL claims using LLM (required)
-    const claims = await extractClaimsWithLLM(response, context.llmVerifier);
-    const results = [];
-    for (const claim of claims) {
-        // ALL claims are verified via LLM semantic analysis
-        try {
-            const result = await verifyClaimWithLLM(claim, context);
-            results.push(result);
-        }
-        catch (err) {
-            results.push({
-                claim,
-                verified: false,
-                confidence: 'low',
-                evidence: 'LLM verification failed',
-                error: err instanceof Error ? err.message : 'Unknown error',
-                timestamp: new Date().toISOString()
-            });
+            }
+            catch {
+                result.llmAssessment = 'LLM assessment failed';
+            }
         }
+        // Determine success
+        result.success = result.unmatchedPatterns.length === 0 &&
+            (result.matchedPatterns.length > 0 || (!test.expectedOutputs?.length && !test.expectedBehavior));
     }
-    const verified = results.filter(r => r.verified).length;
-    const failed = results.filter(r => !r.verified && r.confidence === 'high').length;
-    const inconclusive = results.filter(r => !r.verified && r.confidence !== 'high').length;
-    let overallVerdict;
-    if (failed > 0) {
-        overallVerdict = 'contradicted';
-    }
-    else if (verified === claims.length && claims.length > 0) {
-        overallVerdict = 'verified';
-    }
-    else if (verified > 0) {
-        overallVerdict = 'partially_verified';
-    }
-    else {
-        overallVerdict = 'unverified';
-    }
-    return {
-        responseId: responseId || `response-${Date.now()}`,
-        timestamp: new Date().toISOString(),
-        claims,
-        results,
-        summary: {
-            total: claims.length,
-            verified,
-            failed,
-            inconclusive
-        },
-        overallVerdict
-    };
-}
-/**
- * Determine the best verification strategy for a claim
- */
-export function getVerificationStrategy(claim) {
-    switch (claim.type) {
-        case 'file_created':
-        case 'file_modified':
-        case 'file_deleted':
-        case 'content_contains':
-        case 'config_changed':
-        case 'permission_granted':
-            return 'filesystem';
-        case 'url_accessible':
-        case 'api_response':
-        case 'service_running':
-            return 'network';
-        case 'code_compiles':
-        case 'tests_pass':
-        case 'command_executed':
-        case 'dependency_installed':
-        case 'git_committed':
-        case 'package_published':
-        case 'env_var_set':
-            return 'runtime';
-        case 'error_fixed':
-        case 'feature_implemented':
-        case 'refactor_complete':
-        case 'data_transformed':
-            return 'semantic';
-        case 'database_updated':
-            return 'comparison';
-        case 'generic':
-        default:
-            return 'llm';
+    catch (err) {
+        result.errors = err instanceof Error ? err.message : 'Unknown error';
     }
+    result.duration = Date.now() - startTime;
+    return result;
 }
-/**
- * Prompt for LLM to generate verification code
- */
-const VERIFICATION_CODE_GENERATION_PROMPT = `You are a verification code generator. Given a claim that an AI assistant made, generate code to verify if the claim is TRUE.
+// ============================================================================
+// CLAIM EXTRACTION - LLM extracts claims from responses
+// ============================================================================
+const EXTRACT_CLAIMS_PROMPT = `Extract ALL verifiable claims from this AI assistant response.
 
-CLAIM TO VERIFY:
-Type: {CLAIM_TYPE}
-Description: {CLAIM_DESCRIPTION}
-Evidence: {CLAIM_EVIDENCE}
-Parameters: {CLAIM_PARAMS}
-
-WORKING DIRECTORY: {WORKING_DIR}
-
-Generate a verification test. Choose the most appropriate approach:
+RESPONSE:
+---
+{RESPONSE}
+---
 
-1. SHELL COMMAND - For file operations, git, npm, system checks
-2. JAVASCRIPT - For complex logic, API calls, JSON parsing
-3. API - For HTTP endpoints, external services
+CONTEXT: {CONTEXT}
+WORKING_DIR: {WORKING_DIR}
 
-IMPORTANT RULES:
-- Code must be READ-ONLY and NON-DESTRUCTIVE (no writes, no deletes, no modifications)
-- Code must complete quickly (under 10 seconds)
-- Code must output a clear result that can be parsed
-- For shell: output should be parseable (exit code 0 = verified, non-zero = failed)
-- For JavaScript: must export/return { verified: boolean, evidence: string }
-- Do NOT use interactive commands
-- Do NOT access sensitive data or credentials
+For each claim, determine:
+1. What specific assertion is being made
+2. Category: file_op (created/modified/deleted files), code (compiles/tests pass), command (executed successfully), state (something changed), behavior (feature works), fact (verifiable truth)
+3. How it can be verified (shell command, file check, CLI test, etc.)
+4. Priority: critical (must verify), high (should verify), medium (nice to verify), low (optional)
 
-Respond with JSON:
-{
-  "testType": "shell" | "javascript" | "api",
-  "code": "the verification code",
-  "description": "what this test does",
-  "expectedOutcome": "what success looks like",
-  "safeToRun": true | false,
-  "safetyReason": "why it's safe/unsafe"
-}
+Return JSON array:
+[{
+  "id": "c1",
+  "statement": "the specific claim",
+  "category": "file_op|code|command|state|behavior|fact",
+  "verifiable": true,
+  "priority": "critical|high|medium|low",
+  "context": {"path": "/path/if/relevant", "command": "if relevant"}
+}]
 
-Only output valid JSON, nothing else.`;
+Output ONLY valid JSON array.`;
 /**
- * Generate verification code using LLM
+ * Extract claims from assistant response using LLM
  */
-export async function generateVerificationCode(claim, context) {
-    if (!context.llmVerifier) {
-        return null;
-    }
+async function extractClaims(response, ctx) {
+    if (!ctx.llmVerifier)
+        return [];
     try {
-        const prompt = VERIFICATION_CODE_GENERATION_PROMPT
-            .replace('{CLAIM_TYPE}', claim.type)
-            .replace('{CLAIM_DESCRIPTION}', claim.description)
-            .replace('{CLAIM_EVIDENCE}', claim.evidence)
-            .replace('{CLAIM_PARAMS}', JSON.stringify(claim.params, null, 2))
-            .replace('{WORKING_DIR}', context.workingDirectory);
-        const result = await context.llmVerifier(prompt);
-        // Parse the JSON response
-        const jsonMatch = result.match(/\{[\s\S]*\}/);
-        if (!jsonMatch) {
-            return null;
+        const prompt = EXTRACT_CLAIMS_PROMPT
+            .replace('{RESPONSE}', response.slice(0, 8000))
+            .replace('{CONTEXT}', ctx.conversationHistory?.slice(-3).join('\n') || '')
+            .replace('{WORKING_DIR}', ctx.workingDirectory);
+        const result = await ctx.llmVerifier(prompt);
+        const match = result.match(/\[[\s\S]*\]/);
+        if (match) {
+            return JSON.parse(match[0]);
         }
-        const parsed = JSON.parse(jsonMatch[0]);
-        return {
-            claim,
-            testType: parsed.testType,
-            code: parsed.code,
-            description: parsed.description,
-            expectedOutcome: parsed.expectedOutcome,
-            safetyCheck: parsed.safeToRun
-        };
     }
-    catch (err) {
-        console.error('Failed to generate verification code:', err);
-        return null;
+    catch {
+        // Fall through
     }
+    return [];
 }
+// ============================================================================
+// TEST GENERATION - LLM generates isolated tests for claims
+// ============================================================================
+const GENERATE_TESTS_PROMPT = `Generate isolated runtime tests for these claims.
+
+CLAIMS:
+{CLAIMS}
+
+WORKING_DIR: {WORKING_DIR}
+PLATFORM: {PLATFORM}
+
+For each claim, generate a test that verifies it using:
+- Shell commands (for file checks, git status, etc.)
+- CLI commands (for testing CLI behavior in fresh instance)
+- Expected output patterns
+
+Return JSON array:
+[{
+  "id": "test-1",
+  "description": "what we're testing",
+  "shellCommands": ["ls -la path", "cat file"],
+  "commands": ["/help", "some input"],
+  "expectedOutputs": ["pattern1", "pattern2"],
+  "expectedBehavior": "description for LLM assessment",
+  "requiresBuild": false,
+  "timeout": 30000
+}]
+
+Use READ-ONLY commands only. No destructive operations.
+Output ONLY valid JSON array.`;
 /**
- * Safety patterns to block dangerous code
- */
-const DANGEROUS_PATTERNS = [
-    /\brm\s+-rf?\b/i, // rm commands
-    /\brmdir\b/i, // rmdir
-    /\bdd\s+if=/i, // dd (disk destroyer)
-    /\bmkfs\b/i, // format filesystem
-    /\b>\s*\/dev\//i, // write to devices
-    /\bchmod\s+777\b/i, // dangerous permissions
-    /\bsudo\b/i, // sudo commands
-    /\bcurl.*\|\s*sh\b/i, // pipe to shell
-    /\bwget.*\|\s*sh\b/i, // pipe to shell
-    /\beval\s*\(/i, // eval in JS
-    /new\s+Function\s*\(/i, // Function constructor
-    /child_process/i, // subprocess in JS (unless we control it)
-    /\bexec\s*\(/i, // exec calls
-    /\bspawn\s*\(/i, // spawn calls
-    /writeFile/i, // file writes
-    /appendFile/i, // file appends
-    /unlink\s*\(/i, // file deletion
-    /rmSync/i, // sync deletion
-    /fs\.rm/i, // fs remove
-    /DROP\s+TABLE/i, // SQL injection
-    /DELETE\s+FROM/i, // SQL deletion
-    /TRUNCATE/i, // SQL truncate
-    /;\s*--/, // SQL comment injection
-    /process\.exit/i, // process exit
-    /require\s*\(\s*['"]child/i, // require child_process
-];
-/**
- * Validate that generated code is safe to execute
+ * Generate isolated tests for claims
  */
-export function validateGeneratedCode(test) {
-    // First check the LLM's own safety assessment
-    if (!test.safetyCheck) {
-        return { safe: false, reason: 'LLM marked code as unsafe' };
-    }
-    // Check against dangerous patterns
-    for (const pattern of DANGEROUS_PATTERNS) {
-        if (pattern.test(test.code)) {
-            return {
-                safe: false,
-                reason: `Dangerous pattern detected: ${pattern.source}`
-            };
+async function generateTests(claims, ctx) {
+    if (!ctx.llmVerifier || claims.length === 0)
+        return [];
+    try {
+        const prompt = GENERATE_TESTS_PROMPT
+            .replace('{CLAIMS}', JSON.stringify(claims.slice(0, 10)))
+            .replace('{WORKING_DIR}', ctx.workingDirectory)
+            .replace('{PLATFORM}', process.platform);
+        const result = await ctx.llmVerifier(prompt);
+        const match = result.match(/\[[\s\S]*\]/);
+        if (match) {
+            return JSON.parse(match[0]);
         }
     }
-    // Additional checks for shell commands
-    if (test.testType === 'shell') {
-        // Only allow specific safe commands
-        const safeShellPrefixes = [
-            'ls', 'cat', 'head', 'tail', 'grep', 'find', 'stat', 'file',
-            'test', 'echo', 'pwd', 'wc', 'diff', 'cmp',
-            'git log', 'git status', 'git show', 'git diff', 'git branch',
-            'npm view', 'npm list', 'npm ls',
-            'node -e', 'node --eval',
-            'curl -s', 'curl --silent', 'wget -q',
-            'jq', 'python -c', 'python3 -c',
-            'lsof', 'netstat', 'ss', 'ps',
-            'which', 'type', 'command -v',
-        ];
-        const trimmedCode = test.code.trim().toLowerCase();
-        const startsWithSafe = safeShellPrefixes.some(prefix => trimmedCode.startsWith(prefix.toLowerCase()));
-        if (!startsWithSafe) {
-            // Check if it's a simple test/check command
-            if (!trimmedCode.startsWith('[') && !trimmedCode.startsWith('if ')) {
-                return {
-                    safe: false,
-                    reason: 'Shell command does not start with a known safe prefix'
-                };
-            }
+    catch {
+        // Fall through to basic tests
+    }
+    // Fallback: generate basic tests
+    return claims.filter(c => c.verifiable && (c.priority === 'critical' || c.priority === 'high')).map((c, i) => {
+        const test = {
+            id: `test-${i}`,
+            description: c.statement,
+            commands: [],
+            shellCommands: [],
+            expectedBehavior: c.statement,
+            timeout: 30000
+        };
+        // Add basic verification based on category
+        if (c.category === 'file_op' && c.context['path']) {
+            test.shellCommands = [`test -f "${c.context['path']}" && echo "EXISTS" || echo "NOT_FOUND"`];
+            test.expectedOutputs = ['EXISTS'];
         }
-    }
-    // For JavaScript, ensure it's a simple expression
-    if (test.testType === 'javascript') {
-        // Check code length - very long code is suspicious
-        if (test.code.length > 2000) {
-            return { safe: false, reason: 'JavaScript code too long' };
+        else if (c.category === 'code') {
+            test.shellCommands = ['npm run build 2>&1 | tail -5'];
         }
-    }
-    return { safe: true, reason: 'All safety checks passed' };
-}
-/**
- * Execute a generated verification test
- */
-export async function executeGeneratedTest(test, context) {
-    const baseResult = {
-        claim: test.claim,
-        timestamp: new Date().toISOString()
-    };
-    // Validate safety first
-    const safetyResult = validateGeneratedCode(test);
-    if (!safetyResult.safe) {
-        return {
-            ...baseResult,
-            verified: false,
-            confidence: 'low',
-            evidence: `Generated test blocked: ${safetyResult.reason}`,
-            error: 'Safety validation failed'
-        };
-    }
-    try {
-        switch (test.testType) {
-            case 'shell': {
-                const { stdout, stderr } = await execAsync(test.code, {
-                    cwd: context.workingDirectory,
-                    timeout: 10000, // 10 second timeout
-                    maxBuffer: 1024 * 1024 // 1MB max output
-                });
-                const output = (stdout + stderr).trim();
-                // Shell convention: exit 0 = success
-                return {
-                    ...baseResult,
-                    verified: true,
-                    confidence: 'high',
-                    evidence: `Test passed. Output: ${output.slice(0, 500)}`
-                };
-            }
-            case 'javascript': {
-                // Execute JavaScript in a sandboxed way using node -e
-                const wrappedCode = `
-          const result = (async () => {
-            ${test.code}
-          })();
-          result.then(r => console.log(JSON.stringify(r))).catch(e => {
-            console.log(JSON.stringify({ verified: false, evidence: e.message }));
-          });
-        `;
-                const { stdout } = await execAsync(`node -e ${JSON.stringify(wrappedCode)}`, {
-                    cwd: context.workingDirectory,
-                    timeout: 10000
-                });
-                try {
-                    const result = JSON.parse(stdout.trim());
-                    return {
-                        ...baseResult,
-                        verified: result.verified,
-                        confidence: 'high',
-                        evidence: result.evidence
-                    };
-                }
-                catch {
-                    return {
-                        ...baseResult,
-                        verified: false,
-                        confidence: 'medium',
-                        evidence: `JavaScript output: ${stdout.slice(0, 500)}`
-                    };
-                }
-            }
-            case 'api': {
-                // For API tests, use curl
-                const { stdout } = await execAsync(test.code, {
-                    cwd: context.workingDirectory,
-                    timeout: 15000
-                });
-                // Try to parse as JSON result
-                try {
-                    const result = JSON.parse(stdout.trim());
-                    return {
-                        ...baseResult,
-                        verified: Boolean(result.verified ?? result.success ?? result.ok),
-                        confidence: 'high',
-                        evidence: `API response: ${JSON.stringify(result).slice(0, 500)}`
-                    };
-                }
-                catch {
-                    // Non-JSON response - check for success indicators
-                    const isSuccess = stdout.includes('200') || stdout.includes('success') || stdout.includes('ok');
-                    return {
-                        ...baseResult,
-                        verified: isSuccess,
-                        confidence: 'medium',
-                        evidence: `API output: ${stdout.slice(0, 500)}`
-                    };
-                }
-            }
-            default:
-                return {
-                    ...baseResult,
-                    verified: false,
-                    confidence: 'low',
-                    evidence: `Unknown test type: ${test.testType}`
-                };
+        else if (c.category === 'behavior') {
+            test.commands = ['/help'];
         }
-    }
-    catch (err) {
-        // Command failed (non-zero exit) = verification failed
-        return {
-            ...baseResult,
-            verified: false,
-            confidence: 'high',
-            evidence: `Test failed: ${err instanceof Error ? err.message : 'Unknown error'}`,
-            error: err instanceof Error ? err.message : 'Unknown error'
-        };
-    }
+        return test;
+    });
 }
+// ============================================================================
+// MAIN VERIFICATION API
+// ============================================================================
 /**
- * Verify a claim using LLM-generated runtime test
+ * Verify an assistant response using isolated runtime tests.
+ * This is the main entry point for verification.
  */
-export async function verifyWithGeneratedTest(claim, context) {
-    const baseResult = {
-        claim,
-        timestamp: new Date().toISOString()
-    };
-    // Generate verification code
-    const test = await generateVerificationCode(claim, context);
-    if (!test) {
+export async function verifyResponse(response, ctx, responseId) {
+    const timestamp = new Date().toISOString();
+    const id = responseId || `verify-${Date.now()}`;
+    // Extract claims from response
+    const claims = await extractClaims(response, ctx);
+    if (claims.length === 0) {
         return {
-            ...baseResult,
-            verified: false,
-            confidence: 'low',
-            evidence: 'Failed to generate verification test'
+            responseId: id,
+            timestamp,
+            claims: [],
+            results: [],
+            summary: { total: 0, verified: 0, failed: 0, inconclusive: 0 },
+            overallVerdict: 'unverified',
+            trustScore: 50
         };
     }
-    // Execute the generated test
-    return executeGeneratedTest(test, context);
-}
-/**
- * Full verification using LLM-generated tests
- * This is the most powerful verification method - LLM decides HOW to verify each claim
- */
-export async function verifyResponseWithGeneratedTests(response, context, responseId) {
-    // Extract claims using LLM
-    const claims = context.llmVerifier
-        ? await extractClaimsWithLLM(response, context.llmVerifier)
-        : extractClaims(response);
-    const results = [];
-    for (const claim of claims) {
-        // For each claim, generate and run a custom verification test
-        const result = await verifyWithGeneratedTest(claim, context);
-        results.push(result);
-    }
-    const verified = results.filter(r => r.verified).length;
-    const failed = results.filter(r => !r.verified && r.confidence === 'high').length;
-    const inconclusive = results.filter(r => !r.verified && r.confidence !== 'high').length;
-    let overallVerdict;
-    if (failed > 0) {
-        overallVerdict = 'contradicted';
-    }
-    else if (verified === claims.length && claims.length > 0) {
-        overallVerdict = 'verified';
-    }
-    else if (verified > 0) {
-        overallVerdict = 'partially_verified';
-    }
-    else {
-        overallVerdict = 'unverified';
-    }
-    return {
-        responseId: responseId || `response-${Date.now()}`,
-        timestamp: new Date().toISOString(),
-        claims,
-        results,
-        summary: {
-            total: claims.length,
-            verified,
-            failed,
-            inconclusive
-        },
-        overallVerdict
-    };
-}
-/**
- * Hybrid verification - uses generated tests when available, falls back to predefined tests
- */
-export async function verifyResponseHybrid(response, context, responseId) {
-    const claims = context.llmVerifier
-        ? await extractClaimsWithLLM(response, context.llmVerifier)
-        : extractClaims(response);
-    const results = [];
-    for (const claim of claims) {
-        let result;
-        // Try LLM-generated test first if LLM is available
-        if (context.llmVerifier) {
-            const generatedTest = await generateVerificationCode(claim, context);
-            if (generatedTest) {
-                const safety = validateGeneratedCode(generatedTest);
-                if (safety.safe) {
-                    // Use generated test
-                    result = await executeGeneratedTest(generatedTest, context);
-                    results.push(result);
-                    continue;
-                }
-            }
-        }
-        // Fall back to predefined verification
-        const standardTypes = [
-            'file_created', 'file_modified', 'file_deleted', 'code_compiles',
-            'tests_pass', 'git_committed', 'package_published', 'command_executed',
-            'dependency_installed', 'service_running', 'url_accessible', 'content_contains'
-        ];
-        let test;
-        if (standardTypes.includes(claim.type)) {
-            test = generateVerificationTest(claim);
-        }
-        else {
-            test = generateExtendedVerificationTest(claim, context);
-        }
-        try {
-            result = await test();
-        }
-        catch (err) {
-            result = {
+    // Generate isolated tests for claims
+    const tests = await generateTests(claims, ctx);
+    // Run all isolated tests
+    const testResults = [];
+    for (const test of tests) {
+        const result = await runIsolatedTest(test, ctx.workingDirectory, ctx.llmVerifier);
+        testResults.push(result);
+    }
+    // Map test results back to claims
+    const results = claims.map((claim, i) => {
+        const testResult = testResults[i];
+        if (!testResult) {
+            return {
                 claim,
                 verified: false,
                 confidence: 'low',
-                evidence: 'Verification failed',
-                error: err instanceof Error ? err.message : 'Unknown error',
-                timestamp: new Date().toISOString()
+                evidence: 'No test generated',
+                method: 'skip',
+                timestamp
             };
         }
-        results.push(result);
-    }
+        return {
+            claim,
+            verified: testResult.success,
+            confidence: testResult.success ? 'high' : (testResult.matchedPatterns.length > 0 ? 'medium' : 'low'),
+            evidence: testResult.success
+                ? `Verified in isolated runtime: ${testResult.matchedPatterns.join(', ')}`
+                : `Failed: ${testResult.unmatchedPatterns.join(', ')}`,
+            method: 'isolated-runtime',
+            reasoning: testResult.llmAssessment,
+            executedCode: [...(testResult.test.shellCommands || []), ...(testResult.test.commands || [])].join('\n'),
+            rawOutput: testResult.output.slice(0, 2000),
+            error: testResult.errors || undefined,
+            timestamp
+        };
+    });
+    // Calculate summary
     const verified = results.filter(r => r.verified).length;
     const failed = results.filter(r => !r.verified && r.confidence === 'high').length;
     const inconclusive = results.filter(r => !r.verified && r.confidence !== 'high').length;
+    // Determine verdict
     let overallVerdict;
     if (failed > 0) {
         overallVerdict = 'contradicted';
@@ -1612,165 +406,69 @@ export async function verifyResponseHybrid(response, context, responseId) {
     else {
         overallVerdict = 'unverified';
     }
+    // Calculate trust score
+    const trustScore = claims.length > 0
+        ? Math.round((verified / claims.length) * 100)
+        : 50;
     return {
-        responseId: responseId || `response-${Date.now()}`,
-        timestamp: new Date().toISOString(),
+        responseId: id,
+        timestamp,
         claims,
         results,
-        summary: {
-            total: claims.length,
-            verified,
-            failed,
-            inconclusive
-        },
-        overallVerdict
+        summary: { total: claims.length, verified, failed, inconclusive },
+        overallVerdict,
+        trustScore
     };
 }
-const UNIVERSAL_EXTRACT = `Extract ALL verifiable claims from this AI response. Include explicit claims, implicit claims, state changes, results, assertions.
+/**
+ * Format verification report for display
+ */
+export function formatVerificationReport(report) {
+    const bar = '█'.repeat(Math.round(report.trustScore / 10)) + '░'.repeat(10 - Math.round(report.trustScore / 10));
+    const icon = report.trustScore >= 80 ? '✅' : report.trustScore >= 50 ? '⚠️' : '❌';
+    let out = `╔════════════════════════════════════════════════════════════╗
+║         ISOLATED RUNTIME VERIFICATION REPORT               ║
+╚════════════════════════════════════════════════════════════╝
 
-RESPONSE:
----
-{RESPONSE}
----
-CONTEXT: {CONTEXT}
-DIR: {WORKING_DIR}
+`;
+    out += `Trust: ${icon} ${report.trustScore}/100 [${bar}]
+Verdict: ${report.overallVerdict.toUpperCase()}
 
-Return JSON array: [{"id":"c1","statement":"claim","category":"file_op|code|state|data|behavior|fact|other","verifiable":true/false,"verificationApproach":"how","priority":"critical|high|medium|low","context":{}}]
-Output ONLY valid JSON.`;
-const UNIVERSAL_GEN = `Generate verification code for: {STATEMENT}
-Category: {CATEGORY} | Approach: {APPROACH} | Context: {CONTEXT} | Dir: {WORKING_DIR} | Platform: {PLATFORM}
+Claims: ${report.summary.total} | ✅ ${report.summary.verified} | ❌ ${report.summary.failed} | ❓ ${report.summary.inconclusive}
 
-Use shell/javascript/python. READ-ONLY only.
-Return JSON: {"steps":[{"type":"shell|javascript|python","code":"code","desc":"what"}],"success":"success criteria","failure":"failure criteria","confPass":0-100,"confFail":0-100,"safe":{"ok":true/false,"why":"reason"}}
-Output ONLY valid JSON.`;
-const UNIVERSAL_ASSESS = `Assess: RESPONSE:{RESPONSE} CLAIMS:{CLAIMS} RESULTS:{RESULTS}
-Return JSON: {"trust":0-100,"summary":"text","concerns":[]}
-Output ONLY valid JSON.`;
-const UNSAFE = [/\brm\s/i, /rmdir/i, /sudo/i, /chmod\s*7/i, /eval\s*\(/i, /exec\s*\(/i, /child_process/i, /os\.system/i, /subprocess/i, /curl.*\|.*sh/i, /DROP\s+TABLE/i, /DELETE\s+FROM/i, /kill/i];
-export function validateUniversalCode(c) {
-    for (const p of UNSAFE)
-        if (p.test(c))
-            return { safe: false, reason: p.source };
-    return c.length > 5000 ? { safe: false, reason: 'too long' } : { safe: true, reason: 'ok' };
-}
-async function runUniversalStep(s, cwd) {
-    const v = validateUniversalCode(s.code);
-    if (!v.safe)
-        return { ok: false, out: v.reason };
-    try {
-        if (s.type === 'shell') {
-            const { stdout, stderr } = await execAsync(s.code, { cwd, timeout: 30000, maxBuffer: 5 * 1024 * 1024 });
-            return { ok: true, out: stdout + stderr };
-        }
-        if (s.type === 'javascript') {
-            const w = `(async()=>{try{const fs=require('fs').promises;const r=await(async()=>{${s.code}})();console.log(JSON.stringify({ok:1,r}))}catch(e){console.log(JSON.stringify({ok:0,e:e.message}))}})()`;
-            const { stdout } = await execAsync(`node -e ${JSON.stringify(w)}`, { cwd, timeout: 30000 });
-            return { ok: true, out: stdout };
-        }
-        if (s.type === 'python') {
-            const { stdout, stderr } = await execAsync(`python3 -c ${JSON.stringify(s.code)}`, { cwd, timeout: 30000 });
-            return { ok: true, out: stdout + stderr };
-        }
-        return { ok: false, out: 'unknown type' };
-    }
-    catch (e) {
-        return { ok: false, out: e instanceof Error ? e.message : 'err' };
-    }
-}
-export async function extractUniversalClaims(r, ctx) {
-    if (!ctx.llmVerifier)
-        return extractClaims(r).map((c, i) => ({ id: `c${i}`, statement: c.description, category: c.type, verifiable: true, verificationApproach: 'runtime', priority: 'medium', context: c.params }));
-    try {
-        const p = UNIVERSAL_EXTRACT.replace('{RESPONSE}', r.slice(0, 8000)).replace('{CONTEXT}', ctx.conversationHistory?.slice(-3).join('\n') || '').replace('{WORKING_DIR}', ctx.workingDirectory);
-        const res = await ctx.llmVerifier(p);
-        const m = res.match(/\[[\s\S]*\]/);
-        if (m)
-            return JSON.parse(m[0]);
-    }
-    catch { /* fall through */ }
-    return extractClaims(r).map((c, i) => ({ id: `c${i}`, statement: c.description, category: c.type, verifiable: true, verificationApproach: 'runtime', priority: 'medium', context: c.params }));
-}
-export async function verifyUniversalClaim(claim, ctx) {
-    const base = { claim, timestamp: new Date().toISOString() };
-    if (!claim.verifiable)
-        return { ...base, verified: false, confidence: 0, method: 'skip', evidence: 'Not verifiable', reasoning: 'Cannot verify' };
-    if (!ctx.llmVerifier)
-        return { ...base, verified: false, confidence: 0, method: 'skip', evidence: 'No LLM', reasoning: 'Needs LLM' };
-    try {
-        const p = UNIVERSAL_GEN.replace('{STATEMENT}', claim.statement).replace('{CATEGORY}', claim.category).replace('{APPROACH}', claim.verificationApproach).replace('{CONTEXT}', JSON.stringify(claim.context)).replace('{WORKING_DIR}', ctx.workingDirectory).replace('{PLATFORM}', process.platform);
-        const res = await ctx.llmVerifier(p);
-        const m = res.match(/\{[\s\S]*\}/);
-        if (!m)
-            throw new Error('bad');
-        const plan = JSON.parse(m[0]);
-        if (!plan.safe.ok)
-            return { ...base, verified: false, confidence: 0, method: 'blocked', evidence: plan.safe.why, reasoning: 'Unsafe' };
-        let allOk = true, out = '', code = '';
-        for (const s of plan.steps) {
-            code += s.code + '\n';
-            const r = await runUniversalStep(s, ctx.workingDirectory);
-            out += r.out + '\n';
-            if (!r.ok)
-                allOk = false;
+`;
+    out += `🔬 ISOLATED RUNTIME TESTS:\n`;
+    for (const r of report.results.slice(0, 8)) {
+        const statusIcon = r.verified ? '✅' : r.confidence === 'high' ? '❌' : '❓';
+        out += `  ${statusIcon} [${r.confidence}] ${r.claim.statement.slice(0, 50)}...\n`;
+        if (r.reasoning) {
+            out += `     └─ ${r.reasoning.slice(0, 60)}\n`;
         }
-        return { ...base, verified: allOk, confidence: allOk ? plan.confPass : plan.confFail, method: plan.steps.map(s => s.type).join('+'), evidence: allOk ? plan.success : plan.failure, reasoning: allOk ? 'All passed' : 'Some failed', executedCode: code, rawOutput: out.slice(0, 2000) };
     }
-    catch (e) {
-        return { ...base, verified: false, confidence: 10, method: 'error', evidence: 'Failed', reasoning: e instanceof Error ? e.message : 'err' };
+    if (report.results.length > 8) {
+        out += `  ... +${report.results.length - 8} more\n`;
     }
+    return out;
 }
-export async function verifyResponseUniversal(response, ctx, id) {
-    const claims = await extractUniversalClaims(response, ctx);
-    const results = [];
-    for (const c of claims)
-        results.push(c.verifiable || c.priority === 'critical' || c.priority === 'high' ? await verifyUniversalClaim(c, ctx) : { claim: c, verified: false, confidence: 0, method: 'skip', evidence: 'Low priority', reasoning: 'Skipped', timestamp: new Date().toISOString() });
-    const vClaims = claims.filter(c => c.verifiable).length;
-    const verified = results.filter(r => r.verified).length;
-    const failed = results.filter(r => !r.verified && r.confidence > 50).length;
-    const inconclusive = results.filter(r => !r.verified && r.confidence <= 50 && r.method !== 'skip').length;
-    const avgConf = results.length ? results.reduce((s, r) => s + r.confidence, 0) / results.length : 0;
-    let assessment = '', trust = 0;
-    if (ctx.llmVerifier)
-        try {
-            const p = UNIVERSAL_ASSESS.replace('{RESPONSE}', response.slice(0, 4000)).replace('{CLAIMS}', JSON.stringify(claims.slice(0, 15))).replace('{RESULTS}', JSON.stringify(results.slice(0, 15)));
-            const r = await ctx.llmVerifier(p);
-            const m = r.match(/\{[\s\S]*\}/);
-            if (m) {
-                const a = JSON.parse(m[0]);
-                trust = a.trust;
-                assessment = a.summary + (a.concerns?.length ? ` Concerns: ${a.concerns.join('; ')}` : '');
-            }
-        }
-        catch {
-            trust = Math.round(avgConf * verified / Math.max(vClaims, 1));
-            assessment = `${verified}/${vClaims} verified`;
-        }
-    else {
-        trust = Math.round(avgConf * verified / Math.max(vClaims, 1));
-        assessment = `${verified}/${vClaims} verified`;
+/**
+ * Quick verification - verify only critical/high priority claims
+ */
+export async function quickVerify(response, ctx) {
+    const claims = await extractClaims(response, ctx);
+    const critical = claims.filter(c => c.verifiable && (c.priority === 'critical' || c.priority === 'high')).slice(0, 3);
+    if (critical.length === 0) {
+        return { trustScore: 50, summary: 'No critical claims to verify' };
+    }
+    const tests = await generateTests(critical, ctx);
+    let verified = 0;
+    for (const test of tests) {
+        const result = await runIsolatedTest(test, ctx.workingDirectory, ctx.llmVerifier);
+        if (result.success)
+            verified++;
     }
-    return { responseId: id || `u-${Date.now()}`, originalResponse: response, timestamp: new Date().toISOString(), claims, results, summary: { totalClaims: claims.length, verifiableClaims: vClaims, verified, failed, inconclusive, averageConfidence: Math.round(avgConf) }, overallAssessment: assessment, trustScore: trust };
-}
-export async function quickUniversalVerify(r, ctx) {
-    const claims = await extractUniversalClaims(r, ctx);
-    const crit = claims.filter(c => c.verifiable && (c.priority === 'critical' || c.priority === 'high')).slice(0, 5);
-    if (!crit.length)
-        return { trustScore: 50, summary: 'No critical claims' };
-    let v = 0;
-    for (const c of crit)
-        if ((await verifyUniversalClaim(c, ctx)).verified)
-            v++;
-    return { trustScore: Math.round(v / crit.length * 100), summary: `${v}/${crit.length} critical verified` };
-}
-export function formatUniversalReport(r) {
-    const bar = '█'.repeat(Math.round(r.trustScore / 10)) + '░'.repeat(10 - Math.round(r.trustScore / 10));
-    const icon = r.trustScore >= 80 ? '✅' : r.trustScore >= 50 ? '⚠️' : '❌';
-    let out = `╔════════════════════════════════════════════════════════════╗\n║           UNIVERSAL VERIFICATION REPORT                    ║\n╚════════════════════════════════════════════════════════════╝\n\n`;
-    out += `Trust: ${icon} ${r.trustScore}/100 [${bar}]\n${r.overallAssessment}\n\nClaims: ${r.summary.totalClaims} | ✅ ${r.summary.verified} | ❌ ${r.summary.failed} | ❓ ${r.summary.inconclusive}\n\n`;
-    for (const x of r.results.slice(0, 8))
-        out += `${x.verified ? '✅' : x.confidence > 50 ? '❌' : '❓'} [${x.confidence}%] ${x.claim.statement.slice(0, 55)}...\n`;
-    if (r.results.length > 8)
-        out += `... +${r.results.length - 8} more\n`;
-    return out;
+    return {
+        trustScore: Math.round((verified / critical.length) * 100),
+        summary: `${verified}/${critical.length} critical claims verified`
+    };
 }
 //# sourceMappingURL=responseVerifier.js.map