@veraxhq/verax 0.2.0 → 0.3.0

package/src/cli/commands/run.js

@@ -1,8 +1,8 @@
-import { resolve, join } from 'path';
+import { resolve } from 'path';
 import { existsSync, readFileSync } from 'fs';
 import { fileURLToPath } from 'url';
 import { dirname } from 'path';
-import { UsageError, DataError, CrashError } from '../util/errors.js';
+import { UsageError, DataError } from '../util/errors.js';
 import { generateRunId } from '../util/run-id.js';
 import { getRunPaths, ensureRunDirectories } from '../util/paths.js';
 import { atomicWriteJson, atomicWriteText } from '../util/atomic-write.js';
@@ -16,6 +16,10 @@ import { writeObserveJson } from '../util/observe-writer.js';
 import { detectFindings } from '../util/detection-engine.js';
 import { writeFindingsJson } from '../util/findings-writer.js';
 import { writeSummaryJson } from '../util/summary-writer.js';
+import { computeRuntimeBudget, withTimeout } from '../util/runtime-budget.js';
+import { assertHasLocalSource } from '../util/source-requirement.js';
+import { runWithDeterminism } from '../util/determinism-runner.js';
+import { runDeterminismCheck } from '../../verax/core/determinism/engine.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -26,7 +30,7 @@ function getVersion() {
     const pkg = JSON.parse(readFileSync(pkgPath, 'utf8'));
     return pkg.version;
   } catch {
-    return '0.2.0';
+    return '0.3.0';
   }
 }
 
@@ -35,17 +39,68 @@ function getVersion() {
  * Strict, non-interactive CLI mode with explicit flags
  */
 export async function runCommand(options) {
+  return await runCommandInternal(options);
+}
+
+/**
+ * Internal run command implementation
+ */
+async function runCommandInternal(options) {
   const {
     url,
+    fixture,
     src = '.',
     out = '.verax',
     json = false,
     verbose = false,
+    determinism = false,
+    determinismRuns = 2,
   } = options;
   
+  // PHASE 25: Support fixture mode for determinism
+  let resolvedUrl = url;
+  let fixtureId = null;
+  
+  if (fixture && !url) {
+    // Extract URL from fixture
+    const { resolve } = await import('path');
+    const { existsSync, readFileSync } = await import('fs');
+    const { fileURLToPath } = await import('url');
+    const { dirname } = await import('path');
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = dirname(__filename);
+    
+    const fixturePath = resolve(__dirname, '..', '..', '..', 'test', 'fixtures', 'realistic', fixture);
+    if (existsSync(fixturePath)) {
+      // Try to read package.json or index.html to extract URL
+      const packagePath = resolve(fixturePath, 'package.json');
+      const indexPath = resolve(fixturePath, 'index.html');
+      
+      if (existsSync(packagePath)) {
+        try {
+          const pkg = JSON.parse(readFileSync(packagePath, 'utf-8'));
+          if (pkg.verax && pkg.verax.url) {
+            resolvedUrl = pkg.verax.url;
+            fixtureId = fixture;
+          }
+        } catch {
+          // Ignore parse errors
+        }
+      }
+      
+      // If no URL found, use default localhost URL for fixture
+      if (!resolvedUrl) {
+        resolvedUrl = `http://localhost:5173`; // Default Vite dev server
+        fixtureId = fixture;
+      }
+    } else {
+      throw new DataError(`Fixture not found: ${fixture}`);
+    }
+  }
+  
   // Validate required arguments
-  if (!url) {
-    throw new UsageError('Missing required argument: --url <url>');
+  if (!resolvedUrl) {
+    throw new UsageError('Missing required argument: --url <url> or --fixture <fixture>');
   }
   
   const projectRoot = resolve(process.cwd());
@@ -55,12 +110,16 @@ export async function runCommand(options) {
   if (!existsSync(srcPath)) {
     throw new DataError(`Source directory not found: ${srcPath}`);
   }
+
+  // Enforce local source availability (no URL-only scans)
+  assertHasLocalSource(srcPath);
   
   // Create event emitter
   const events = new RunEventEmitter();
   
   // Setup event handlers
   if (json) {
+    // In JSON mode, emit events as JSONL (one JSON object per line)
     events.on('*', (event) => {
       console.log(JSON.stringify(event));
     });
@@ -75,6 +134,141 @@ export async function runCommand(options) {
   let runId = null;
   let paths = null;
   let startedAt = null;
+  let watchdogTimer = null;
+  let budget = null;
+  let timedOut = false;
+  
+  // Graceful finalization function
+  const finalizeOnTimeout = async (reason) => {
+    if (timedOut) return; // Prevent double finalization
+    timedOut = true;
+    
+    events.stopHeartbeat();
+    
+    if (paths && runId && startedAt) {
+      try {
+        const failedAt = new Date().toISOString();
+        atomicWriteJson(paths.runStatusJson, {
+          contractVersion: 1,
+          artifactVersions: paths.artifactVersions,
+          status: 'FAILED',
+          runId,
+          startedAt,
+          failedAt,
+          error: reason,
+        });
+        
+        atomicWriteJson(paths.runMetaJson, {
+          contractVersion: 1,
+          artifactVersions: paths.artifactVersions,
+          veraxVersion: getVersion(),
+          nodeVersion: process.version,
+          platform: process.platform,
+          cwd: projectRoot,
+          command: 'run',
+          args: { url, src, out },
+          url,
+          src: srcPath,
+          startedAt,
+          completedAt: failedAt,
+          error: reason,
+        });
+        
+        try {
+          writeSummaryJson(paths.summaryJson, {
+            runId,
+            status: 'FAILED',
+            startedAt,
+            completedAt: failedAt,
+            command: 'run',
+            url: resolvedUrl,
+            notes: `Run timed out: ${reason}`,
+          }, {
+            expectationsTotal: 0,
+            attempted: 0,
+            observed: 0,
+            silentFailures: 0,
+            coverageGaps: 0,
+            unproven: 0,
+            informational: 0,
+          });
+        } catch (summaryError) {
+          // Ignore summary write errors during timeout handling
+        }
+      } catch (statusError) {
+        // Ignore errors when writing failure status
+      }
+    }
+    
+    events.emit('error', {
+      message: reason,
+      type: 'timeout',
+    });
+  };
+  
+  // PHASE 25: If determinism mode, wrap execution
+  if (determinism) {
+    const scanFn = async (runConfig) => {
+      // Execute a single scan run
+      const singleRunId = generateRunId();
+      const singlePaths = getRunPaths(projectRoot, out, singleRunId);
+      ensureRunDirectories(singlePaths);
+      
+      // Execute scan (reuse existing logic but with single run)
+      // This is a simplified version - in production, you'd extract the scan logic
+      const { scan } = await import('../../verax/index.js');
+      const scanResult = await scan(
+        projectRoot,
+        resolvedUrl,
+        null, // manifestPath
+        null, // scanBudgetOverride
+        {}, // safetyFlags
+        singleRunId
+      );
+      
+      return {
+        runId: singleRunId,
+        artifactPaths: {
+          findings: singlePaths.findingsJson,
+          runStatus: singlePaths.runStatusJson,
+          summary: singlePaths.summaryJson,
+          learn: singlePaths.learnJson,
+          observe: singlePaths.observeJson,
+          runDir: singlePaths.baseDir
+        }
+      };
+    };
+    
+    const determinismResult = await runWithDeterminism(scanFn, {
+      runs: determinismRuns,
+      out,
+      url: resolvedUrl,
+      fixture: fixtureId,
+      src,
+      verbose,
+      json
+    });
+    
+    // PHASE 25: Output determinism report path
+    if (!json) {
+      console.log(`\nDeterminism check complete.`);
+      console.log(`Verdict: ${determinismResult.verdict}`);
+      console.log(`Report: ${determinismResult.reportPath}`);
+    } else {
+      console.log(JSON.stringify({
+        type: 'determinism:complete',
+        verdict: determinismResult.verdict,
+        reportPath: determinismResult.reportPath,
+        summary: determinismResult.summary
+      }));
+    }
+    
+    return {
+      success: determinismResult.verdict === 'DETERMINISTIC' || determinismResult.verdict === 'NON_DETERMINISTIC_EXPECTED',
+      verdict: determinismResult.verdict,
+      reportPath: determinismResult.reportPath
+    };
+  }
   
   try {
     // Generate run ID
@@ -128,6 +322,8 @@ export async function runCommand(options) {
     startedAt = now.toISOString();
     
     atomicWriteJson(paths.runStatusJson, {
+      contractVersion: 1,
+      artifactVersions: paths.artifactVersions,
       status: 'RUNNING',
       runId,
       startedAt,
@@ -135,27 +331,69 @@ export async function runCommand(options) {
     
     // Write metadata
     atomicWriteJson(paths.runMetaJson, {
+      contractVersion: 1,
+      artifactVersions: paths.artifactVersions,
       veraxVersion: getVersion(),
       nodeVersion: process.version,
       platform: process.platform,
       cwd: projectRoot,
       command: 'run',
-      args: { url, src, out },
-      url,
+      args: { url: resolvedUrl, fixture: fixtureId, src, out },
+      url: resolvedUrl,
+      fixtureId: fixtureId,
       src: srcPath,
       startedAt,
       completedAt: null,
       error: null,
     });
     
-    // Simulate learning phase (placeholder)
+    // Extract expectations first to compute budget
     events.emit('phase:started', {
       phase: 'Learn',
       message: 'Analyzing project structure...',
     });
     
-    // Extract expectations
-    const { expectations, skipped } = await extractExpectations(projectProfile, projectProfile.sourceRoot);
+    events.startHeartbeat('Learn', json);
+    
+    let expectations, skipped;
+    try {
+      // Extract expectations (quick operation, no timeout needed here)
+      const result = await extractExpectations(projectProfile, projectProfile.sourceRoot);
+      expectations = result.expectations;
+      skipped = result.skipped;
+    } finally {
+      events.stopHeartbeat();
+    }
+    
+    // Compute runtime budget based on expectations count
+    budget = computeRuntimeBudget({
+      expectationsCount: expectations.length,
+      mode: 'run',
+      framework: projectProfile.framework,
+      fileCount: projectProfile.fileCount || expectations.length,
+    });
+    
+    // Set up global watchdog timer
+    watchdogTimer = setTimeout(async () => {
+      await finalizeOnTimeout(`Global timeout exceeded: ${budget.totalMaxMs}ms`);
+      // Exit with code 0 (tool executed, just timed out)
+      process.exit(0);
+    }, budget.totalMaxMs);
+    
+    // Wrap Learn phase with timeout
+    try {
+      await withTimeout(
+        budget.learnMaxMs,
+        Promise.resolve(), // Learn phase already completed
+        'Learn'
+      );
+    } catch (error) {
+      if (error.message.includes('timeout')) {
+        await finalizeOnTimeout(`Learn phase timeout: ${budget.learnMaxMs}ms`);
+        process.exit(0);
+      }
+      throw error;
+    }
     
     // For now, emit a placeholder trace event
     events.emit('phase:completed', {
@@ -163,39 +401,60 @@ export async function runCommand(options) {
       message: 'Project analysis complete',
     });
     
-    // Observe phase
+    // Observe phase with timeout
     events.emit('phase:started', {
       phase: 'Observe',
       message: 'Launching browser and observing expectations...',
     });
     
+    events.startHeartbeat('Observe', json);
+    
     let observeData = null;
-    if (expectations.length > 0) {
-      try {
-        observeData = await observeExpectations(
-          expectations,
-          url,
-          paths.evidenceDir,
-          (progress) => {
-            events.emit(progress.event, progress);
+    try {
+      if (expectations.length > 0) {
+        try {
+          observeData = await withTimeout(
+            budget.observeMaxMs,
+            observeExpectations(
+              expectations,
+              url,
+              paths.evidenceDir,
+              (progress) => {
+                events.emit(progress.event, progress);
+              }
+            ),
+            'Observe'
+          );
+        } catch (error) {
+          if (error.message.includes('timeout')) {
+            events.emit('observe:error', {
+              message: `Observe phase timeout: ${budget.observeMaxMs}ms`,
+            });
+            observeData = {
+              observations: [],
+              stats: { attempted: 0, observed: 0, notObserved: 0 },
+              observedAt: new Date().toISOString(),
+            };
+          } else {
+            events.emit('observe:error', {
+              message: error.message,
+            });
+            observeData = {
+              observations: [],
+              stats: { attempted: 0, observed: 0, notObserved: 0 },
+              observedAt: new Date().toISOString(),
+            };
           }
-        );
-      } catch (error) {
-        events.emit('observe:error', {
-          message: error.message,
-        });
+        }
+      } else {
         observeData = {
           observations: [],
           stats: { attempted: 0, observed: 0, notObserved: 0 },
           observedAt: new Date().toISOString(),
         };
       }
-    } else {
-      observeData = {
-        observations: [],
-        stats: { attempted: 0, observed: 0, notObserved: 0 },
-        observedAt: new Date().toISOString(),
-      };
+    } finally {
+      events.stopHeartbeat();
     }
     
     events.emit('phase:completed', {
@@ -203,32 +462,53 @@ export async function runCommand(options) {
       message: 'Browser observation complete',
     });
     
-      // Detect phase
+      // Detect phase with timeout
       events.emit('phase:started', {
         phase: 'Detect',
         message: 'Analyzing findings and detecting silent failures...',
       });
     
+      events.startHeartbeat('Detect', json);
+    
       let detectData = null;
       try {
-        // Use already-extracted expectations
-        const learnData = {
-          expectations,
-          skipped,
-        };
-      
-        detectData = await detectFindings(learnData, observeData, projectRoot, (progress) => {
-          events.emit(progress.event, progress);
-        });
-      } catch (error) {
-        events.emit('detect:error', {
-          message: error.message,
-        });
-        detectData = {
-          findings: [],
-          stats: { total: 0, silentFailures: 0, observed: 0, coverageGaps: 0, unproven: 0, informational: 0 },
-          detectedAt: new Date().toISOString(),
-        };
+        try {
+          // Use already-extracted expectations
+          const learnData = {
+            expectations,
+            skipped,
+          };
+        
+          detectData = await withTimeout(
+            budget.detectMaxMs,
+            detectFindings(learnData, observeData, projectRoot, (progress) => {
+              events.emit(progress.event, progress);
+            }),
+            'Detect'
+          );
+        } catch (error) {
+          if (error.message.includes('timeout')) {
+            events.emit('detect:error', {
+              message: `Detect phase timeout: ${budget.detectMaxMs}ms`,
+            });
+            detectData = {
+              findings: [],
+              stats: { total: 0, silentFailures: 0, observed: 0, coverageGaps: 0, unproven: 0, informational: 0 },
+              detectedAt: new Date().toISOString(),
+            };
+          } else {
+            events.emit('detect:error', {
+              message: error.message,
+            });
+            detectData = {
+              findings: [],
+              stats: { total: 0, silentFailures: 0, observed: 0, coverageGaps: 0, unproven: 0, informational: 0 },
+              detectedAt: new Date().toISOString(),
+            };
+          }
+        }
+      } finally {
+        events.stopHeartbeat();
       }
     
       events.emit('phase:completed', {
@@ -236,37 +516,39 @@ export async function runCommand(options) {
         message: 'Silent failure detection complete',
       });
     
+    // Clear watchdog timer on successful completion
+    if (watchdogTimer) {
+      clearTimeout(watchdogTimer);
+      watchdogTimer = null;
+    }
+    
     // Emit finalize phase
     events.emit('phase:started', {
       phase: 'Finalize Artifacts',
       message: 'Writing run results...',
     });
     
-    const completedAt = new Date().toISOString();
-    
-    // Write completed status
-    atomicWriteJson(paths.runStatusJson, {
-      status: 'COMPLETE',
-      runId,
-      startedAt,
-      completedAt,
-    });
+    events.stopHeartbeat();
     
-    // Update metadata with completion time
-    atomicWriteJson(paths.runMetaJson, {
-      veraxVersion: getVersion(),
-      nodeVersion: process.version,
-      platform: process.platform,
-      cwd: projectRoot,
-      command: 'run',
-      args: { url, src, out },
-      url,
-      src: srcPath,
-      startedAt,
-      completedAt,
-      error: null,
-    });
+    const completedAt = new Date().toISOString();
     
+    const runDurationMs = completedAt && startedAt ? (Date.parse(completedAt) - Date.parse(startedAt)) : 0;
+    const metrics = {
+      learnMs: observeData?.timings?.learnMs || 0,
+      observeMs: observeData?.timings?.observeMs || observeData?.timings?.totalMs || 0,
+      detectMs: detectData?.timings?.detectMs || detectData?.timings?.totalMs || 0,
+      totalMs: runDurationMs > 0 ? runDurationMs : (budget?.ms || 0)
+    };
+    const findingsCounts = detectData?.findingsCounts || {
+      HIGH: 0,
+      MEDIUM: 0,
+      LOW: 0,
+      UNKNOWN: 0,
+    };
+
+    // Write detect results (or empty if detection failed)
+    const findingsResult = writeFindingsJson(paths.baseDir, detectData);
+
     // Write summary with stable digest
     writeSummaryJson(paths.summaryJson, {
       runId,
@@ -276,6 +558,8 @@ export async function runCommand(options) {
       command: 'run',
       url,
       notes: 'Run completed successfully',
+      metrics,
+      findingsCounts,
     }, {
       expectationsTotal: expectations.length,
       attempted: observeData.stats?.attempted || 0,
@@ -284,36 +568,15 @@ export async function runCommand(options) {
       coverageGaps: detectData.stats?.coverageGaps || 0,
       unproven: detectData.stats?.unproven || 0,
       informational: detectData.stats?.informational || 0,
+      ...metrics,
+      ...findingsCounts,
     });
     
-    // Write detect results (or empty if detection failed)
-    writeFindingsJson(paths.baseDir, detectData);
-    
-    // Write traces (at least phase events)
-    const traces = [
-      {
-        type: 'phase:started',
-        timestamp: startedAt,
-        phase: 'Detect Project',
-      },
-      {
-        type: 'phase:completed',
-        timestamp: new Date().toISOString(),
-        phase: 'Detect Project',
-      },
-      {
-        type: 'phase:started',
-        timestamp: new Date().toISOString(),
-        phase: 'Learn',
-      },
-      {
-        type: 'phase:completed',
-        timestamp: completedAt,
-        phase: 'Learn',
-      },
-    ];
-    
-    const tracesContent = traces.map(t => JSON.stringify(t)).join('\n') + '\n';
+    // Write traces (include all events including heartbeats)
+    const allEvents = events.getEvents();
+    const tracesContent = allEvents
+      .map(e => JSON.stringify(e))
+      .join('\n') + '\n';
     atomicWriteText(paths.tracesJsonl, tracesContent);
     
     // Write project profile
@@ -324,26 +587,155 @@ export async function runCommand(options) {
     
     // Write observe results
     writeObserveJson(paths.baseDir, observeData);
+
+    // PHASE 6: Verify artifacts after all writers finish
+    const { verifyRun } = await import('../../verax/core/artifacts/verifier.js');
+    const verification = verifyRun(paths.baseDir, paths.artifactVersions);
+    
+    // Determine final status based on verification
+    let finalStatus = 'COMPLETE';
+    if (!verification.ok) {
+      finalStatus = 'INVALID';
+    } else if (verification.warnings.length > 0) {
+      finalStatus = 'VALID_WITH_WARNINGS';
+    }
+    
+    // PHASE 21.2: Compute determinism summary for run.status.json
+    let determinismSummary = null;
+    try {
+      const decisionsPath = resolve(paths.baseDir, 'decisions.json');
+      if (existsSync(decisionsPath)) {
+        const decisions = JSON.parse(readFileSync(decisionsPath, 'utf-8'));
+        const { DecisionRecorder } = await import('../../verax/core/determinism-model.js');
+        const recorder = DecisionRecorder.fromExport(decisions);
+        const { computeDeterminismVerdict } = await import('../../verax/core/determinism/contract.js');
+        const verdict = computeDeterminismVerdict(recorder);
+        
+        determinismSummary = {
+          verdict: verdict.verdict, // DETERMINISTIC or NON_DETERMINISTIC
+          message: verdict.message,
+          adaptiveEventsCount: verdict.adaptiveEvents.length
+        };
+      }
+    } catch (error) {
+      // Ignore errors - determinism summary is optional
+    }
+    
+    // Write completed status with contract + enforcement snapshot + verification + determinism
+    atomicWriteJson(paths.runStatusJson, {
+      contractVersion: 1,
+      artifactVersions: paths.artifactVersions,
+      status: finalStatus,
+      runId,
+      startedAt,
+      completedAt,
+      enforcement: findingsResult?.payload?.enforcement || null,
+      verification: {
+        ok: verification.ok,
+        status: finalStatus,
+        errorsCount: verification.errors.length,
+        warningsCount: verification.warnings.length,
+        verifiedAt: verification.verifiedAt
+      },
+      // PHASE 21.2: Determinism summary
+      determinismSummary: determinismSummary
+    });
+    
+    // Update metadata with completion time
+    atomicWriteJson(paths.runMetaJson, {
+      contractVersion: 1,
+      artifactVersions: paths.artifactVersions,
+      veraxVersion: getVersion(),
+      nodeVersion: process.version,
+      platform: process.platform,
+      cwd: projectRoot,
+      command: 'run',
+      args: { url: resolvedUrl, fixture: fixtureId, src, out },
+      url: resolvedUrl,
+      fixtureId: fixtureId,
+      src: srcPath,
+      startedAt,
+      completedAt,
+      error: null,
+    });
     
     events.emit('phase:completed', {
       phase: 'Finalize Artifacts',
       message: 'Run artifacts written',
     });
     
+    // Emit final summary event
+    if (json) {
+      events.emit('run:complete', {
+        runId,
+        url,
+        command: 'run',
+        findingsCounts,
+        metrics,
+        digest: {
+          expectationsTotal: expectations.length,
+          attempted: observeData.stats?.attempted || 0,
+          observed: observeData.stats?.observed || 0,
+          silentFailures: detectData.stats?.silentFailures || 0,
+          coverageGaps: detectData.stats?.coverageGaps || 0,
+          unproven: detectData.stats?.unproven || 0,
+          informational: detectData.stats?.informational || 0,
+        }
+      });
+    }
+    
     // Print summary if not JSON mode
     if (!json) {
       console.log('\nRun complete.');
       console.log(`Run ID: ${runId}`);
       console.log(`Artifacts: ${paths.baseDir}`);
+      
+      // PHASE 21.2: Display determinism truth
+      if (determinismSummary) {
+        console.log('');
+        if (determinismSummary.verdict === 'DETERMINISTIC') {
+          console.log('Deterministic: YES');
+        } else {
+          console.log(`Deterministic: NO (${determinismSummary.message})`);
+          if (determinismSummary.adaptiveEventsCount > 0) {
+            console.log(`  Adaptive events detected: ${determinismSummary.adaptiveEventsCount}`);
+          }
+        }
+      }
+      
+      // PHASE 6: Display verification results
+      const { formatVerificationOutput } = await import('../../verax/core/artifacts/verifier.js');
+      const verificationOutput = formatVerificationOutput(verification, verbose);
+      console.log('');
+      console.log(verificationOutput);
+      
+      // PHASE 21.9: Display performance metrics
+      const { loadPerformanceReport } = await import('../../verax/core/perf/perf.report.js');
+      const { formatPerformanceMetrics } = await import('../../verax/core/perf/perf.display.js');
+      const perfReport = loadPerformanceReport(projectRoot, runId);
+      if (perfReport) {
+        console.log('');
+        console.log(formatPerformanceMetrics(perfReport));
+      }
     }
     
     return { runId, paths, success: true };
   } catch (error) {
+    // Clear watchdog timer on error
+    if (watchdogTimer) {
+      clearTimeout(watchdogTimer);
+      watchdogTimer = null;
+    }
+    
+    events.stopHeartbeat();
+    
     // Mark run as FAILED if we have paths
     if (paths && runId && startedAt) {
       try {
         const failedAt = new Date().toISOString();
         atomicWriteJson(paths.runStatusJson, {
+          contractVersion: 1,
+          artifactVersions: paths.artifactVersions,
           status: 'FAILED',
           runId,
           startedAt,
@@ -353,13 +745,16 @@ export async function runCommand(options) {
         
         // Update metadata
         atomicWriteJson(paths.runMetaJson, {
+          contractVersion: 1,
+          artifactVersions: paths.artifactVersions,
           veraxVersion: getVersion(),
           nodeVersion: process.version,
           platform: process.platform,
           cwd: projectRoot,
           command: 'run',
-          args: { url, src, out },
-          url,
+          args: { url: resolvedUrl || url, fixture: fixtureId, src, out },
+          url: resolvedUrl || url,
+          fixtureId: fixtureId,
           src: srcPath,
           startedAt,
           completedAt: failedAt,
@@ -374,7 +769,7 @@ export async function runCommand(options) {
             startedAt,
             completedAt: failedAt,
             command: 'run',
-            url,
+            url: resolvedUrl || url,
             notes: `Run failed: ${error.message}`,
           }, {
             expectationsTotal: 0,