@veraxhq/verax 0.2.0 → 0.3.0

package/src/verax/observe/index.js

@@ -1,21 +1,13 @@
 import { resolve, dirname } from 'path';
-import { mkdirSync, readFileSync, existsSync } from 'fs';
+import { mkdirSync } from 'fs';
 import { createBrowser, navigateToUrl, closeBrowser } from './browser.js';
-import { discoverAllInteractions } from './interaction-discovery.js';
 import { captureScreenshot } from './evidence-capture.js';
-import { runInteraction } from './interaction-runner.js';
-import { writeTraces } from './traces-writer.js';
-import { getBaseOrigin, isExternalUrl } from './domain-boundary.js';
-
+import { getBaseOrigin } from './domain-boundary.js';
 import { DEFAULT_SCAN_BUDGET } from '../shared/scan-budget.js';
-import { executeProvenExpectations } from './expectation-executor.js';
-import { isProvenExpectation } from '../shared/expectation-prover.js';
 import { PageFrontier } from './page-frontier.js';
-import { deriveObservedExpectation, shouldAttemptRepeatObservedExpectation, evaluateObservedExpectation } from './observed-expectation.js';
-import { computeRouteBudget } from '../core/budget-engine.js';
-import { loadPreviousSnapshot, saveSnapshot, buildSnapshot, compareSnapshots, shouldSkipInteractionIncremental } from '../core/incremental-store.js';
 import SilenceTracker from '../core/silence-model.js';
-import { DecisionRecorder, recordBudgetProfile, recordTimeoutConfig, recordAdaptiveStabilization, recordTruncation, recordEnvironment } from '../core/determinism-model.js';
+import { DecisionRecorder, recordBudgetProfile, recordTimeoutConfig, recordEnvironment } from '../core/determinism-model.js';
+import { setupManifestAndExpectations, processTraversalResults, writeDeterminismArtifacts } from './observe-helpers.js';
 
 /**
  * OBSERVE PHASE - Execute interactions and capture runtime behavior
@@ -58,135 +50,51 @@ export async function observe(url, manifestPath = null, scanBudgetOverride = nul
   recordEnvironment(decisionRecorder, { browserType: 'chromium', viewport: { width: 1280, height: 720 } });
   
   // Phase 5: Detect projectDir if not provided (for backwards compatibility with tests)
-  if (!projectDir) {
-    projectDir = process.cwd();
+  // Use a local variable to avoid reassigning function parameter
+  let resolvedProjectDir = projectDir;
+  if (!resolvedProjectDir) {
+    resolvedProjectDir = process.cwd();
   }
-  
+
   // Phase 4: Extract safety flags
   const { allowWrites = false, allowRiskyActions = false, allowCrossOrigin = false } = safetyFlags;
   let blockedNetworkWrites = [];
   let blockedCrossOrigin = [];
-  
-  // Phase 4: Setup network interception firewall
-  await page.route('**/*', (route) => {
-    const request = route.request();
-    const method = request.method();
-    const requestUrl = request.url();
-    const resourceType = request.resourceType();
-    
-    // Check cross-origin blocking (skip for file:// URLs)
-    if (!allowCrossOrigin && !requestUrl.startsWith('file://')) {
-      try {
-        const reqOrigin = new URL(requestUrl).origin;
-        if (reqOrigin !== baseOrigin) {
-          blockedCrossOrigin.push({
-            url: requestUrl,
-            origin: reqOrigin,
-            method,
-            resourceType,
-            timestamp: Date.now()
-          });
-          
-          silenceTracker.record({
-            scope: 'safety',
-            reason: 'cross_origin_blocked',
-            description: `Cross-origin request blocked: ${method} ${requestUrl}`,
-            context: { url: requestUrl, origin: reqOrigin, method, baseOrigin },
-            impact: 'request_blocked'
-          });
-          
-          return route.abort('blockedbyclient');
-        }
-      } catch (e) {
-        // Invalid URL, allow and let browser handle
-      }
-    }
-    
-    // Check write method blocking
-    if (!allowWrites && ['POST', 'PUT', 'PATCH', 'DELETE'].includes(method)) {
-      // Check if it's a GraphQL mutation (best-effort)
-      const isGraphQLMutation = requestUrl.includes('/graphql') && method === 'POST';
-      
-      blockedNetworkWrites.push({
-        url: requestUrl,
-        method,
-        resourceType,
-        isGraphQLMutation,
-        timestamp: Date.now()
-      });
-      
-      silenceTracker.record({
-        scope: 'safety',
-        reason: 'blocked_network_write',
-        description: `Network write blocked: ${method} ${requestUrl}${isGraphQLMutation ? ' (GraphQL mutation)' : ''}`,
-        context: { url: requestUrl, method, resourceType, isGraphQLMutation },
-        impact: 'write_blocked'
-      });
-      
-      return route.abort('blockedbyclient');
-    }
-    
-    // Allow request
-    route.continue();
-  });
+
+  // PHASE 21.3: Setup network interception using safety observer
+  const { setupNetworkInterception } = await import('./observers/safety-observer.js');
+  // Create temporary context for network interception setup
+  const tempContext = {
+    page,
+    baseOrigin,
+    safetyFlags: { allowWrites, allowCrossOrigin },
+    silenceTracker,
+    blockedNetworkWrites,
+    blockedCrossOrigin
+  };
+  await setupNetworkInterception(tempContext);
   
   try {
     await navigateToUrl(page, url, scanBudget);
 
-    const projectDir = manifestPath ? dirname(dirname(dirname(manifestPath))) : process.cwd();
-    const observeDir = resolve(projectDir, '.veraxverax', 'observe');
-    const screenshotsDir = resolve(observeDir, 'screenshots');
+    // If manifestPath is provided, derive projectDir from it (override default)
+    if (manifestPath) {
+      resolvedProjectDir = dirname(dirname(dirname(manifestPath)));
+    }
+    if (!runId) {
+      throw new Error('runId is required');
+    }
+    const { getScreenshotDir } = await import('../core/run-id.js');
+    const screenshotsDir = getScreenshotDir(resolvedProjectDir, runId);
     mkdirSync(screenshotsDir, { recursive: true });
     
     const timestamp = Date.now();
     const initialScreenshot = resolve(screenshotsDir, `initial-${timestamp}.png`);
     await captureScreenshot(page, initialScreenshot);
     
-    // 1) Execute PROVEN expectations first (if manifest exists)
-    let manifest = null;
-    let expectationResults = null;
-    let expectationCoverageGaps = [];
-    let incrementalMode = false;
-    let snapshotDiff = null;
-    let oldSnapshot = null;
-
-    if (manifestPath && existsSync(manifestPath)) {
-      try {
-        const manifestContent = readFileSync(manifestPath, 'utf-8');
-        manifest = JSON.parse(manifestContent);
-        
-        // SCALE INTELLIGENCE: Load previous snapshot for incremental mode
-        oldSnapshot = loadPreviousSnapshot(projectDir);
-        if (oldSnapshot) {
-          const currentSnapshot = buildSnapshot(manifest, []);
-          snapshotDiff = compareSnapshots(oldSnapshot, currentSnapshot);
-          incrementalMode = !snapshotDiff.hasChanges; // Use incremental if nothing changed
-        }
-        
-        const provenCount = (manifest.staticExpectations || []).filter(exp => isProvenExpectation(exp)).length;
-        if (provenCount > 0) {
-          expectationResults = await executeProvenExpectations(
-            page,
-            manifest,
-            url,
-            screenshotsDir,
-            scanBudget,
-            startTime,
-            projectDir
-          );
-          expectationCoverageGaps = expectationResults.coverageGaps || [];
-        }
-      } catch (err) {
-        // Record manifest load/expectation execution failure as silence
-        silenceTracker.record({
-          scope: 'discovery',
-          reason: 'discovery_error',
-          description: 'Manifest load or expectation execution failed',
-          context: { error: err?.message },
-          impact: 'incomplete_check'
-        });
-      }
-    }
+    // PHASE 21.3: Setup manifest and expectations using helper
+    const { manifest, expectationResults, expectationCoverageGaps, incrementalMode, snapshotDiff, oldSnapshot } = 
+      await setupManifestAndExpectations(manifestPath, resolvedProjectDir, page, url, screenshotsDir, scanBudget, startTime, silenceTracker);
 
     // Reset to start URL before traversal
     await navigateToUrl(page, url, scanBudget);
@@ -200,680 +108,72 @@ export async function observe(url, manifestPath = null, scanBudgetOverride = nul
     let totalInteractionsExecuted = 0;
     let remainingInteractionsGaps = [];
 
-    let nextPageUrl = frontier.getNextUrl();
-
-    while (nextPageUrl && Date.now() - startTime < scanBudget.maxScanDurationMs) {
-      if (frontier.isPageLimitExceeded()) {
-        // PHASE 6: Record truncation decision
-        recordTruncation(decisionRecorder, 'pages', {
-          limit: scanBudget.maxPages,
-          reached: frontier.pagesVisited
-        });
-        
-        silenceTracker.record({
-          scope: 'page',
-          reason: 'page_limit_exceeded',
-          description: `Reached maximum of ${scanBudget.maxPages} pages visited`,
-          context: { pagesVisited: frontier.pagesVisited, maxPages: scanBudget.maxPages },
-          impact: 'blocks_nav'
-        });
-        break;
-      }
-
-      // Check if we're already on the target page (from navigation via link click)
-      const currentUrl = page.url();
-      const normalizedNext = frontier.normalizeUrl(nextPageUrl);
-      const normalizedCurrent = frontier.normalizeUrl(currentUrl);
-      const alreadyOnPage = normalizedCurrent === normalizedNext;
-
-      if (!alreadyOnPage) {
-        // Navigate to next page
-        try {
-          await navigateToUrl(page, nextPageUrl, scanBudget);
-        } catch (error) {
-          // Record navigation failure as silence and skip
-          silenceTracker.record({
-            scope: 'navigation',
-            reason: 'navigation_timeout',
-            description: 'Navigation to page failed',
-            context: { targetUrl: nextPageUrl },
-            impact: 'blocks_nav'
-          });
-          const normalizedFailed = frontier.normalizeUrl(nextPageUrl);
-          if (!frontier.visited.has(normalizedFailed)) {
-            frontier.visited.add(normalizedFailed);
-            frontier.markVisited();
-          }
-          nextPageUrl = frontier.getNextUrl();
-          continue;
-        }
-      }
-
-      // Mark as visited and increment counter
-      // getNextUrl() marks as visited in the set but doesn't call markVisited() to increment counter
-      // So we need to call markVisited() here when we process a page
-      if (!alreadyOnPage) {
-        // We navigated via getNextUrl() - it already marked in visited set, now increment counter
-        // (getNextUrl() marks in visited set but doesn't increment pagesVisited)
-        frontier.markVisited();
-      } else {
-        // We navigated via link click (alreadyOnPage=true) - mark as visited and increment
-        if (!frontier.visited.has(normalizedNext)) {
-          frontier.visited.add(normalizedNext);
-          frontier.markVisited();
-        } else {
-          // Already marked as visited, but still increment counter since we're processing it
-          frontier.markVisited();
-        }
-      }
-
-      // Discover ALL links on this page and add to frontier BEFORE executing interactions
-      try {
-        const currentLinks = await page.locator('a[href]').all();
-        for (const link of currentLinks) {
-          try {
-            const href = await link.getAttribute('href');
-            if (href && !href.startsWith('#') && !href.startsWith('javascript:')) {
-              const resolvedUrl = href.startsWith('http') ? href : new URL(href, page.url()).href;
-              if (!isExternalUrl(resolvedUrl, baseOrigin)) {
-                frontier.addUrl(resolvedUrl);
-              }
-            }
-          } catch (error) {
-            // Record invalid URL discovery as silence
-            silenceTracker.record({
-              scope: 'discovery',
-              reason: 'discovery_error',
-              description: 'Invalid or unreadable link during discovery',
-              context: { pageUrl: page.url() },
-              impact: 'incomplete_check'
-            });
-          }
-        }
-      } catch (error) {
-        // Record link discovery failure as silence
-        silenceTracker.record({
-          scope: 'discovery',
-          reason: 'discovery_error',
-          description: 'Link discovery failed on page',
-          context: { pageUrl: page.url() },
-          impact: 'incomplete_check'
-        });
-      }
-
-      // SCALE INTELLIGENCE: Compute adaptive budget for this route
-      // Reuse currentUrl from above (already captured at line 95)
-      const routeBudget = manifest ? computeRouteBudget(manifest, currentUrl, scanBudget) : scanBudget;
-      
-      // Discover ALL interactions on this page
-      // Note: discoverAllInteractions already returns sorted interactions deterministically
-      const { interactions, coverage: pageCoverage } = await discoverAllInteractions(page, baseOrigin, routeBudget);
-      totalInteractionsDiscovered += interactions.length;
-      
-      // SCALE INTELLIGENCE: Apply adaptive budget cap (interactions are already sorted deterministically)
-      // Stable sorting ensures determinism: same interactions → same order
-      const sortedInteractions = interactions.slice(0, routeBudget.maxInteractionsPerPage);
-
-      // Track if we navigated during interaction execution
-      let navigatedToNewPage = false;
-      let navigatedPageUrl = null;
-      let remainingInteractionsStartIndex = 0;
-
-      // Execute discovered interactions on this page (sorted for determinism)
-      for (let i = 0; i < sortedInteractions.length; i++) {
-        if (Date.now() - startTime > scanBudget.maxScanDurationMs) {
-          // PHASE 6: Record truncation decision
-          recordTruncation(decisionRecorder, 'time', {
-            limit: scanBudget.maxScanDurationMs,
-            elapsed: Date.now() - startTime
-          });
-          
-          // Mark remaining interactions as COVERAGE_GAP
-          silenceTracker.record({
-            scope: 'interaction',
-            reason: 'scan_time_exceeded',
-            description: `Scan time limit (${scanBudget.maxScanDurationMs}ms) exceeded`,
-            context: { 
-              elapsed: Date.now() - startTime,
-              maxDuration: scanBudget.maxScanDurationMs,
-              remainingInteractions: sortedInteractions.length - i
-            },
-            impact: 'blocks_nav',
-            count: sortedInteractions.length - i
-          });
-          remainingInteractionsStartIndex = i;
-          break;
-        }
-
-        if (totalInteractionsExecuted >= routeBudget.maxInteractionsPerPage) {
-          // PHASE 6: Record truncation decision
-          recordTruncation(decisionRecorder, 'interactions', {
-            limit: routeBudget.maxInteractionsPerPage,
-            reached: totalInteractionsExecuted,
-            scope: 'per_page'
-          });
-          
-          // Route-specific budget exceeded
-          silenceTracker.record({
-            scope: 'interaction',
-            reason: 'route_interaction_limit_exceeded',
-            description: `Reached max ${routeBudget.maxInteractionsPerPage} interactions per page`,
-            context: {
-              currentPage: page.url(),
-              executed: totalInteractionsExecuted,
-              maxPerPage: routeBudget.maxInteractionsPerPage,
-              remainingInteractions: sortedInteractions.length - i
-            },
-            impact: 'affects_expectations',
-            count: sortedInteractions.length - i
-          });
-          remainingInteractionsStartIndex = i;
-          break;
-        }
-
-        if (totalInteractionsExecuted >= scanBudget.maxTotalInteractions) {
-          // PHASE 6: Record truncation decision
-          recordTruncation(decisionRecorder, 'interactions', {
-            limit: scanBudget.maxTotalInteractions,
-            reached: totalInteractionsExecuted,
-            scope: 'total'
-          });
-          
-          // Mark remaining interactions as COVERAGE_GAP with reason 'budget_exceeded'
-          silenceTracker.record({
-            scope: 'interaction',
-            reason: 'interaction_limit_exceeded',
-            description: `Reached max ${scanBudget.maxTotalInteractions} total interactions`,
-            context: {
-              executed: totalInteractionsExecuted,
-              maxTotal: scanBudget.maxTotalInteractions,
-              remainingInteractions: sortedInteractions.length - i
-            },
-            impact: 'blocks_nav',
-            count: sortedInteractions.length - i
-          });
-          remainingInteractionsStartIndex = i;
-          break;
-        }
-
-        const interaction = sortedInteractions[i];
-        
-        // SCALE INTELLIGENCE: Check if interaction should be skipped in incremental mode
-        if (incrementalMode && manifest && oldSnapshot && snapshotDiff) {
-          const shouldSkip = shouldSkipInteractionIncremental(interaction, currentUrl, oldSnapshot, snapshotDiff);
-          if (shouldSkip) {
-            // Create a trace for skipped interaction (marked as incremental - will not produce findings)
-            const skippedTrace = {
-              interaction: {
-                type: interaction.type,
-                selector: interaction.selector,
-                label: interaction.label
-              },
-              before: { url: currentUrl },
-              after: { url: currentUrl },
-              incremental: true, // Mark as skipped in incremental mode - detect phase will skip this
-              resultType: 'INCREMENTAL_SKIP'
-            };
-            traces.push(skippedTrace);
-            
-            // Track incremental skip as silence
-            silenceTracker.record({
-              scope: 'interaction',
-              reason: 'incremental_unchanged',
-              description: `Skipped re-observation (unchanged in incremental mode): ${interaction.label}`,
-              context: {
-                currentPage: currentUrl,
-                selector: interaction.selector,
-                interactionLabel: interaction.label,
-                type: interaction.type
-              },
-              impact: 'affects_expectations'
-            });
-            
-            skippedInteractions.push({
-              interaction: {
-                type: interaction.type,
-                selector: interaction.selector,
-                label: interaction.label,
-                text: interaction.text
-              },
-              outcome: 'SKIPPED',
-              reason: 'incremental_unchanged',
-              url: currentUrl,
-              evidence: {
-                selector: interaction.selector,
-                label: interaction.label,
-                incremental: true
-              }
-            });
-            continue;
-          }
-        }
-
-        // Skip dangerous interactions (logout, delete, etc.) with explicit reason
-        const skipCheck = frontier.shouldSkipInteraction(interaction);
-        if (skipCheck.skip) {
-          // Track safety skip as silence
-          silenceTracker.record({
-            scope: 'interaction',
-            reason: skipCheck.reason === 'destructive' ? 'destructive_text' : 'unsafe_pattern',
-            description: `Skipped potentially dangerous interaction: ${interaction.label}`,
-            context: {
-              currentPage: page.url(),
-              selector: interaction.selector,
-              interactionLabel: interaction.label,
-              text: interaction.text,
-              skipReason: skipCheck.reason,
-              skipMessage: skipCheck.message
-            },
-            impact: 'unknown_behavior'
-          });
-          
-          skippedInteractions.push({
-            interaction: {
-              type: interaction.type,
-              selector: interaction.selector,
-              label: interaction.label,
-              text: interaction.text
-            },
-            outcome: 'SKIPPED',
-            reason: skipCheck.reason || 'safety_policy',
-            url: page.url(),
-            evidence: {
-              selector: interaction.selector,
-              label: interaction.label,
-              text: interaction.text,
-              sourcePage: page.url()
-            }
-          });
-          continue;
-        }
-
-        // Phase 4: Check action classification and safety mode
-        const { shouldBlockAction } = await import('../core/action-classifier.js');
-        const blockCheck = shouldBlockAction(interaction, { allowWrites, allowRiskyActions });
-        
-        if (blockCheck.shouldBlock) {
-          // Track blocked action as silence
-          silenceTracker.record({
-            scope: 'safety',
-            reason: 'blocked_action',
-            description: `Action blocked by safety mode: ${interaction.label} (${blockCheck.classification})`,
-            context: {
-              currentPage: page.url(),
-              selector: interaction.selector,
-              interactionLabel: interaction.label,
-              text: interaction.text,
-              classification: blockCheck.classification,
-              blockReason: blockCheck.reason
-            },
-            impact: 'action_blocked'
-          });
-          
-          skippedInteractions.push({
-            interaction: {
-              type: interaction.type,
-              selector: interaction.selector,
-              label: interaction.label,
-              text: interaction.text
-            },
-            outcome: 'BLOCKED',
-            reason: 'safety_mode',
-            classification: blockCheck.classification,
-            url: page.url(),
-            evidence: {
-              selector: interaction.selector,
-              label: interaction.label,
-              text: interaction.text,
-              classification: blockCheck.classification,
-              sourcePage: page.url()
-            }
-          });
-          
-          // Create a minimal trace for blocked interactions so they appear in output
-          const blockedTrace = {
-            interaction: {
-              type: interaction.type,
-              selector: interaction.selector,
-              label: interaction.label,
-              text: interaction.text
-            },
-            before: {
-              url: page.url(),
-              screenshot: null
-            },
-            after: {
-              url: page.url(),
-              screenshot: null
-            },
-            policy: {
-              actionBlocked: true,
-              classification: blockCheck.classification,
-              reason: blockCheck.reason
-            },
-            outcome: 'BLOCKED_BY_SAFETY_MODE',
-            timestamp: Date.now()
-          };
-          traces.push(blockedTrace);
-          
-          continue;
-        }
-
-        const beforeUrl = page.url();
-        const interactionIndex = totalInteractionsExecuted;
-        const trace = await runInteraction(
-          page,
-          interaction,
-          timestamp,
-          interactionIndex,
-          screenshotsDir,
-          baseOrigin,
-          startTime,
-          routeBudget, // Use route-specific budget
-          null,
-          silenceTracker // Pass silence tracker
-        );
-        
-        // Mark trace with incremental flag if applicable
-        if (incrementalMode && trace) {
-          trace.incremental = false; // This interaction was executed, not skipped
-        }
-
-        let repeatTrace = null;
-
-        if (trace) {
-          const matchingExpectation = expectationResults?.results?.find(r => r.trace?.interaction?.selector === trace.interaction.selector);
-          if (matchingExpectation) {
-            trace.expectationDriven = true;
-            trace.expectationId = matchingExpectation.expectationId;
-            trace.expectationOutcome = matchingExpectation.outcome;
-          } else {
-            const observedExpectation = deriveObservedExpectation(interaction, trace, baseOrigin);
-            if (observedExpectation) {
-              trace.observedExpectation = observedExpectation;
-              trace.resultType = 'OBSERVED_EXPECTATION';
-              observedExpectations.push(observedExpectation);
-
-              const repeatEligible = shouldAttemptRepeatObservedExpectation(observedExpectation, trace);
-              const budgetAllowsRepeat = repeatEligible &&
-                (Date.now() - startTime) < scanBudget.maxScanDurationMs &&
-                (totalInteractionsExecuted + 1) < scanBudget.maxTotalInteractions;
-
-              if (budgetAllowsRepeat) {
-                const repeatIndex = totalInteractionsExecuted + 1;
-                const repeatResult = await repeatObservedInteraction(
-                  page,
-                  interaction,
-                  observedExpectation,
-                  timestamp,
-                  repeatIndex,
-                  screenshotsDir,
-                  baseOrigin,
-                  startTime,
-                  scanBudget
-                );
-
-                if (repeatResult) {
-                  const repeatEvaluation = repeatResult.repeatEvaluation;
-                  trace.observedExpectation.repeatAttempted = true;
-                  trace.observedExpectation.repeated = repeatEvaluation.outcome === 'VERIFIED';
-                  trace.observedExpectation.repeatOutcome = repeatEvaluation.outcome;
-                  trace.observedExpectation.repeatReason = repeatEvaluation.reason;
-
-                  if (repeatEvaluation.outcome === 'OBSERVED_BREAK') {
-                    trace.observedExpectation.outcome = 'OBSERVED_BREAK';
-                    trace.observedExpectation.reason = 'inconsistent_on_repeat';
-                    trace.observedExpectation.confidenceLevel = 'LOW';
-                  } else if (trace.observedExpectation.repeated && trace.observedExpectation.outcome === 'VERIFIED') {
-                    trace.observedExpectation.confidenceLevel = 'MEDIUM';
-                  }
-
-                  repeatTrace = repeatResult.repeatTrace;
-                }
-              }
-            } else {
-              trace.unprovenResult = true;
-              trace.resultType = 'UNPROVEN_RESULT';
-            }
-          }
-
-          traces.push(trace);
-          totalInteractionsExecuted++;
-
-          if (repeatTrace) {
-            traces.push(repeatTrace);
-            totalInteractionsExecuted++;
-          }
-
-          const afterUrl = trace.after?.url || page.url();
-          const navigatedSameOrigin = afterUrl && afterUrl !== beforeUrl && !isExternalUrl(afterUrl, baseOrigin);
-          if (navigatedSameOrigin && interaction.type === 'link') {
-            // Link navigation - add new page to frontier (if not already visited)
-            const normalizedAfter = frontier.normalizeUrl(afterUrl);
-            const wasAlreadyVisited = frontier.visited.has(normalizedAfter);
-            if (!wasAlreadyVisited) {
-              const added = frontier.addUrl(afterUrl);
-              // If frontier was capped, record coverage gap
-              if (!added && frontier.frontierCapped) {
-                remainingInteractionsGaps.push({
-                  interaction: {
-                    type: 'link',
-                    selector: interaction.selector,
-                    label: interaction.label
-                  },
-                  reason: 'frontier_capped',
-                  url: afterUrl
-                });
-              }
-            }
-            
-            // Stay on the new page and continue executing interactions there
-            navigatedToNewPage = true;
-            navigatedPageUrl = afterUrl;
-            
-            // Discover links on the new page immediately
-            try {
-              const newPageLinks = await page.locator('a[href]').all();
-              for (const link of newPageLinks) {
-                try {
-                  const href = await link.getAttribute('href');
-                  if (href && !href.startsWith('#') && !href.startsWith('javascript:')) {
-                    const resolvedUrl = href.startsWith('http') ? href : new URL(href, page.url()).href;
-                    if (!isExternalUrl(resolvedUrl, baseOrigin)) {
-                      frontier.addUrl(resolvedUrl);
-                    }
-                  }
-                } catch (error) {
-                  // Record invalid URL discovery as silence
-                  silenceTracker.record({
-                    scope: 'discovery',
-                    reason: 'discovery_error',
-                    description: 'Invalid or unreadable link during discovery',
-                    context: { pageUrl: page.url() },
-                    impact: 'incomplete_check'
-                  });
-                }
-              }
-            } catch (error) {
-              // Record link discovery failure as silence
-              silenceTracker.record({
-                scope: 'discovery',
-                reason: 'discovery_error',
-                description: 'Link discovery failed on page',
-                context: { pageUrl: page.url() },
-                impact: 'incomplete_check'
-              });
-            }
-            
-            // Break to restart loop on new page
-            break;
-          }
-        }
-      }
-
-      // Mark remaining interactions as COVERAGE_GAP if we stopped early
-      if (remainingInteractionsStartIndex > 0 && remainingInteractionsStartIndex < sortedInteractions.length && !navigatedToNewPage) {
-        for (let j = remainingInteractionsStartIndex; j < sortedInteractions.length; j++) {
-          const reason = totalInteractionsExecuted >= scanBudget.maxTotalInteractions ? 'budget_exceeded' : 
-                        (totalInteractionsExecuted >= routeBudget.maxInteractionsPerPage ? 'route_budget_exceeded' : 'budget_exceeded');
-          remainingInteractionsGaps.push({
-            interaction: {
-              type: sortedInteractions[j].type,
-              selector: sortedInteractions[j].selector,
-              label: sortedInteractions[j].label
-            },
-            reason: reason,
-            url: currentUrl
-          });
-        }
-      }
-
-      // If we navigated to a new page, stay on it and continue (next iteration will handle it)
-      if (navigatedToNewPage && navigatedPageUrl) {
-        // Don't mark as visited yet - we'll do it at the start of next iteration
-        // This ensures the page counter is incremented when we process the page
-        // Set nextPageUrl to the navigated page so we process it in the next iteration
-        nextPageUrl = navigatedPageUrl;
-        continue;
-      }
-
-      // After executing all interactions on current page, move to next page in frontier
-      nextPageUrl = frontier.getNextUrl();
-    }
-
-    // Combine all coverage gaps
-    if (remainingInteractionsGaps.length > 0) {
-      expectationCoverageGaps.push(...remainingInteractionsGaps.map(gap => ({
-        expectationId: null,
-        type: gap.interaction.type,
-        reason: gap.reason,
-        fromPath: gap.url,
-        source: null,
-        evidence: {
-          interaction: gap.interaction
-        }
-      })));
-    }
-    
-    // Record frontier capping as coverage gap if it occurred
-    if (frontier.frontierCapped) {
-      expectationCoverageGaps.push({
-        expectationId: null,
-        type: 'navigation',
-        reason: 'frontier_capped',
-        fromPath: page.url(),
-        source: null,
-        evidence: {
-          message: `Frontier capped at ${scanBudget.maxUniqueUrls || 'unlimited'} unique URLs`
-        }
-      });
-    }
-
-    // Build coverage object matching writeTraces expected format
-    const coverage = {
-      candidatesDiscovered: totalInteractionsDiscovered,
-      candidatesSelected: totalInteractionsExecuted,
-      cap: scanBudget.maxTotalInteractions,
-      capped: totalInteractionsExecuted >= scanBudget.maxTotalInteractions || remainingInteractionsGaps.length > 0,
-      pagesVisited: frontier.pagesVisited,
-      pagesDiscovered: frontier.pagesDiscovered,
-      skippedInteractions: skippedInteractions.length,
-      interactionsDiscovered: totalInteractionsDiscovered,
-      interactionsExecuted: totalInteractionsExecuted
-    };
-    
-    // Ensure we increment pagesVisited when we navigate via getNextUrl()
-    // getNextUrl() marks as visited but doesn't increment counter - we do it here
-    // BUT: when alreadyOnPage is true, we've already marked it, so don't double-count
-
-    // Record warnings
-    if (coverage.capped) {
-      observeWarnings.push({
-        code: 'INTERACTIONS_CAPPED',
-        message: `Interaction execution capped. Visited ${coverage.pagesVisited} pages, discovered ${coverage.pagesDiscovered}, executed ${coverage.candidatesSelected} of ${coverage.candidatesDiscovered} interactions. Coverage incomplete.`
-      });
-    }
+    // PHASE 21.3: Run main traversal loop
+    const { runTraversalLoop } = await import('./observe-runner.js');
+    const traversalResult = await runTraversalLoop({
+      page,
+      url,
+      baseOrigin,
+      scanBudget,
+      startTime,
+      frontier,
+      manifest,
+      expectationResults,
+      incrementalMode,
+      oldSnapshot,
+      snapshotDiff,
+      currentUrl: page.url(),
+      screenshotsDir,
+      timestamp,
+      decisionRecorder,
+      silenceTracker,
+      traces,
+      skippedInteractions,
+      observedExpectations,
+      totalInteractionsDiscovered,
+      totalInteractionsExecuted,
+      remainingInteractionsGaps,
+      allowWrites,
+      allowRiskyActions
+    });
+
+    // PHASE 21.3: Process traversal results using helper
+    const finalTraces = [...traversalResult.traces];
+    const finalSkippedInteractions = [...traversalResult.skippedInteractions];
+    const finalObservedExpectations = [...traversalResult.observedExpectations];
+    remainingInteractionsGaps = [...traversalResult.remainingInteractionsGaps];
     
-    if (skippedInteractions.length > 0) {
-      observeWarnings.push({
-        code: 'INTERACTIONS_SKIPPED',
-        message: `Skipped ${skippedInteractions.length} dangerous interactions`,
-        details: skippedInteractions
-      });
-    }
-
-    // Append expectation traces for completeness
-    if (expectationResults && expectationResults.results) {
-      for (const result of expectationResults.results) {
-        if (result.trace) {
-          result.trace.expectationDriven = true;
-          result.trace.expectationId = result.expectationId;
-          result.trace.expectationOutcome = result.outcome;
-          traces.push(result.trace);
-        }
-      }
-    }
-
-    const observation = writeTraces(projectDir, url, traces, coverage, observeWarnings, observedExpectations, silenceTracker, runId);
-
-    // Add silence tracking to observation result
-    observation.silences = silenceTracker.getDetailedSummary();
-
-    if (expectationResults) {
-      observation.expectationExecution = {
-        totalProvenExpectations: expectationResults.totalProvenExpectations,
-        executedCount: expectationResults.executedCount,
-        coverageGapsCount: expectationCoverageGaps.length,
-        results: expectationResults.results.map(r => ({
-          expectationId: r.expectationId,
-          type: r.type,
-          fromPath: r.fromPath,
-          outcome: r.outcome,
-          reason: r.reason
-        }))
-      };
-      observation.expectationCoverageGaps = expectationCoverageGaps;
-    }
-    
-    // SCALE INTELLIGENCE: Save snapshot for next incremental run
-    if (manifest) {
-      // Build snapshot from current run (extract interactions from traces)
-      const observedInteractions = traces
-        .filter(t => t.interaction && !t.incremental)
-        .map(t => ({
-          type: t.interaction?.type,
-          selector: t.interaction?.selector,
-          url: t.before?.url || url
-        }));
-      
-      const currentSnapshot = buildSnapshot(manifest, observedInteractions);
-      saveSnapshot(projectDir, currentSnapshot);
-      
-      // Add incremental mode metadata to observation
-      observation.incremental = {
-        enabled: incrementalMode,
-        snapshotDiff: snapshotDiff,
-        skippedInteractionsCount: skippedInteractions.filter(s => s.reason === 'incremental_unchanged').length
-      };
-    }
+    const observation = await processTraversalResults(
+      traversalResult,
+      expectationResults,
+      expectationCoverageGaps,
+      remainingInteractionsGaps,
+      frontier,
+      scanBudget,
+      page,
+      url,
+      finalTraces,
+      finalSkippedInteractions,
+      finalObservedExpectations,
+      silenceTracker,
+      manifest,
+      incrementalMode,
+      snapshotDiff,
+      resolvedProjectDir,
+      runId
+    );
 
     await closeBrowser(browser);
     
-    // PHASE 6: Export determinism decisions to run directory
-    if (runId && projectDir) {
-      const runsDir = resolve(projectDir, '.verax', 'runs', runId);
-      mkdirSync(runsDir, { recursive: true });
-      const decisionsPath = resolve(runsDir, 'decisions.json');
-      const decisionsData = JSON.stringify(decisionRecorder.export(), null, 2);
-      require('fs').writeFileSync(decisionsPath, decisionsData, 'utf-8');
-    }
+    // PHASE 21.3: Write determinism artifacts using helper
+    await writeDeterminismArtifacts(resolvedProjectDir, runId, decisionRecorder);
     
     // Phase 4: Add safety mode statistics
     const safetyBlocks = {
-      actionsBlocked: skippedInteractions.filter(s => s.reason === 'safety_mode').length,
+      actionsBlocked: finalSkippedInteractions.filter(s => s.reason === 'safety_mode').length,
       networkWritesBlocked: blockedNetworkWrites.length,
       crossOriginBlocked: blockedCrossOrigin.length,
-      blockedActions: skippedInteractions.filter(s => s.reason === 'safety_mode').map(s => ({
+      blockedActions: finalSkippedInteractions.filter(s => s.reason === 'safety_mode').map(s => ({
         label: s.interaction.label,
         classification: s.classification,
         url: s.url
@@ -893,56 +193,4 @@ export async function observe(url, manifestPath = null, scanBudgetOverride = nul
   }
 }
 
-async function repeatObservedInteraction(
-  page,
-  interaction,
-  observedExpectation,
-  timestamp,
-  interactionIndex,
-  screenshotsDir,
-  baseOrigin,
-  startTime,
-  scanBudget
-) {
-  const selector = observedExpectation.evidence?.selector || interaction.selector;
-  if (!selector) return null;
-
-  const locator = page.locator(selector).first();
-  const count = await locator.count();
-  if (count === 0) {
-    return null;
-  }
-
-  const repeatInteraction = {
-    ...interaction,
-    element: locator
-  };
-
-  const repeatTrace = await runInteraction(
-    page,
-    repeatInteraction,
-    timestamp,
-    interactionIndex,
-    screenshotsDir,
-    baseOrigin,
-    startTime,
-    scanBudget,
-    null,
-    null // No silence tracker for repeat executions (not counted as new silence)
-  );
-
-  if (!repeatTrace) {
-    return null;
-  }
-
-  repeatTrace.repeatExecution = true;
-  repeatTrace.repeatOfObservedExpectationId = observedExpectation.id;
-  repeatTrace.resultType = 'OBSERVED_EXPECTATION_REPEAT';
-
-  const repeatEvaluation = evaluateObservedExpectation(observedExpectation, repeatTrace);
-
-  return {
-    repeatTrace,
-    repeatEvaluation
-  };
-}
+// PHASE 21.3: repeatObservedInteraction moved to observe-runner.js