@veraxhq/verax 0.1.0 → 0.2.0

package/src/verax/shared/url-normalizer.js

@@ -0,0 +1,162 @@
+/**
+ * URL normalization for frontier deduplication and canonical form
+ * Prevents infinite frontier growth from utm_* params and other tracking/session identifiers
+ */
+
+/**
+ * Tracking parameter prefixes to remove from URLs
+ */
+const TRACKING_PARAM_PREFIXES = [
+  'utm_',
+  'gclid',
+  'fbclid',
+  'msclkid',
+  'click_id',
+  'session',
+  'sid',
+  'tracking',
+  'ref',
+  'source',
+  'campaign',
+  'medium'
+];
+
+/**
+ * Check if a parameter name should be dropped
+ * @param {string} name - Parameter name
+ * @returns {boolean}
+ */
+function isTrackingParam(name) {
+  const lowerName = name.toLowerCase();
+  return TRACKING_PARAM_PREFIXES.some(prefix =>
+    lowerName.startsWith(prefix.toLowerCase())
+  );
+}
+
+/**
+ * Normalize a URL to canonical form:
+ * 1. Remove hash fragments
+ * 2. Sort query parameters alphabetically
+ * 3. Drop tracking/session parameters
+ * 4. Decode percent-encoding for consistency
+ * 5. Ensure protocol and host are lowercase
+ *
+ * @param {string} url - URL to normalize
+ * @returns {string} Normalized URL
+ */
+export function normalizeUrl(url) {
+  try {
+    const parsed = new URL(url);
+
+    // Step 1: Lowercase protocol and host
+    parsed.protocol = parsed.protocol.toLowerCase();
+    parsed.hostname = parsed.hostname.toLowerCase();
+
+    // Step 2: Remove hash
+    parsed.hash = '';
+
+    // Step 3: Remove and drop tracking params
+    const params = new URLSearchParams(parsed.search);
+    const filteredParams = new URLSearchParams();
+
+    // Sort and filter params
+    const paramEntries = Array.from(params.entries()).sort((a, b) => a[0].localeCompare(b[0]));
+    for (const [name, value] of paramEntries) {
+      if (!isTrackingParam(name)) {
+        filteredParams.append(name, value);
+      }
+    }
+
+    // Step 4: Reconstruct search string (sorted)
+    parsed.search = filteredParams.toString();
+
+    // Step 5: Return full canonical URL
+    return parsed.toString();
+  } catch (err) {
+    // If URL parsing fails, return original
+    return url;
+  }
+}
+
+/**
+ * Alias for normalizeUrl for clarity in frontier deduplication contexts
+ * @param {string} url - URL to canonicalize
+ * @returns {string} Canonical URL
+ */
+export function canonicalizeUrl(url) {
+  return normalizeUrl(url);
+}
+
+/**
+ * Drop tracking parameters from a URL
+ * Preserves hash and other non-tracking parameters
+ *
+ * @param {string} url - URL to clean
+ * @returns {string} URL with tracking params removed
+ */
+export function dropTrackingParams(url) {
+  try {
+    const parsed = new URL(url);
+    const params = new URLSearchParams(parsed.search);
+
+    // Filter out tracking params
+    for (const name of params.keys()) {
+      if (isTrackingParam(name)) {
+        params.delete(name);
+      }
+    }
+
+    parsed.search = params.toString();
+    return parsed.toString();
+  } catch (err) {
+    return url;
+  }
+}
+
+/**
+ * Check if two URLs are equivalent in canonical form
+ * @param {string} url1 - First URL
+ * @param {string} url2 - Second URL
+ * @returns {boolean} True if canonically equivalent
+ */
+export function areUrlsEquivalent(url1, url2) {
+  return normalizeUrl(url1) === normalizeUrl(url2);
+}
+
+/**
+ * Extract normalized domain from URL
+ * @param {string} url - URL
+ * @returns {string} Domain (protocol://hostname)
+ */
+export function getDomain(url) {
+  try {
+    const parsed = new URL(url);
+    return `${parsed.protocol}//${parsed.hostname}`;
+  } catch (err) {
+    return '';
+  }
+}
+
+/**
+ * Count tracked parameters in a URL
+ * Useful for diagnostics
+ * @param {string} url - URL
+ * @returns {number} Count of tracking parameters
+ */
+export function countTrackingParams(url) {
+  try {
+    const parsed = new URL(url);
+    const params = new URLSearchParams(parsed.search);
+    let count = 0;
+
+    for (const name of params.keys()) {
+      if (isTrackingParam(name)) {
+        count++;
+      }
+    }
+
+    return count;
+  } catch (err) {
+    return 0;
+  }
+}

package/src/verax/shared/zip-artifacts.js

@@ -0,0 +1,65 @@
+/**
+ * Wave 5 — Artifact Packaging
+ * 
+ * Creates a zip file containing scan artifacts.
+ * Cross-platform using archiver library.
+ */
+
+import { createWriteStream } from 'fs';
+import { dirname, resolve, basename } from 'path';
+import { mkdirSync } from 'fs';
+
+// Dynamic import for archiver (dev dependency)
+let archiver;
+async function getArchiver() {
+  if (!archiver) {
+    archiver = (await import('archiver')).default;
+  }
+  return archiver;
+}
+
+/**
+ * Create a zip file containing artifacts from a run directory
+ * @param {string} runDir - Directory containing artifacts (e.g., .verax/runs/<runId>)
+ * @param {string} outputPath - Full path where zip should be created (optional, defaults to runDir/artifacts.zip)
+ * @returns {Promise<string>} Path to created zip file
+ */
+export async function createArtifactsZip(runDir, outputPath = null) {
+  const Archiver = await getArchiver();
+  
+  return new Promise((resolvePromise, reject) => {
+    // Determine output path
+    if (!outputPath) {
+      outputPath = resolve(runDir, 'artifacts.zip');
+    }
+    
+    // Ensure parent directory exists
+    mkdirSync(dirname(outputPath), { recursive: true });
+    
+    // Create write stream
+    const output = createWriteStream(outputPath);
+    const archive = Archiver('zip', {
+      zlib: { level: 9 } // Maximum compression
+    });
+    
+    // Handle errors
+    archive.on('error', (err) => {
+      reject(err);
+    });
+    
+    output.on('close', () => {
+      resolvePromise(outputPath);
+    });
+    
+    // Pipe archive data to file
+    archive.pipe(output);
+    
+    // Add directory contents to archive
+    // Use glob pattern to include all files recursively
+    archive.directory(runDir, basename(runDir), { date: new Date() });
+    
+    // Finalize the archive
+    archive.finalize();
+  });
+}
+

package/src/verax/validate/context-validator.js

@@ -0,0 +1,244 @@
+/**
+ * Wave 1 — Context Validator
+ *
+ * Validates that the target URL matches the project being analyzed.
+ * Checks if extracted routes exist on the live site by:
+ * 1. Fetching the homepage and parsing internal links
+ * 2. Checking if any extracted route paths match internal links
+ * 3. For SPAs, also checking if routes return 200 (SPA fallback)
+ */
+
+import { chromium } from 'playwright';
+import { parse } from 'node-html-parser';
+
+const CONTEXT_CHECK_TIMEOUT_MS = 8000;
+const MAX_ROUTES_TO_CHECK = 20;
+const MAX_LINKS_TO_PARSE = 100;
+
+function normalizePathForContext(path) {
+	if (!path) return '/';
+
+	let normalized = path.split('#')[0].split('?')[0];
+	if (!normalized.startsWith('/')) {
+		normalized = '/' + normalized;
+	}
+
+	// Normalize common static site patterns
+	if (normalized.toLowerCase() === '/index.html') {
+		return '/';
+	}
+
+	if (normalized.toLowerCase().endsWith('.html')) {
+		normalized = normalized.slice(0, -5) || '/';
+	}
+
+	if (normalized.endsWith('/') && normalized !== '/') {
+		normalized = normalized.slice(0, -1) || '/';
+	}
+
+	return normalized || '/';
+}
+
+/**
+ * Validate context by checking if project routes match live site
+ * @param {Object} manifest - Manifest with routes and projectType
+ * @param {string} baseUrl - Target URL to validate against
+ * @param {boolean} forced - Whether --force flag was used
+ * @returns {Promise<Object>} Context check result
+ */
+export async function validateContext(manifest, baseUrl, forced = false) {
+	const publicRoutes = manifest.publicRoutes || [];
+	const projectType = manifest.projectType || 'unknown';
+
+	// If no routes extracted, context validation doesn't apply
+	if (publicRoutes.length === 0) {
+		return {
+			ran: false,
+			forced: forced,
+			verdict: null,
+			matchedRoutesCount: 0,
+			matchedLinksCount: 0,
+			sampleMatched: [],
+			reason: 'no_routes_extracted'
+		};
+	}
+
+	// For file:// URLs (local development/testing), assume context is valid if any route is extracted
+	// since we can't reliably validate local file system paths
+	if (baseUrl.startsWith('file://')) {
+		return {
+			ran: true,
+			forced: forced,
+			verdict: 'VALID_CONTEXT',
+			matchedRoutesCount: publicRoutes.length,
+			matchedLinksCount: publicRoutes.length,
+			sampleMatched: publicRoutes.slice(0, 5),
+			reason: 'file_protocol_skip_validation'
+		};
+	}
+
+	let baseOrigin;
+	let basePathCandidate;
+	try {
+		const urlObj = new URL(baseUrl);
+		baseOrigin = urlObj.origin;
+		// Treat the requested URL path as a candidate route so static sites without links still match
+		const basePath = normalizePathForContext(urlObj.pathname);
+		// Include the base path in the internal link set so at least one known route can match
+		// when the homepage itself is part of the manifest.
+		basePathCandidate = basePath;
+	} catch (error) {
+		return {
+			ran: false,
+			forced: forced,
+			verdict: null,
+			matchedRoutesCount: 0,
+			matchedLinksCount: 0,
+			sampleMatched: [],
+			reason: 'invalid_url'
+		};
+	}
+
+	// Normalize route paths for comparison
+	const normalizedRoutes = publicRoutes
+		.slice(0, MAX_ROUTES_TO_CHECK)
+		.map(route => normalizePathForContext(route));
+
+	const browser = await chromium.launch({ headless: true });
+	const context = await browser.newContext({
+		viewport: { width: 1280, height: 720 }
+	});
+	const page = await context.newPage();
+
+	try {
+		// Fetch homepage and parse internal links
+		const response = await page.goto(baseUrl, {
+			waitUntil: 'domcontentloaded',
+			timeout: CONTEXT_CHECK_TIMEOUT_MS
+		});
+
+		await page.waitForTimeout(500); // Allow SPA to render
+
+		const html = await page.content();
+		const root = parse(html);
+		const links = root.querySelectorAll('a[href]');
+
+		// Extract internal links
+		const internalLinks = new Set();
+		if (basePathCandidate) {
+			internalLinks.add(basePathCandidate);
+		}
+		for (const link of links.slice(0, MAX_LINKS_TO_PARSE)) {
+			const href = link.getAttribute('href');
+			if (!href) continue;
+
+			try {
+				// Resolve relative URLs
+				const resolvedUrl = new URL(href, baseUrl);
+				if (resolvedUrl.origin === baseOrigin) {
+					const normalizedPath = normalizePathForContext(resolvedUrl.pathname);
+					internalLinks.add(normalizedPath);
+				}
+			} catch (e) {
+				// If href is relative, try direct path matching
+				if (href.startsWith('/') || (!href.startsWith('http') && !href.startsWith('#'))) {
+					const path = href.split('#')[0].split('?')[0];
+					const normalizedPath = normalizePathForContext(path);
+					internalLinks.add(normalizedPath);
+				}
+			}
+		}
+
+		// Check route reachability for SPAs (may return 200 due to fallback)
+		const routeReachabilityChecks = [];
+		if (projectType === 'react_spa' || projectType.startsWith('nextjs_')) {
+			// Sample a few routes to check if they return 200 (SPA fallback)
+			const routesToCheck = normalizedRoutes.slice(0, 5);
+			for (const routePath of routesToCheck) {
+				const candidates = new Set([routePath]);
+				if (routePath === '/') {
+					candidates.add('/index.html');
+				} else if (!routePath.endsWith('.html')) {
+					candidates.add(`${routePath}.html`);
+				}
+
+				for (const candidate of candidates) {
+					try {
+						const routeUrl = baseOrigin + candidate;
+						const routeResponse = await page.goto(routeUrl, {
+							waitUntil: 'domcontentloaded',
+							timeout: CONTEXT_CHECK_TIMEOUT_MS
+						});
+						if (routeResponse && routeResponse.status() >= 200 && routeResponse.status() < 300) {
+							routeReachabilityChecks.push(routePath);
+							break;
+						}
+					} catch (e) {
+						// Route check failed
+					}
+				}
+			}
+		}
+
+		// Find intersection: routes that match internal links or are reachable
+		const matchedRoutes = new Set();
+		const matchedLinks = new Set();
+		const sampleMatched = [];
+
+		for (const route of normalizedRoutes) {
+			if (internalLinks.has(route)) {
+				matchedRoutes.add(route);
+				matchedLinks.add(route);
+				if (sampleMatched.length < 5) {
+					sampleMatched.push(route);
+				}
+			} else if (routeReachabilityChecks.includes(route)) {
+				matchedRoutes.add(route);
+				if (sampleMatched.length < 5) {
+					sampleMatched.push(route);
+				}
+			}
+		}
+
+		const matchedRoutesCount = matchedRoutes.size;
+		const matchedLinksCount = matchedLinks.size;
+		const totalRoutes = normalizedRoutes.length;
+
+		// Require at least a majority of routes to match to trust the context
+		const requiredMatches = totalRoutes > 0 ? Math.max(1, Math.ceil(totalRoutes / 2)) : 0;
+
+		let verdict = null;
+		if (totalRoutes > 0 && matchedRoutesCount < requiredMatches) {
+			verdict = forced ? 'INVALID_CONTEXT_FORCED' : 'INVALID_CONTEXT';
+		} else {
+			verdict = 'VALID_CONTEXT';
+		}
+
+		return {
+			ran: true,
+			forced: forced,
+			verdict: verdict,
+			matchedRoutesCount: matchedRoutesCount,
+			matchedLinksCount: matchedLinksCount,
+			totalRoutesChecked: totalRoutes,
+			sampleMatched: sampleMatched,
+			internalLinksFound: internalLinks.size,
+			reason: matchedRoutesCount < requiredMatches ? 'insufficient_route_match' : 'routes_matched'
+		};
+
+	} catch (error) {
+		// Context check failed - can't determine validity
+		return {
+			ran: true,
+			forced: forced,
+			verdict: forced ? 'INVALID_CONTEXT_FORCED' : 'INVALID_CONTEXT',
+			matchedRoutesCount: 0,
+			matchedLinksCount: 0,
+			sampleMatched: [],
+			reason: 'context_check_failed',
+			error: error.message
+		};
+	} finally {
+		await browser.close();
+	}
+}