@rankcli/agent-runtime 0.0.1

package/src/audit/deno-entry.ts

@@ -0,0 +1,66 @@
+/**
+ * Deno-compatible entry point for the audit module.
+ * This excludes Node-specific code and dependencies.
+ */
+
+// Types
+export type { AuditIssue, AuditReport, HealthScore, PageAudit } from './types.js';
+export { ISSUE_DEFINITIONS } from './types.js';
+
+// Main engine
+export { runFullAudit, formatReport } from './engine.js';
+export type { AuditOptions } from './engine.js';
+
+// Individual check modules (all use isomorphic fetch)
+export { runCrawlabilityChecks, checkRobotsTxt, checkSitemap, checkRedirects } from './checks/crawlability.js';
+export { analyzeOnPage } from './checks/on-page.js';
+export { analyzeLinks } from './checks/links.js';
+export { analyzeImages } from './checks/images.js';
+export { analyzePerformance } from './checks/performance.js';
+export { analyzeSecurity, checkCertificate } from './checks/security.js';
+export { analyzeStructuredData, suggestSchemaTypes } from './checks/structured-data.js';
+export { analyzeMobile, checkMobileResources } from './checks/mobile.js';
+
+// Advanced checks
+export { analyzeHreflang } from './checks/hreflang.js';
+export { analyzeSocialMeta } from './checks/social-meta.js';
+export { detectSoft404 } from './checks/soft-404.js';
+export { analyzeAnchorText } from './checks/anchor-text.js';
+export { analyzeCanonicalAdvanced } from './checks/canonical-advanced.js';
+export { analyzePagination } from './checks/pagination.js';
+export { analyzeRedirects } from './checks/redirect-analysis.js';
+
+// New competitor-parity checks
+export { runAIReadinessChecks, checkLlmsTxt, checkAIBotBlocking } from './checks/ai-readiness.js';
+export { analyzeSERPPreview } from './checks/serp-preview.js';
+export { analyzeLocalSEO } from './checks/local-seo.js';
+export { analyzeSecurityHeaders } from './checks/security-headers.js';
+export { analyzeContentFreshness } from './checks/content-freshness.js';
+export { analyzeDOMStructure } from './checks/dom-analysis.js';
+export { analyzeModernImages } from './checks/modern-images.js';
+export { detectTechnologies } from './checks/tech-detection.js';
+export { analyzeKeywords } from './checks/keyword-analysis.js';
+
+// URL Safety (Google Safe Browsing-style local hash database)
+export { urlSafetyDatabase, analyzeUrlSafety } from './checks/url-safety.js';
+
+// Advanced SEO checks
+export { analyzeTrackerBloat } from './checks/tracker-bloat.js';
+export { analyzeClientRendering } from './checks/client-rendering.js';
+export { analyzeRedirectChain } from './checks/redirect-chain.js';
+export { analyzeResponsiveImages } from './checks/responsive-images.js';
+export { analyzeConversionElements } from './checks/conversion-elements.js';
+export { analyzeKeywordPlacement } from './checks/keyword-placement.js';
+export { analyzeTopicalClusters } from './checks/topical-clusters.js';
+export { analyzePlatformPresence } from './checks/platform-presence.js';
+export { analyzeInteractiveTools } from './checks/interactive-tools.js';
+export { analyzeFunnelIntent } from './checks/funnel-intent.js';
+export { analyzeNavBoostSignals } from './checks/navboost-signals.js';
+export { analyzeEntitySEO } from './checks/entity-seo.js';
+export { analyzeFreshnessSignals } from './checks/freshness-signals.js';
+
+// Page discovery (multi-page crawling)
+export { discoverPagesFromLinks, parseSitemap, mergePages } from './discovery/link-crawler.js';
+export { discoverRoutesFromRepo, detectFramework, routesToUrls } from './discovery/repo-routes.js';
+export type { DiscoveredPage } from './discovery/link-crawler.js';
+export type { RouteInfo } from './discovery/repo-routes.js';

package/src/audit/discovery/index.ts

@@ -0,0 +1,15 @@
+/**
+ * Discovery Module
+ *
+ * Discovers pages to audit from various sources:
+ * - Internal links from homepage
+ * - Sitemap
+ * - Route files (with repo access)
+ */
+
+export { discoverPagesFromLinks, parseSitemap, mergePages } from './link-crawler.js';
+export type { DiscoveredPage } from './link-crawler.js';
+
+// Route discovery is exported separately
+export { discoverRoutesFromRepo } from './repo-routes.js';
+export type { RouteInfo } from './repo-routes.js';

package/src/audit/discovery/link-crawler.ts

@@ -0,0 +1,232 @@
+/**
+ * Link Crawler Module
+ *
+ * Discovers pages from a website by following internal links from the homepage.
+ * Prioritizes important pages like /pricing, /about, /features, etc.
+ */
+
+export interface DiscoveredPage {
+  url: string;
+  source: 'homepage' | 'sitemap' | 'internal-link';
+  priority: number; // 1-10, higher = more important
+}
+
+// Priority patterns - pages we want to audit first
+const HIGH_PRIORITY_PATTERNS = [
+  /^\/(pricing|about|features|contact|demo|signup|login|blog)\/?$/i,
+  /^\/(products?|services?|solutions?|plans?)\/?$/i,
+  /^\/(docs?|documentation|help|support|faq)\/?$/i,
+  /^\/(careers?|jobs?|team|company)\/?$/i,
+];
+
+const MEDIUM_PRIORITY_PATTERNS = [
+  /^\/(privacy|terms|legal|security)\/?$/i,
+  /^\/[a-z0-9-]+\/?$/i, // Top-level pages
+];
+
+// Patterns to exclude
+const EXCLUDE_PATTERNS = [
+  /\.(png|jpg|jpeg|gif|svg|webp|ico|pdf|zip|tar|gz)$/i, // Files
+  /^(mailto:|tel:|javascript:|#)/i, // Non-HTTP
+  /\/(api|_next|static|assets|cdn)\//i, // Technical paths
+  /[?&](utm_|ref=|source=)/i, // Tracking params
+];
+
+/**
+ * Extract all internal links from HTML content
+ */
+function extractLinks(html: string, baseUrl: string): string[] {
+  const base = new URL(baseUrl);
+  const links: Set<string> = new Set();
+
+  // Match all href attributes
+  const hrefRegex = /href=["']([^"']+)["']/gi;
+  let match;
+
+  while ((match = hrefRegex.exec(html)) !== null) {
+    try {
+      const href = match[1];
+
+      // Skip excluded patterns
+      if (EXCLUDE_PATTERNS.some(pattern => pattern.test(href))) {
+        continue;
+      }
+
+      // Parse the URL
+      let url: URL;
+      if (href.startsWith('http://') || href.startsWith('https://')) {
+        url = new URL(href);
+      } else if (href.startsWith('/')) {
+        url = new URL(href, base);
+      } else {
+        continue; // Skip relative paths that don't start with /
+      }
+
+      // Only include same-domain links
+      if (url.hostname !== base.hostname) {
+        continue;
+      }
+
+      // Normalize the URL
+      const normalized = normalizeUrl(url);
+      if (normalized) {
+        links.add(normalized);
+      }
+    } catch {
+      // Invalid URL, skip
+    }
+  }
+
+  return Array.from(links);
+}
+
+/**
+ * Normalize a URL for deduplication
+ */
+function normalizeUrl(url: URL): string | null {
+  // Remove hash
+  url.hash = '';
+
+  // Remove common tracking parameters
+  const paramsToRemove = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content', 'utm_term', 'ref', 'source'];
+  for (const param of paramsToRemove) {
+    url.searchParams.delete(param);
+  }
+
+  // Remove trailing slash for consistency
+  let pathname = url.pathname;
+  if (pathname.length > 1 && pathname.endsWith('/')) {
+    pathname = pathname.slice(0, -1);
+  }
+
+  // Skip root (we already have it)
+  if (pathname === '' || pathname === '/') {
+    return null;
+  }
+
+  return `${url.protocol}//${url.hostname}${pathname}${url.search}`;
+}
+
+/**
+ * Calculate priority score for a URL
+ */
+function calculatePriority(url: string): number {
+  try {
+    const pathname = new URL(url).pathname;
+
+    // High priority pages (8-10)
+    for (const pattern of HIGH_PRIORITY_PATTERNS) {
+      if (pattern.test(pathname)) {
+        return 9;
+      }
+    }
+
+    // Medium priority pages (5-7)
+    for (const pattern of MEDIUM_PRIORITY_PATTERNS) {
+      if (pattern.test(pathname)) {
+        return 6;
+      }
+    }
+
+    // Lower priority for deep paths (2-4)
+    const depth = (pathname.match(/\//g) || []).length;
+    if (depth >= 3) {
+      return 2;
+    }
+    if (depth === 2) {
+      return 4;
+    }
+
+    // Default priority for other pages
+    return 5;
+  } catch {
+    return 1;
+  }
+}
+
+/**
+ * Discover pages from HTML content
+ *
+ * @param baseUrl - The base URL of the site
+ * @param html - The HTML content of the homepage
+ * @param maxPages - Maximum number of pages to return
+ * @returns Array of discovered pages sorted by priority
+ */
+export function discoverPagesFromLinks(
+  baseUrl: string,
+  html: string,
+  maxPages: number = 5
+): DiscoveredPage[] {
+  const links = extractLinks(html, baseUrl);
+
+  // Score and sort links
+  const pages: DiscoveredPage[] = links.map(url => ({
+    url,
+    source: 'internal-link' as const,
+    priority: calculatePriority(url),
+  }));
+
+  // Sort by priority (descending) and take top N
+  pages.sort((a, b) => b.priority - a.priority);
+
+  return pages.slice(0, maxPages);
+}
+
+/**
+ * Parse sitemap XML to extract URLs
+ */
+export function parseSitemap(xml: string, baseUrl: string): DiscoveredPage[] {
+  const pages: DiscoveredPage[] = [];
+  const base = new URL(baseUrl);
+
+  // Simple regex-based XML parsing for <loc> tags
+  const locRegex = /<loc>([^<]+)<\/loc>/gi;
+  let match;
+
+  while ((match = locRegex.exec(xml)) !== null) {
+    try {
+      const url = new URL(match[1]);
+
+      // Only include same-domain URLs
+      if (url.hostname !== base.hostname) {
+        continue;
+      }
+
+      // Skip excluded patterns
+      if (EXCLUDE_PATTERNS.some(pattern => pattern.test(url.pathname))) {
+        continue;
+      }
+
+      pages.push({
+        url: url.href,
+        source: 'sitemap',
+        priority: calculatePriority(url.href),
+      });
+    } catch {
+      // Invalid URL, skip
+    }
+  }
+
+  return pages;
+}
+
+/**
+ * Merge pages from multiple sources, deduplicating by URL
+ */
+export function mergePages(
+  ...pageSets: DiscoveredPage[][]
+): DiscoveredPage[] {
+  const seen = new Map<string, DiscoveredPage>();
+
+  for (const pages of pageSets) {
+    for (const page of pages) {
+      const existing = seen.get(page.url);
+      if (!existing || page.priority > existing.priority) {
+        seen.set(page.url, page);
+      }
+    }
+  }
+
+  return Array.from(seen.values())
+    .sort((a, b) => b.priority - a.priority);
+}

package/src/audit/discovery/repo-routes.ts

@@ -0,0 +1,347 @@
+/**
+ * Route Discovery Module
+ *
+ * Discovers routes from repository files based on framework-specific patterns.
+ * Supports: Next.js (App/Pages), Astro, React Router, Vue Router, SvelteKit
+ */
+
+export interface RouteInfo {
+  path: string;
+  type: 'static' | 'dynamic';
+  sourceFile: string;
+}
+
+type Framework = 'nextjs' | 'astro' | 'react' | 'react-vite' | 'vue' | 'sveltekit' | 'unknown';
+
+/**
+ * Detect framework from package.json and file structure
+ */
+export function detectFramework(
+  packageJson: { dependencies?: Record<string, string>; devDependencies?: Record<string, string> } | null,
+  files: string[]
+): Framework {
+  if (!packageJson) return 'unknown';
+
+  const deps = { ...packageJson.dependencies, ...packageJson.devDependencies };
+
+  // Check for Next.js
+  if (deps['next']) {
+    return 'nextjs';
+  }
+
+  // Check for Astro
+  if (deps['astro']) {
+    return 'astro';
+  }
+
+  // Check for SvelteKit
+  if (deps['@sveltejs/kit']) {
+    return 'sveltekit';
+  }
+
+  // Check for Vue with router
+  if (deps['vue-router'] || deps['@vue/router']) {
+    return 'vue';
+  }
+
+  // Check for React with Vite (our pattern)
+  if (deps['react'] && deps['vite']) {
+    return 'react-vite';
+  }
+
+  // Check for React
+  if (deps['react'] || deps['react-dom']) {
+    // Check if it has router files
+    const hasRouterFile = files.some(f =>
+      f.match(/routes?\.(tsx?|jsx?)$/) ||
+      f.includes('router') ||
+      f.endsWith('App.tsx') ||
+      f.endsWith('App.jsx')
+    );
+    if (hasRouterFile) {
+      return 'react';
+    }
+  }
+
+  return 'unknown';
+}
+
+/**
+ * Discover routes based on framework
+ */
+export async function discoverRoutesFromRepo(
+  files: string[],
+  framework: Framework,
+  fetchFile: (path: string) => Promise<string | null>
+): Promise<RouteInfo[]> {
+  switch (framework) {
+    case 'nextjs':
+      return discoverNextRoutes(files);
+    case 'astro':
+      return discoverAstroRoutes(files);
+    case 'react':
+    case 'react-vite':
+      return discoverReactRouterRoutes(files, fetchFile);
+    case 'vue':
+      return discoverVueRouterRoutes(files, fetchFile);
+    case 'sveltekit':
+      return discoverSvelteRoutes(files);
+    default:
+      return []; // Fall back to link crawling
+  }
+}
+
+/**
+ * Next.js: file-based routing (App Router and Pages Router)
+ */
+function discoverNextRoutes(files: string[]): RouteInfo[] {
+  const routes: RouteInfo[] = [];
+
+  for (const file of files) {
+    // App Router: app/**/page.tsx
+    const appMatch = file.match(/^app\/(.*)\/page\.(tsx?|jsx?)$/);
+    if (appMatch) {
+      let path = '/' + appMatch[1];
+      // Handle route groups (parentheses)
+      path = path.replace(/\/\([^)]+\)/g, '');
+      // Handle root page
+      if (path === '/') path = '/';
+
+      const isDynamic = path.includes('[');
+      routes.push({
+        path,
+        type: isDynamic ? 'dynamic' : 'static',
+        sourceFile: file
+      });
+      continue;
+    }
+
+    // App Router: app/page.tsx (root)
+    if (file.match(/^app\/page\.(tsx?|jsx?)$/)) {
+      routes.push({
+        path: '/',
+        type: 'static',
+        sourceFile: file
+      });
+      continue;
+    }
+
+    // Pages Router: pages/**/*.tsx
+    const pagesMatch = file.match(/^pages\/(.+)\.(tsx?|jsx?)$/);
+    if (pagesMatch) {
+      const pagePath = pagesMatch[1];
+
+      // Skip special files
+      if (pagePath.startsWith('_') || pagePath.startsWith('api/')) {
+        continue;
+      }
+
+      let path = '/' + pagePath;
+      // Handle index files
+      path = path.replace(/\/index$/, '');
+      if (path === '') path = '/';
+
+      const isDynamic = path.includes('[');
+      routes.push({
+        path,
+        type: isDynamic ? 'dynamic' : 'static',
+        sourceFile: file
+      });
+    }
+  }
+
+  // Deduplicate by path (prefer App Router)
+  const seen = new Map<string, RouteInfo>();
+  for (const route of routes) {
+    if (!seen.has(route.path)) {
+      seen.set(route.path, route);
+    }
+  }
+
+  return Array.from(seen.values());
+}
+
+/**
+ * Astro: file-based routing
+ */
+function discoverAstroRoutes(files: string[]): RouteInfo[] {
+  const routes: RouteInfo[] = [];
+
+  for (const file of files) {
+    // Match src/pages/**/*.astro or src/pages/**/*.md(x)
+    const match = file.match(/^src\/pages\/(.+)\.(astro|md|mdx)$/);
+    if (match) {
+      let path = '/' + match[1];
+
+      // Handle index files
+      path = path.replace(/\/index$/, '');
+      if (path === '') path = '/';
+
+      const isDynamic = path.includes('[');
+      routes.push({
+        path,
+        type: isDynamic ? 'dynamic' : 'static',
+        sourceFile: file
+      });
+    }
+  }
+
+  return routes;
+}
+
+/**
+ * SvelteKit: file-based routing
+ */
+function discoverSvelteRoutes(files: string[]): RouteInfo[] {
+  const routes: RouteInfo[] = [];
+
+  for (const file of files) {
+    // Match src/routes/**/+page.svelte
+    const match = file.match(/^src\/routes\/(.*)\/?\+page\.svelte$/);
+    if (match) {
+      let path = '/' + (match[1] || '');
+
+      // Handle route groups (parentheses)
+      path = path.replace(/\/\([^)]+\)/g, '');
+
+      // Clean up double slashes
+      path = path.replace(/\/+/g, '/');
+      if (path === '') path = '/';
+
+      const isDynamic = path.includes('[');
+      routes.push({
+        path,
+        type: isDynamic ? 'dynamic' : 'static',
+        sourceFile: file
+      });
+    }
+  }
+
+  return routes;
+}
+
+/**
+ * React Router: parse JSX/TSX for Route definitions
+ */
+async function discoverReactRouterRoutes(
+  files: string[],
+  fetchFile: (path: string) => Promise<string | null>
+): Promise<RouteInfo[]> {
+  // Find likely router files
+  const routerFiles = files.filter(f =>
+    f.match(/routes?\.(tsx?|jsx?)$/) ||
+    f.includes('router') ||
+    f.endsWith('App.tsx') ||
+    f.endsWith('App.jsx') ||
+    f.includes('Router')
+  );
+
+  const routes: RouteInfo[] = [];
+
+  for (const file of routerFiles) {
+    const content = await fetchFile(file);
+    if (!content) continue;
+
+    // Match various route definition patterns:
+    // <Route path="/about" ...
+    // <Route path='/about' ...
+    // { path: "/about" ...
+    // { path: '/about' ...
+    // path="/about"
+    const patterns = [
+      /<Route[^>]*\spath=["']([^"']+)["']/g,
+      /{\s*path:\s*["']([^"']+)["']/g,
+      /createBrowserRouter\([^)]*path:\s*["']([^"']+)["']/g,
+    ];
+
+    for (const pattern of patterns) {
+      let match;
+      while ((match = pattern.exec(content)) !== null) {
+        const path = match[1];
+
+        // Skip catch-all and layout routes
+        if (path === '*' || path.includes('*')) continue;
+
+        routes.push({
+          path,
+          type: path.includes(':') ? 'dynamic' : 'static',
+          sourceFile: file
+        });
+      }
+    }
+  }
+
+  // Deduplicate
+  const seen = new Set<string>();
+  return routes.filter(r => {
+    if (seen.has(r.path)) return false;
+    seen.add(r.path);
+    return true;
+  });
+}
+
+/**
+ * Vue Router: parse router configuration
+ */
+async function discoverVueRouterRoutes(
+  files: string[],
+  fetchFile: (path: string) => Promise<string | null>
+): Promise<RouteInfo[]> {
+  // Find router configuration files
+  const routerFiles = files.filter(f =>
+    f.match(/router\/index\.(ts|js)$/) ||
+    f.match(/router\.(ts|js)$/) ||
+    f.includes('routes')
+  );
+
+  const routes: RouteInfo[] = [];
+
+  for (const file of routerFiles) {
+    const content = await fetchFile(file);
+    if (!content) continue;
+
+    // Match route definitions: { path: '/about' ...
+    const pathRegex = /path:\s*["']([^"']+)["']/g;
+    let match;
+
+    while ((match = pathRegex.exec(content)) !== null) {
+      const path = match[1];
+
+      // Skip catch-all routes
+      if (path.includes('*') || path === '/:pathMatch(.*)') continue;
+
+      routes.push({
+        path,
+        type: path.includes(':') ? 'dynamic' : 'static',
+        sourceFile: file
+      });
+    }
+  }
+
+  // Deduplicate
+  const seen = new Set<string>();
+  return routes.filter(r => {
+    if (seen.has(r.path)) return false;
+    seen.add(r.path);
+    return true;
+  });
+}
+
+/**
+ * Convert static routes to full URLs
+ */
+export function routesToUrls(
+  routes: RouteInfo[],
+  baseUrl: string,
+  maxUrls: number = 5
+): string[] {
+  const base = new URL(baseUrl);
+
+  return routes
+    .filter(r => r.type === 'static')
+    .slice(0, maxUrls)
+    .map(r => {
+      const url = new URL(r.path, base);
+      return url.href;
+    });
+}