@crawlith/core 0.1.0

package/src/crawler/fetcher.ts

@@ -0,0 +1,233 @@
+import { request } from 'undici';
+import { IPGuard } from '../core/security/ipGuard.js';
+import { RateLimiter } from '../core/network/rateLimiter.js';
+import { RetryPolicy } from '../core/network/retryPolicy.js';
+import { ResponseLimiter } from '../core/network/responseLimiter.js';
+import { RedirectController } from '../core/network/redirectController.js';
+import { ProxyAdapter } from '../core/network/proxyAdapter.js';
+import { ScopeManager } from '../core/scope/scopeManager.js';
+import { version } from '../utils/version.js';
+
+export interface RedirectStep {
+  url: string;
+  status: number;
+  target: string;
+}
+
+export interface FetchResult {
+  status: number
+  | 'blocked_internal_ip'
+  | 'blocked_by_domain_filter'
+  | 'blocked_subdomain'
+  | 'oversized'
+  | 'failed_after_retries'
+  | 'network_error'
+  | 'redirect_limit_exceeded'
+  | 'redirect_loop'
+  | 'proxy_connection_failed';
+  headers: Record<string, string | string[] | undefined>;
+  body: string;
+  redirectChain: RedirectStep[];
+  etag: string | null;
+  lastModified: string | null;
+  finalUrl: string;
+  retries?: number;
+  bytesReceived?: number;
+}
+
+export interface FetchOptions {
+  etag?: string;
+  lastModified?: string;
+  rate?: number;
+  maxBytes?: number;
+  crawlDelay?: number;
+}
+
+export class Fetcher {
+  private userAgent = 'crawlith/1.0';
+  private rateLimiter: RateLimiter;
+  private proxyAdapter: ProxyAdapter;
+  private scopeManager?: ScopeManager;
+  private maxRedirects: number;
+
+  constructor(options: {
+    rate?: number;
+    proxyUrl?: string;
+    scopeManager?: ScopeManager;
+    maxRedirects?: number;
+    userAgent?: string;
+  } = {}) {
+    this.rateLimiter = new RateLimiter(options.rate || 2);
+    this.proxyAdapter = new ProxyAdapter(options.proxyUrl);
+    this.scopeManager = options.scopeManager;
+    this.maxRedirects = Math.min(options.maxRedirects ?? 2, 11);
+    this.userAgent = options.userAgent || `crawlith/${version}`;
+  }
+
+  async fetch(url: string, options: FetchOptions = {}): Promise<FetchResult> {
+    const maxBytes = options.maxBytes || 2000000;
+    const redirectChain: RedirectStep[] = [];
+    const redirectController = new RedirectController(this.maxRedirects, url);
+
+    let currentUrl = url;
+    let totalRetries = 0;
+
+    // Use a while(true) and explicit return/continue to handle redirects
+    while (true) {
+      const urlObj = new URL(currentUrl);
+
+      // 1. SSRF Guard
+      const isSafe = await IPGuard.validateHost(urlObj.hostname);
+      if (!isSafe) {
+        return this.errorResult('blocked_internal_ip', currentUrl, redirectChain, totalRetries);
+      }
+
+      // 2. Scope Validation (Domain & Subdomain)
+      if (this.scopeManager) {
+        const eligibility = this.scopeManager.isUrlEligible(currentUrl);
+        if (eligibility !== 'allowed') {
+          return this.errorResult(eligibility, currentUrl, redirectChain, totalRetries);
+        }
+      }
+
+      // 3. Rate Limiting
+      await this.rateLimiter.waitForToken(urlObj.hostname, options.crawlDelay);
+
+      try {
+        // 4. Retry Strategy
+        const result = await RetryPolicy.execute(
+          async (attempt) => {
+            if (attempt > 0) totalRetries++;
+
+            const headers: Record<string, string> = {
+              'User-Agent': this.userAgent
+            };
+
+            // Conditional GET only for the FIRST request in a chain
+            if (redirectChain.length === 0) {
+              if (options.etag) headers['If-None-Match'] = options.etag;
+              if (options.lastModified) headers['If-Modified-Since'] = options.lastModified;
+            }
+
+            const res = await request(currentUrl, {
+              method: 'GET',
+              headers,
+              maxRedirections: 0,
+              dispatcher: this.proxyAdapter.dispatcher,
+              headersTimeout: 10000,
+              bodyTimeout: 10000
+            });
+
+            if (RetryPolicy.isRetryableStatus(res.statusCode)) {
+              await res.body.dump();
+              throw new Error(`Status ${res.statusCode}`);
+            }
+
+            return res;
+          },
+          (error) => RetryPolicy.isNetworkError(error) || error.message.startsWith('Status ')
+        );
+
+        const status = result.statusCode;
+        const resHeaders = result.headers;
+
+        const getHeader = (name: string): string | null => {
+          const val = resHeaders[name.toLowerCase()];
+          if (Array.isArray(val)) return val[0];
+          return (val as string) || null;
+        };
+
+        const etag = getHeader('etag');
+        const lastModified = getHeader('last-modified');
+
+        // Handle Redirects
+        if (status >= 300 && status < 400 && status !== 304) {
+          const location = getHeader('location');
+          if (location) {
+            let targetUrl: string;
+            try {
+              targetUrl = new URL(location, currentUrl).toString();
+            } catch (_e) {
+              // Bad redirect location, treat as final but maybe error?
+              const body = await ResponseLimiter.streamToString(result.body, maxBytes);
+              return { status, headers: resHeaders, body, redirectChain, etag: null, lastModified: null, finalUrl: currentUrl, retries: totalRetries };
+            }
+
+            const redirectError = redirectController.nextHop(targetUrl);
+            if (redirectError) {
+              await result.body.dump();
+              return this.errorResult(redirectError, currentUrl, redirectChain, totalRetries);
+            }
+
+            redirectChain.push({ url: currentUrl, status, target: targetUrl });
+            await result.body.dump();
+            currentUrl = targetUrl;
+            continue; // Next iteration for redirect target
+          }
+        }
+
+        // 5. Max Response Size (Streaming)
+        let bytesReceived = 0;
+        try {
+          const body = status === 304 ? '' : await ResponseLimiter.streamToString(
+            result.body,
+            maxBytes,
+            (bytes) => { bytesReceived = bytes; }
+          );
+
+          return {
+            status,
+            headers: resHeaders,
+            body,
+            redirectChain,
+            etag,
+            lastModified,
+            finalUrl: currentUrl,
+            retries: totalRetries,
+            bytesReceived
+          };
+        } catch (e: any) {
+          if (e.message === 'Oversized response') {
+            return {
+              status: 'oversized',
+              headers: resHeaders,
+              body: '',
+              redirectChain,
+              etag: null,
+              lastModified: null,
+              finalUrl: currentUrl,
+              retries: totalRetries,
+              bytesReceived
+            };
+          }
+          throw e;
+        }
+
+      } catch (error: any) {
+        // Map common network errors to specific statuses if needed
+        const isProxyError = error.message?.toLowerCase().includes('proxy') || error.code === 'ECONNREFUSED';
+        const finalStatus = isProxyError ? 'proxy_connection_failed' : 'network_error';
+
+        return this.errorResult(
+          totalRetries >= RetryPolicy.DEFAULT_CONFIG.maxRetries ? 'failed_after_retries' : finalStatus,
+          currentUrl,
+          redirectChain,
+          totalRetries
+        );
+      }
+    }
+  }
+
+  private errorResult(status: any, finalUrl: string, redirectChain: RedirectStep[], retries: number): FetchResult {
+    return {
+      status,
+      headers: {},
+      body: '',
+      redirectChain,
+      etag: null,
+      lastModified: null,
+      finalUrl,
+      retries
+    };
+  }
+}

package/src/crawler/metricsRunner.ts

@@ -0,0 +1,124 @@
+import { getDb } from '../db/index.js';
+import { loadGraphFromSnapshot } from '../db/graphLoader.js';
+import { MetricsRepository } from '../db/repositories/MetricsRepository.js';
+import { SnapshotRepository } from '../db/repositories/SnapshotRepository.js';
+import { PageRepository } from '../db/repositories/PageRepository.js';
+import { computePageRank } from '../graph/pagerank.js';
+import { calculateMetrics } from '../graph/metrics.js';
+import { computeHITS } from '../scoring/hits.js';
+
+export function runPostCrawlMetrics(snapshotId: number, maxDepth: number, limitReached: boolean = false) {
+    const db = getDb();
+    const metricsRepo = new MetricsRepository(db);
+    const snapshotRepo = new SnapshotRepository(db);
+    const pageRepo = new PageRepository(db);
+
+    const snapshot = snapshotRepo.getSnapshot(snapshotId);
+    if (!snapshot) {
+        console.error(`Snapshot ${snapshotId} not found`);
+        return;
+    }
+
+    console.log('Loading graph for metrics calculation...');
+    const graph = loadGraphFromSnapshot(snapshotId);
+
+    console.log('Computing PageRank...');
+    computePageRank(graph);
+
+    console.log('Computing HITS...');
+    computeHITS(graph);
+
+    console.log('Updating metrics in DB...');
+    const nodes = graph.getNodes();
+
+    const tx = db.transaction(() => {
+        for (const node of nodes) {
+            const pageId = pageRepo.getIdByUrl(snapshot.site_id, node.url);
+            if (!pageId) continue;
+
+            const existing = metricsRepo.getMetricsForPage(snapshotId, pageId);
+
+            metricsRepo.insertMetrics({
+                snapshot_id: snapshotId,
+                page_id: pageId,
+                authority_score: node.authorityScore ?? null,
+                hub_score: node.hubScore ?? null,
+                pagerank: node.pageRank ?? null,
+                pagerank_score: node.pageRankScore ?? null,
+                link_role: node.linkRole ?? null,
+                crawl_status: existing?.crawl_status ?? null,
+                word_count: existing?.word_count ?? null,
+                thin_content_score: existing?.thin_content_score ?? null,
+                external_link_ratio: existing?.external_link_ratio ?? null,
+                orphan_score: existing?.orphan_score ?? null,
+                duplicate_cluster_id: node.duplicateClusterId ?? null,
+                duplicate_type: node.duplicateType ?? null,
+                is_cluster_primary: node.isClusterPrimary ? 1 : 0
+            });
+
+            // Update page-level crawl trap data
+            if (node.crawlTrapFlag || node.redirectChain?.length || node.bytesReceived) {
+                pageRepo.upsertPage({
+                    site_id: snapshot.site_id,
+                    normalized_url: node.url,
+                    last_seen_snapshot_id: snapshotId,
+                    redirect_chain: node.redirectChain ? JSON.stringify(node.redirectChain) : null,
+                    bytes_received: node.bytesReceived ?? null,
+                    crawl_trap_flag: node.crawlTrapFlag ? 1 : 0,
+                    crawl_trap_risk: node.crawlTrapRisk ?? null,
+                    trap_type: node.trapType ?? null,
+                });
+            }
+        }
+
+        // Save duplicate clusters
+        if (graph.duplicateClusters.length > 0) {
+            const clusterStmt = db.prepare(`
+                INSERT OR REPLACE INTO duplicate_clusters (id, snapshot_id, type, size, representative, severity)
+                VALUES (?, ?, ?, ?, ?, ?)
+            `);
+            for (const cluster of graph.duplicateClusters) {
+                clusterStmt.run(cluster.id, snapshotId, cluster.type, cluster.size, cluster.representative, cluster.severity);
+            }
+        }
+
+        // Save content clusters
+        if (graph.contentClusters.length > 0) {
+            const contentStmt = db.prepare(`
+                INSERT OR REPLACE INTO content_clusters (id, snapshot_id, count, primary_url, risk, shared_path_prefix)
+                VALUES (?, ?, ?, ?, ?, ?)
+            `);
+            for (const cluster of graph.contentClusters) {
+                contentStmt.run(cluster.id, snapshotId, cluster.count, cluster.primaryUrl, cluster.risk, cluster.sharedPathPrefix ?? null);
+            }
+        }
+    });
+    tx();
+
+    console.log('Computing aggregate stats...');
+    const metrics = calculateMetrics(graph, maxDepth);
+
+    let totalScore = 0;
+    let totalWeight = 0;
+    for (const node of nodes) {
+        const score = node.authorityScore || node.pageRankScore || 0;
+        const depth = node.depth;
+        const weight = 1 / (depth + 1);
+        totalScore += score * weight;
+        totalWeight += weight;
+    }
+    const healthScore = totalWeight > 0 ? (totalScore / totalWeight) * 100 : 0;
+
+    const thinCountRow = db.prepare('SELECT count(*) as count FROM metrics WHERE snapshot_id = ? AND thin_content_score >= 70').get(snapshotId) as { count: number };
+
+    snapshotRepo.updateSnapshotStatus(snapshotId, 'completed', {
+        node_count: metrics.totalPages,
+        edge_count: metrics.totalEdges,
+        health_score: healthScore,
+        orphan_count: metrics.orphanPages.length,
+        thin_content_count: thinCountRow.count,
+        limit_reached: limitReached ? 1 : 0
+    });
+
+    console.log('Metrics calculation complete.');
+}

package/src/crawler/normalize.ts

@@ -0,0 +1,108 @@
+/**
+ * Normalizes a URL string based on specific rules.
+ */
+export interface NormalizeOptions {
+  stripQuery?: boolean;
+}
+
+const TRACKING_PARAMS = new Set([
+  'utm_source',
+  'utm_medium',
+  'utm_campaign',
+  'utm_term',
+  'utm_content',
+  'fbclid',
+  'gclid',
+  'msclkid'
+]);
+
+const SKIP_EXTENSIONS = new Set([
+  '.pdf', '.jpg', '.png', '.svg', '.webp', '.gif',
+  '.zip', '.xml', '.json', '.mp4'
+]);
+
+export function normalizeUrl(input: string, base: string, options: NormalizeOptions = {}): string | null {
+  try {
+    // 1. Resolve absolute URL
+    let u: URL;
+    if (base) {
+      u = new URL(input, base);
+    } else {
+      u = new URL(input);
+    }
+
+    // 2. Allow only http/https
+    if (u.protocol !== 'http:' && u.protocol !== 'https:') {
+      return null;
+    }
+
+    // 3. Lowercase hostname
+    u.hostname = u.hostname.toLowerCase();
+
+    // 4. Remove default ports
+    if ((u.protocol === 'http:' && u.port === '80') || (u.protocol === 'https:' && u.port === '443')) {
+      u.port = '';
+    }
+
+    // 5. Remove hash fragments
+    u.hash = '';
+
+    // 6. Query params handling
+    const params = new URLSearchParams(u.search);
+    const newParams = new URLSearchParams();
+
+    // Check if we should strip all query params
+    if (options.stripQuery) {
+      u.search = '';
+    } else {
+      // Filter tracking params
+      let hasParams = false;
+      for (const [key, value] of params) {
+        // Remove utm_* and other tracking params
+        if (key.startsWith('utm_') || TRACKING_PARAMS.has(key)) {
+          continue;
+        }
+        newParams.append(key, value);
+        hasParams = true;
+      }
+
+      // Sort for consistency
+      newParams.sort();
+
+      if (hasParams || newParams.toString()) {
+        u.search = newParams.toString();
+      } else {
+        u.search = '';
+      }
+    }
+
+    // 7. Normalize trailing slash
+    // 8. Collapse duplicate slashes in pathname
+    let pathname = u.pathname;
+
+    // Collapse duplicate slashes
+    pathname = pathname.replace(/\/+/g, '/');
+
+    // Remove trailing slash unless root
+    if (pathname.length > 1 && pathname.endsWith('/')) {
+      pathname = pathname.slice(0, -1);
+    }
+
+    u.pathname = pathname;
+
+    // 9. Skip non-HTML assets by extension
+    const lastDotIndex = u.pathname.lastIndexOf('.');
+    if (lastDotIndex !== -1) {
+      const ext = u.pathname.slice(lastDotIndex).toLowerCase();
+      if (SKIP_EXTENSIONS.has(ext)) {
+        return null;
+      }
+    }
+
+    // 10. Return final string
+    return u.toString();
+
+  } catch (_e) {
+    return null;
+  }
+}

package/src/crawler/parser.ts

@@ -0,0 +1,190 @@
+import * as cheerio from 'cheerio';
+import crypto from 'node:crypto';
+import { normalizeUrl } from './normalize.js';
+import { SimHash } from '../graph/simhash.js';
+
+export interface ParseLink {
+  url: string;
+  weight: number;
+}
+
+export interface ParseResult {
+  links: ParseLink[];
+  html: string;
+  canonical: string | null;
+  noindex: boolean;
+  nofollow: boolean;
+  contentHash: string;
+  simhash?: string;
+  uniqueTokenRatio?: number;
+  soft404Score: number;
+  soft404Signals: string[];
+}
+
+export class Parser {
+  /**
+   * Parses HTML content to extract metadata and links.
+   */
+  parse(html: string, baseUrl: string, status: number): ParseResult {
+    const $ = cheerio.load(html);
+
+    // 1. Robots Meta
+    let noindex = false;
+    let nofollow = false;
+    const robotsMeta = $('meta[name="robots"]').attr('content');
+    if (robotsMeta) {
+      const directives = robotsMeta.toLowerCase().split(',').map(s => s.trim());
+      if (directives.includes('noindex') || directives.includes('none')) noindex = true;
+      if (directives.includes('nofollow') || directives.includes('none')) nofollow = true;
+    }
+
+    // 2. Canonical
+    let canonical: string | null = null;
+    const canonicalLink = $('link[rel="canonical"]').attr('href');
+    if (canonicalLink) {
+      try {
+        // Resolve relative canonicals
+        const u = new URL(canonicalLink, baseUrl);
+        // Normalize minimally (remove default ports, lowercase host, etc)
+        // We don't strip query by default for canonical as it might be relevant
+        canonical = normalizeUrl(u.toString(), '', { stripQuery: false });
+      } catch (_e) {
+        // Invalid canonical URL, ignore
+      }
+    }
+
+    // 3. Links
+    const links = new Map<string, number>();
+    if (!nofollow) { // Don't extract links if nofollow is set
+      $('a').each((_, element) => {
+        const href = $(element).attr('href');
+        const rel = $(element).attr('rel');
+        const isNofollow = rel && rel.toLowerCase().includes('nofollow');
+
+        if (href && !isNofollow) {
+          try {
+            const absoluteUrl = new URL(href, baseUrl);
+            if (absoluteUrl.protocol === 'http:' || absoluteUrl.protocol === 'https:') {
+              absoluteUrl.hash = '';
+              const urlStr = absoluteUrl.toString();
+
+              // Calculate Weight
+              let weight = 1.0; // Default: Body
+
+              // Semantic Check
+              const $el = $(element);
+              if ($el.closest('nav').length > 0 || $el.closest('header').length > 0) {
+                weight = 0.7;
+              } else if ($el.closest('footer').length > 0) {
+                weight = 0.4;
+              } else {
+                // Secondary check: Common attributes
+                const parentText = ($el.parent().attr('class') || '') + ($el.parent().attr('id') || '');
+                const grandParentText = ($el.parent().parent().attr('class') || '') + ($el.parent().parent().attr('id') || '');
+                const combinedContext = (parentText + grandParentText).toLowerCase();
+
+                if (combinedContext.includes('nav') || combinedContext.includes('menu')) {
+                  weight = 0.7;
+                } else if (combinedContext.includes('footer')) {
+                  weight = 0.4;
+                }
+              }
+
+              // Store highest weight if multiple links to same URL
+              const currentMax = links.get(urlStr) || 0;
+              if (weight > currentMax) {
+                links.set(urlStr, weight);
+              }
+            }
+          } catch (_e) {
+            // Invalid URL
+          }
+        }
+      });
+    }
+
+    // 4. Content Hash (ignoring script/style/comments)
+    // Clone body to avoid modifying the loaded doc (though we don't reuse it)
+    // Actually cheerio load gives us a fresh instance.
+    $('script').remove();
+    $('style').remove();
+    $('noscript').remove();
+    $('iframe').remove();
+
+    const cleanText = $('body').text().replace(/\s+/g, ' ').trim();
+    const contentHash = crypto.createHash('sha256').update(cleanText).digest('hex');
+
+    // 4b. Simhash & Token calculation (limit to 50k chars for performance)
+    const limitedText = cleanText.substring(0, 50000).toLowerCase();
+    const tokens = limitedText.split(/\W+/).filter(t => t.length > 0);
+    const uniqueTokens = new Set(tokens);
+    const uniqueTokenRatio = tokens.length > 0 ? (uniqueTokens.size / tokens.length) : 0;
+    const simhash = SimHash.generate(tokens).toString();
+
+    // 5. Soft 404 Detection
+    let soft404Score = 0;
+    const soft404Signals: string[] = [];
+
+    if (status === 200) {
+      const title = $('title').text().toLowerCase();
+      const h1Text = $('h1').first().text().toLowerCase();
+      const bodyText = cleanText.toLowerCase();
+
+      const errorPatterns = ['404', 'not found', 'error', 'doesn\'t exist', 'unavailable', 'invalid'];
+
+      // Pattern checks
+      for (const pattern of errorPatterns) {
+        if (title.includes(pattern)) {
+          soft404Score += 0.4;
+          soft404Signals.push(`title_pattern_${pattern.replace(/\s+/g, '_')}`);
+          break;
+        }
+      }
+
+      for (const pattern of errorPatterns) {
+        if (h1Text.includes(pattern)) {
+          soft404Score += 0.3;
+          soft404Signals.push(`h1_pattern_${pattern.replace(/\s+/g, '_')}`);
+          break;
+        }
+      }
+
+      if (bodyText.includes('page not found') || bodyText.includes('404 error')) {
+        soft404Score += 0.2;
+        soft404Signals.push('body_error_phrase');
+      }
+
+      // Content length check (Word count approximation)
+      const words = cleanText.split(/\s+/).filter(w => w.length > 0);
+      if (words.length < 50) {
+        soft404Score += 0.3;
+        soft404Signals.push('very_low_word_count');
+      } else if (words.length < 150) {
+        soft404Score += 0.1;
+        soft404Signals.push('low_word_count');
+      }
+
+      // Link count check
+      if (links.size === 0) {
+        soft404Score += 0.2;
+        soft404Signals.push('no_outbound_links');
+      }
+
+      // Cap at 1.0
+      soft404Score = Math.min(1.0, soft404Score);
+    }
+
+    return {
+      links: Array.from(links.entries()).map(([url, weight]) => ({ url, weight })),
+      html: html, // pass raw HTML for analysis
+      canonical,
+      noindex,
+      nofollow,
+      contentHash,
+      simhash,
+      uniqueTokenRatio,
+      soft404Score,
+      soft404Signals
+    }
+  }
+}