@crawlith/core 0.1.1 → 0.1.2

package/src/crawler/fetcher.ts

@@ -1,251 +0,0 @@
-import { request, Dispatcher } from 'undici';
-import * as net from 'net';
-import { IPGuard } from '../core/security/ipGuard.js';
-import { RateLimiter } from '../core/network/rateLimiter.js';
-import { RetryPolicy } from '../core/network/retryPolicy.js';
-import { ResponseLimiter } from '../core/network/responseLimiter.js';
-import { RedirectController } from '../core/network/redirectController.js';
-import { ProxyAdapter } from '../core/network/proxyAdapter.js';
-import { ScopeManager } from '../core/scope/scopeManager.js';
-import { version } from '../utils/version.js';
-
-export interface RedirectStep {
-  url: string;
-  status: number;
-  target: string;
-}
-
-export interface FetchResult {
-  status: number
-  | 'blocked_internal_ip'
-  | 'blocked_by_domain_filter'
-  | 'blocked_subdomain'
-  | 'oversized'
-  | 'failed_after_retries'
-  | 'network_error'
-  | 'redirect_limit_exceeded'
-  | 'redirect_loop'
-  | 'proxy_connection_failed';
-  headers: Record<string, string | string[] | undefined>;
-  body: string;
-  redirectChain: RedirectStep[];
-  etag: string | null;
-  lastModified: string | null;
-  finalUrl: string;
-  retries?: number;
-  bytesReceived?: number;
-}
-
-export interface FetchOptions {
-  etag?: string;
-  lastModified?: string;
-  rate?: number;
-  maxBytes?: number;
-  crawlDelay?: number;
-}
-
-export class Fetcher {
-  private userAgent = 'crawlith/1.0';
-  private rateLimiter: RateLimiter;
-  private proxyAdapter: ProxyAdapter;
-  private secureDispatcher: Dispatcher;
-  private scopeManager?: ScopeManager;
-  private maxRedirects: number;
-
-  constructor(options: {
-    rate?: number;
-    proxyUrl?: string;
-    scopeManager?: ScopeManager;
-    maxRedirects?: number;
-    userAgent?: string;
-  } = {}) {
-    this.rateLimiter = new RateLimiter(options.rate || 2);
-    this.proxyAdapter = new ProxyAdapter(options.proxyUrl);
-
-    if (this.proxyAdapter.dispatcher) {
-      this.secureDispatcher = this.proxyAdapter.dispatcher;
-    } else {
-      this.secureDispatcher = IPGuard.getSecureDispatcher();
-    }
-
-    this.scopeManager = options.scopeManager;
-    this.maxRedirects = Math.min(options.maxRedirects ?? 2, 11);
-    this.userAgent = options.userAgent || `crawlith/${version}`;
-  }
-
-  async fetch(url: string, options: FetchOptions = {}): Promise<FetchResult> {
-    const maxBytes = options.maxBytes || 2000000;
-    const redirectChain: RedirectStep[] = [];
-    const redirectController = new RedirectController(this.maxRedirects, url);
-
-    let currentUrl = url;
-    let totalRetries = 0;
-
-    // Use a while(true) and explicit return/continue to handle redirects
-    while (true) {
-      const urlObj = new URL(currentUrl);
-
-      // 1. SSRF Guard (IP Literals only)
-      // We only check explicit IP literals here to fail fast.
-      // For domains, we rely on the secureDispatcher (which uses IPGuard.secureLookup)
-      // to resolve and validate the IP at connection time, preventing TOCTOU attacks.
-      if (net.isIP(urlObj.hostname)) {
-        if (IPGuard.isInternal(urlObj.hostname)) {
-          return this.errorResult('blocked_internal_ip', currentUrl, redirectChain, totalRetries);
-        }
-      }
-
-      // 2. Scope Validation (Domain & Subdomain)
-      if (this.scopeManager) {
-        const eligibility = this.scopeManager.isUrlEligible(currentUrl);
-        if (eligibility !== 'allowed') {
-          return this.errorResult(eligibility, currentUrl, redirectChain, totalRetries);
-        }
-      }
-
-      // 3. Rate Limiting
-      await this.rateLimiter.waitForToken(urlObj.hostname, options.crawlDelay);
-
-      try {
-        // 4. Retry Strategy
-        const result = await RetryPolicy.execute(
-          async (attempt) => {
-            if (attempt > 0) totalRetries++;
-
-            const headers: Record<string, string> = {
-              'User-Agent': this.userAgent
-            };
-
-            // Conditional GET only for the FIRST request in a chain
-            if (redirectChain.length === 0) {
-              if (options.etag) headers['If-None-Match'] = options.etag;
-              if (options.lastModified) headers['If-Modified-Since'] = options.lastModified;
-            }
-
-            const res = await request(currentUrl, {
-              method: 'GET',
-              headers,
-              maxRedirections: 0,
-              dispatcher: this.secureDispatcher,
-              headersTimeout: 10000,
-              bodyTimeout: 10000
-            });
-
-            if (RetryPolicy.isRetryableStatus(res.statusCode)) {
-              await res.body.dump();
-              throw new Error(`Status ${res.statusCode}`);
-            }
-
-            return res;
-          },
-          (error) => RetryPolicy.isNetworkError(error) || error.message.startsWith('Status ')
-        );
-
-        const status = result.statusCode;
-        const resHeaders = result.headers;
-
-        const getHeader = (name: string): string | null => {
-          const val = resHeaders[name.toLowerCase()];
-          if (Array.isArray(val)) return val[0];
-          return (val as string) || null;
-        };
-
-        const etag = getHeader('etag');
-        const lastModified = getHeader('last-modified');
-
-        // Handle Redirects
-        if (status >= 300 && status < 400 && status !== 304) {
-          const location = getHeader('location');
-          if (location) {
-            let targetUrl: string;
-            try {
-              targetUrl = new URL(location, currentUrl).toString();
-            } catch (_e) {
-              // Bad redirect location, treat as final but maybe error?
-              const body = await ResponseLimiter.streamToString(result.body, maxBytes);
-              return { status, headers: resHeaders, body, redirectChain, etag: null, lastModified: null, finalUrl: currentUrl, retries: totalRetries };
-            }
-
-            const redirectError = redirectController.nextHop(targetUrl);
-            if (redirectError) {
-              await result.body.dump();
-              return this.errorResult(redirectError, currentUrl, redirectChain, totalRetries);
-            }
-
-            redirectChain.push({ url: currentUrl, status, target: targetUrl });
-            await result.body.dump();
-            currentUrl = targetUrl;
-            continue; // Next iteration for redirect target
-          }
-        }
-
-        // 5. Max Response Size (Streaming)
-        let bytesReceived = 0;
-        try {
-          const body = status === 304 ? '' : await ResponseLimiter.streamToString(
-            result.body,
-            maxBytes,
-            (bytes) => { bytesReceived = bytes; }
-          );
-
-          return {
-            status,
-            headers: resHeaders,
-            body,
-            redirectChain,
-            etag,
-            lastModified,
-            finalUrl: currentUrl,
-            retries: totalRetries,
-            bytesReceived
-          };
-        } catch (e: any) {
-          if (e.message === 'Oversized response') {
-            return {
-              status: 'oversized',
-              headers: resHeaders,
-              body: '',
-              redirectChain,
-              etag: null,
-              lastModified: null,
-              finalUrl: currentUrl,
-              retries: totalRetries,
-              bytesReceived
-            };
-          }
-          throw e;
-        }
-
-      } catch (error: any) {
-        // Map common network errors to specific statuses if needed
-        const isProxyError = error.message?.toLowerCase().includes('proxy') || error.code === 'ECONNREFUSED';
-
-        if (error.code === 'EBLOCKED' || error.message?.includes('Blocked internal IP')) {
-          return this.errorResult('blocked_internal_ip', currentUrl, redirectChain, totalRetries);
-        }
-
-        const finalStatus = isProxyError ? 'proxy_connection_failed' : 'network_error';
-
-        return this.errorResult(
-          totalRetries >= RetryPolicy.DEFAULT_CONFIG.maxRetries ? 'failed_after_retries' : finalStatus,
-          currentUrl,
-          redirectChain,
-          totalRetries
-        );
-      }
-    }
-  }
-
-  private errorResult(status: any, finalUrl: string, redirectChain: RedirectStep[], retries: number): FetchResult {
-    return {
-      status,
-      headers: {},
-      body: '',
-      redirectChain,
-      etag: null,
-      lastModified: null,
-      finalUrl,
-      retries
-    };
-  }
-}

package/src/crawler/metricsRunner.ts

@@ -1,137 +0,0 @@
-import { getDb } from '../db/index.js';
-import { loadGraphFromSnapshot } from '../db/graphLoader.js';
-import { MetricsRepository } from '../db/repositories/MetricsRepository.js';
-import { SnapshotRepository } from '../db/repositories/SnapshotRepository.js';
-import { PageRepository } from '../db/repositories/PageRepository.js';
-import { computePageRank } from '../graph/pagerank.js';
-import { calculateMetrics } from '../graph/metrics.js';
-import { computeHITS } from '../scoring/hits.js';
-import { EngineContext } from '../events.js';
-import { calculateHealthScore, collectCrawlIssues } from '../scoring/health.js';
-
-import { Graph } from '../graph/graph.js';
-
-export function runPostCrawlMetrics(snapshotId: number, maxDepth: number, context?: EngineContext, limitReached: boolean = false, graphInstance?: Graph) {
-    const db = getDb();
-    const metricsRepo = new MetricsRepository(db);
-    const snapshotRepo = new SnapshotRepository(db);
-    const pageRepo = new PageRepository(db);
-
-    const graph = graphInstance || loadGraphFromSnapshot(snapshotId);
-
-    // Fallback emitter
-    const emit = (event: any) => {
-        if (context) {
-            context.emit(event);
-        } else {
-            if (event.type === 'error') console.error(event.message);
-            else if (event.type !== 'debug') console.log(event.message || event.phase);
-        }
-    };
-
-    const snapshot = snapshotRepo.getSnapshot(snapshotId);
-    if (!snapshot) {
-        emit({ type: 'error', message: `Snapshot ${snapshotId} not found` });
-        return;
-    }
-
-    if (!graphInstance) {
-        emit({ type: 'metrics:start', phase: 'Loading graph' });
-    }
-
-    emit({ type: 'metrics:start', phase: 'Computing PageRank' });
-    computePageRank(graph);
-
-    emit({ type: 'metrics:start', phase: 'Computing HITS' });
-    computeHITS(graph);
-
-    emit({ type: 'metrics:start', phase: 'Updating metrics in DB' });
-    const nodes = graph.getNodes();
-
-    // Pre-fetch all page IDs to avoid N+1 queries
-    // Use getPagesIdentityBySnapshot to avoid loading full page content (HTML) into memory again
-    const pages = pageRepo.getPagesIdentityBySnapshot(snapshotId);
-    const urlToId = new Map<string, number>();
-    for (const p of pages) {
-        urlToId.set(p.normalized_url, p.id);
-    }
-
-    const clusterStmt = db.prepare(`
-        INSERT OR REPLACE INTO duplicate_clusters (id, snapshot_id, type, size, representative, severity)
-        VALUES (?, ?, ?, ?, ?, ?)
-    `);
-
-    const contentStmt = db.prepare(`
-        INSERT OR REPLACE INTO content_clusters (id, snapshot_id, count, primary_url, risk, shared_path_prefix)
-        VALUES (?, ?, ?, ?, ?, ?)
-    `);
-
-    const tx = db.transaction(() => {
-        for (const node of nodes) {
-            const pageId = urlToId.get(node.url);
-            if (!pageId) continue;
-
-
-            metricsRepo.insertMetrics({
-                snapshot_id: snapshotId,
-                page_id: pageId,
-                authority_score: node.authorityScore ?? null,
-                hub_score: node.hubScore ?? null,
-                pagerank: node.pageRank ?? null,
-                pagerank_score: node.pageRankScore ?? null,
-                link_role: node.linkRole ?? null,
-                crawl_status: node.crawlStatus ?? null,
-                word_count: node.wordCount ?? null,
-                thin_content_score: node.thinContentScore ?? null,
-                external_link_ratio: node.externalLinkRatio ?? null,
-                orphan_score: node.orphanScore ?? null,
-                duplicate_cluster_id: node.duplicateClusterId ?? null,
-                duplicate_type: node.duplicateType ?? null,
-                is_cluster_primary: node.isClusterPrimary ? 1 : 0
-            });
-
-            // Update page-level crawl trap data
-            if (node.crawlTrapFlag || node.redirectChain?.length || node.bytesReceived) {
-                pageRepo.upsertPage({
-                    site_id: snapshot.site_id,
-                    normalized_url: node.url,
-                    last_seen_snapshot_id: snapshotId,
-                    redirect_chain: node.redirectChain ? JSON.stringify(node.redirectChain) : null,
-                    bytes_received: node.bytesReceived ?? null,
-                    crawl_trap_flag: node.crawlTrapFlag ? 1 : 0,
-                    crawl_trap_risk: node.crawlTrapRisk ?? null,
-                    trap_type: node.trapType ?? null,
-                });
-            }
-        }
-
-        // Save duplicate clusters
-        for (const cluster of graph.duplicateClusters) {
-            clusterStmt.run(cluster.id, snapshotId, cluster.type, cluster.size, cluster.representative, cluster.severity);
-        }
-
-        // Save content clusters
-        for (const cluster of graph.contentClusters) {
-            contentStmt.run(cluster.id, snapshotId, cluster.count, cluster.primaryUrl, cluster.risk, cluster.sharedPathPrefix ?? null);
-        }
-    });
-    tx();
-
-    emit({ type: 'metrics:start', phase: 'Computing aggregate stats' });
-    const metrics = calculateMetrics(graph, maxDepth);
-
-    // Calculate penalty-based health score (matches CLI)
-    const issues = collectCrawlIssues(graph, metrics);
-    const health = calculateHealthScore(metrics.totalPages, issues);
-
-    snapshotRepo.updateSnapshotStatus(snapshotId, 'completed', {
-        node_count: metrics.totalPages,
-        edge_count: metrics.totalEdges,
-        health_score: health.score,
-        orphan_count: issues.orphanPages,
-        thin_content_count: issues.thinContent,
-        limit_reached: limitReached ? 1 : 0
-    });
-
-    emit({ type: 'metrics:complete', durationMs: 0 });
-}

package/src/crawler/normalize.ts

@@ -1,108 +0,0 @@
-/**
- * Normalizes a URL string based on specific rules.
- */
-export interface NormalizeOptions {
-  stripQuery?: boolean;
-}
-
-const TRACKING_PARAMS = new Set([
-  'utm_source',
-  'utm_medium',
-  'utm_campaign',
-  'utm_term',
-  'utm_content',
-  'fbclid',
-  'gclid',
-  'msclkid'
-]);
-
-const SKIP_EXTENSIONS = new Set([
-  '.pdf', '.jpg', '.png', '.svg', '.webp', '.gif',
-  '.zip', '.xml', '.json', '.mp4'
-]);
-
-export function normalizeUrl(input: string, base: string, options: NormalizeOptions = {}): string | null {
-  try {
-    // 1. Resolve absolute URL
-    let u: URL;
-    if (base) {
-      u = new URL(input, base);
-    } else {
-      u = new URL(input);
-    }
-
-    // 2. Allow only http/https
-    if (u.protocol !== 'http:' && u.protocol !== 'https:') {
-      return null;
-    }
-
-    // 3. Lowercase hostname
-    u.hostname = u.hostname.toLowerCase();
-
-    // 4. Remove default ports
-    if ((u.protocol === 'http:' && u.port === '80') || (u.protocol === 'https:' && u.port === '443')) {
-      u.port = '';
-    }
-
-    // 5. Remove hash fragments
-    u.hash = '';
-
-    // 6. Query params handling
-    const params = new URLSearchParams(u.search);
-    const newParams = new URLSearchParams();
-
-    // Check if we should strip all query params
-    if (options.stripQuery) {
-      u.search = '';
-    } else {
-      // Filter tracking params
-      let hasParams = false;
-      for (const [key, value] of params) {
-        // Remove utm_* and other tracking params
-        if (key.startsWith('utm_') || TRACKING_PARAMS.has(key)) {
-          continue;
-        }
-        newParams.append(key, value);
-        hasParams = true;
-      }
-
-      // Sort for consistency
-      newParams.sort();
-
-      if (hasParams || newParams.toString()) {
-        u.search = newParams.toString();
-      } else {
-        u.search = '';
-      }
-    }
-
-    // 7. Normalize trailing slash
-    // 8. Collapse duplicate slashes in pathname
-    let pathname = u.pathname;
-
-    // Collapse duplicate slashes
-    pathname = pathname.replace(/\/+/g, '/');
-
-    // Remove trailing slash unless root
-    if (pathname.length > 1 && pathname.endsWith('/')) {
-      pathname = pathname.slice(0, -1);
-    }
-
-    u.pathname = pathname;
-
-    // 9. Skip non-HTML assets by extension
-    const lastDotIndex = u.pathname.lastIndexOf('.');
-    if (lastDotIndex !== -1) {
-      const ext = u.pathname.slice(lastDotIndex).toLowerCase();
-      if (SKIP_EXTENSIONS.has(ext)) {
-        return null;
-      }
-    }
-
-    // 10. Return final string
-    return u.toString();
-
-  } catch (_e) {
-    return null;
-  }
-}

package/src/crawler/parser.ts

@@ -1,190 +0,0 @@
-import * as cheerio from 'cheerio';
-import crypto from 'node:crypto';
-import { normalizeUrl } from './normalize.js';
-import { SimHash } from '../graph/simhash.js';
-
-export interface ParseLink {
-  url: string;
-  weight: number;
-}
-
-export interface ParseResult {
-  links: ParseLink[];
-  html: string;
-  canonical: string | null;
-  noindex: boolean;
-  nofollow: boolean;
-  contentHash: string;
-  simhash?: string;
-  uniqueTokenRatio?: number;
-  soft404Score: number;
-  soft404Signals: string[];
-}
-
-export class Parser {
-  /**
-   * Parses HTML content to extract metadata and links.
-   */
-  parse(html: string, baseUrl: string, status: number): ParseResult {
-    const $ = cheerio.load(html);
-
-    // 1. Robots Meta
-    let noindex = false;
-    let nofollow = false;
-    const robotsMeta = $('meta[name="robots"]').attr('content');
-    if (robotsMeta) {
-      const directives = robotsMeta.toLowerCase().split(',').map(s => s.trim());
-      if (directives.includes('noindex') || directives.includes('none')) noindex = true;
-      if (directives.includes('nofollow') || directives.includes('none')) nofollow = true;
-    }
-
-    // 2. Canonical
-    let canonical: string | null = null;
-    const canonicalLink = $('link[rel="canonical"]').attr('href');
-    if (canonicalLink) {
-      try {
-        // Resolve relative canonicals
-        const u = new URL(canonicalLink, baseUrl);
-        // Normalize minimally (remove default ports, lowercase host, etc)
-        // We don't strip query by default for canonical as it might be relevant
-        canonical = normalizeUrl(u.toString(), '', { stripQuery: false });
-      } catch (_e) {
-        // Invalid canonical URL, ignore
-      }
-    }
-
-    // 3. Links
-    const links = new Map<string, number>();
-    if (!nofollow) { // Don't extract links if nofollow is set
-      $('a').each((_, element) => {
-        const href = $(element).attr('href');
-        const rel = $(element).attr('rel');
-        const isNofollow = rel && rel.toLowerCase().includes('nofollow');
-
-        if (href && !isNofollow) {
-          try {
-            const absoluteUrl = new URL(href, baseUrl);
-            if (absoluteUrl.protocol === 'http:' || absoluteUrl.protocol === 'https:') {
-              absoluteUrl.hash = '';
-              const urlStr = absoluteUrl.toString();
-
-              // Calculate Weight
-              let weight = 1.0; // Default: Body
-
-              // Semantic Check
-              const $el = $(element);
-              if ($el.closest('nav').length > 0 || $el.closest('header').length > 0) {
-                weight = 0.7;
-              } else if ($el.closest('footer').length > 0) {
-                weight = 0.4;
-              } else {
-                // Secondary check: Common attributes
-                const parentText = ($el.parent().attr('class') || '') + ($el.parent().attr('id') || '');
-                const grandParentText = ($el.parent().parent().attr('class') || '') + ($el.parent().parent().attr('id') || '');
-                const combinedContext = (parentText + grandParentText).toLowerCase();
-
-                if (combinedContext.includes('nav') || combinedContext.includes('menu')) {
-                  weight = 0.7;
-                } else if (combinedContext.includes('footer')) {
-                  weight = 0.4;
-                }
-              }
-
-              // Store highest weight if multiple links to same URL
-              const currentMax = links.get(urlStr) || 0;
-              if (weight > currentMax) {
-                links.set(urlStr, weight);
-              }
-            }
-          } catch (_e) {
-            // Invalid URL
-          }
-        }
-      });
-    }
-
-    // 4. Content Hash (ignoring script/style/comments)
-    // Clone body to avoid modifying the loaded doc (though we don't reuse it)
-    // Actually cheerio load gives us a fresh instance.
-    $('script').remove();
-    $('style').remove();
-    $('noscript').remove();
-    $('iframe').remove();
-
-    const cleanText = $('body').text().replace(/\s+/g, ' ').trim();
-    const contentHash = crypto.createHash('sha256').update(cleanText).digest('hex');
-
-    // 4b. Simhash & Token calculation (limit to 50k chars for performance)
-    const limitedText = cleanText.substring(0, 50000).toLowerCase();
-    const tokens = limitedText.split(/\W+/).filter(t => t.length > 0);
-    const uniqueTokens = new Set(tokens);
-    const uniqueTokenRatio = tokens.length > 0 ? (uniqueTokens.size / tokens.length) : 0;
-    const simhash = SimHash.generate(tokens).toString();
-
-    // 5. Soft 404 Detection
-    let soft404Score = 0;
-    const soft404Signals: string[] = [];
-
-    if (status === 200) {
-      const title = $('title').text().toLowerCase();
-      const h1Text = $('h1').first().text().toLowerCase();
-      const bodyText = cleanText.toLowerCase();
-
-      const errorPatterns = ['404', 'not found', 'error', 'doesn\'t exist', 'unavailable', 'invalid'];
-
-      // Pattern checks
-      for (const pattern of errorPatterns) {
-        if (title.includes(pattern)) {
-          soft404Score += 0.4;
-          soft404Signals.push(`title_pattern_${pattern.replace(/\s+/g, '_')}`);
-          break;
-        }
-      }
-
-      for (const pattern of errorPatterns) {
-        if (h1Text.includes(pattern)) {
-          soft404Score += 0.3;
-          soft404Signals.push(`h1_pattern_${pattern.replace(/\s+/g, '_')}`);
-          break;
-        }
-      }
-
-      if (bodyText.includes('page not found') || bodyText.includes('404 error')) {
-        soft404Score += 0.2;
-        soft404Signals.push('body_error_phrase');
-      }
-
-      // Content length check (Word count approximation)
-      const words = cleanText.split(/\s+/).filter(w => w.length > 0);
-      if (words.length < 50) {
-        soft404Score += 0.3;
-        soft404Signals.push('very_low_word_count');
-      } else if (words.length < 150) {
-        soft404Score += 0.1;
-        soft404Signals.push('low_word_count');
-      }
-
-      // Link count check
-      if (links.size === 0) {
-        soft404Score += 0.2;
-        soft404Signals.push('no_outbound_links');
-      }
-
-      // Cap at 1.0
-      soft404Score = Math.min(1.0, soft404Score);
-    }
-
-    return {
-      links: Array.from(links.entries()).map(([url, weight]) => ({ url, weight })),
-      html: html, // pass raw HTML for analysis
-      canonical,
-      noindex,
-      nofollow,
-      contentHash,
-      simhash,
-      uniqueTokenRatio,
-      soft404Score,
-      soft404Signals
-    }
-  }
-}