@crawlith/core 0.1.1 → 0.1.2

package/src/crawler/sitemap.ts

@@ -1,76 +0,0 @@
-import { request } from 'undici';
-import * as cheerio from 'cheerio';
-import { normalizeUrl } from './normalize.js';
-import { EngineContext } from '../events.js';
-
-export class Sitemap {
-  constructor(private context?: EngineContext) {}
-
-  /**
-   * Fetches and parses a sitemap (or sitemap index) to extract URLs.
-   * Recursively handles sitemap indexes with loop detection and depth limits.
-   */
-  async fetch(url: string): Promise<string[]> {
-    const visited = new Set<string>();
-    const urls = new Set<string>();
-
-    await this.processSitemap(url, visited, urls);
-
-    return Array.from(urls);
-  }
-
-  private async processSitemap(url: string, visited: Set<string>, urls: Set<string>) {
-    if (visited.has(url)) return;
-    visited.add(url);
-
-    // Hard limit on number of sitemaps to fetch to prevent abuse
-    if (visited.size > 50) return;
-
-    try {
-      const res = await request(url, {
-        maxRedirections: 3,
-        headers: { 'User-Agent': 'crawlith/1.0' },
-        headersTimeout: 10000,
-        bodyTimeout: 10000
-      });
-
-      if (res.statusCode >= 200 && res.statusCode < 300) {
-        const xml = await res.body.text();
-        // Basic validation: must verify it looks like XML
-        if (!xml.trim().startsWith('<')) return;
-
-        const $ = cheerio.load(xml, { xmlMode: true });
-
-        // Check if it's a sitemap index
-        const sitemaps = $('sitemap > loc');
-        if (sitemaps.length > 0) {
-          const childSitemaps: string[] = [];
-          sitemaps.each((_, el) => {
-            const loc = $(el).text().trim();
-            if (loc) childSitemaps.push(loc);
-          });
-
-          // Process children sequentially to avoid massive concurrency spike
-          for (const childUrl of childSitemaps) {
-            await this.processSitemap(childUrl, visited, urls);
-          }
-        } else {
-          // It's a URL Set
-          $('url > loc').each((_, el) => {
-            const loc = $(el).text().trim();
-            if (loc) {
-              const normalized = normalizeUrl(loc, '');
-              if (normalized) {
-                urls.add(normalized);
-              }
-            }
-          });
-        }
-      } else {
-        await res.body.dump();
-      }
-    } catch (e) {
-      this.context?.emit({ type: 'warn', message: `Failed to fetch sitemap ${url}`, context: e });
-    }
-  }
-}

package/src/crawler/trap.ts

@@ -1,96 +0,0 @@
-
-export type TrapType = 'faceted_navigation' | 'calendar_trap' | 'pagination_loop' | 'session_trap';
-
-export interface TrapResult {
-    risk: number;
-    type: TrapType | null;
-}
-
-export class TrapDetector {
-    private pathCounters = new Map<string, Set<string>>();
-    private paginationCounters = new Map<string, number>();
-    private sessionParams = new Set(['sid', 'session', 'phpsessid', 'sessid', 'token']);
-
-    // Configurable thresholds
-    private PARAM_EXPLOSION_THRESHOLD = 30;
-    private PAGINATION_THRESHOLD = 50;
-
-    constructor(options: { paramThreshold?: number, paginationThreshold?: number } = {}) {
-        if (options.paramThreshold) this.PARAM_EXPLOSION_THRESHOLD = options.paramThreshold;
-        if (options.paginationThreshold) this.PAGINATION_THRESHOLD = options.paginationThreshold;
-    }
-
-    /**
-     * Checks if a URL represents a potential crawl trap.
-     */
-    checkTrap(rawUrl: string, _depth: number): TrapResult {
-        let risk = 0;
-        let type: TrapType | null = null;
-
-        try {
-            const u = new URL(rawUrl);
-            const params = new URLSearchParams(u.search);
-            const pathname = u.pathname;
-            const pathKey = `${u.origin}${pathname}`;
-
-            // 1. Session IDs / Tracking Parameters
-            for (const [key] of params) {
-                if (this.sessionParams.has(key.toLowerCase()) || key.toLowerCase().includes('session')) {
-                    risk = Math.max(risk, 0.9);
-                    type = 'session_trap';
-                }
-            }
-
-            // 2. Calendar Pattern Detection
-            // Matches /2023/12/01, /2023-12-01, /12-2023 etc
-            const calendarRegex = /\/\d{4}[-/]\d{2}[-/]\d{2}\/|\/\d{2}[-/]\d{2}[-/]\d{4}\//;
-            if (calendarRegex.test(pathname)) {
-                risk = Math.max(risk, 0.7);
-                type = 'calendar_trap';
-            }
-
-            // 3. Pagination Loop
-            const pageParam = params.get('page') || params.get('p') || params.get('pg');
-            if (pageParam && /^\d+$/.test(pageParam)) {
-                const pageNum = parseInt(pageParam, 10);
-                const currentMaxPage = this.paginationCounters.get(pathKey) || 0;
-
-                if (pageNum > currentMaxPage) {
-                    this.paginationCounters.set(pathKey, pageNum);
-                }
-
-                if (pageNum > this.PAGINATION_THRESHOLD) {
-                    risk = Math.max(risk, 0.85);
-                    type = 'pagination_loop';
-                }
-            }
-
-            // 4. Infinite Parameter Explosion (Faceted Navigation)
-            if (params.size > 0) {
-                const paramSet = this.pathCounters.get(pathKey) || new Set<string>();
-                params.sort();
-                const paramKey = params.toString();
-                paramSet.add(paramKey);
-                this.pathCounters.set(pathKey, paramSet);
-
-                if (paramSet.size > this.PARAM_EXPLOSION_THRESHOLD) {
-                    risk = Math.max(risk, 0.95);
-                    if (!type) type = 'faceted_navigation';
-                }
-            }
-
-        } catch (_e) {
-            // Invalid URL
-        }
-
-        return { risk, type };
-    }
-
-    /**
-     * Resets internal state (useful for multi-crawl sessions if needed)
-     */
-    reset() {
-        this.pathCounters.clear();
-        this.paginationCounters.clear();
-    }
-}

package/src/db/graphLoader.ts

@@ -1,135 +0,0 @@
-import { getDb } from './index.js';
-import { PageRepository } from './repositories/PageRepository.js';
-import { EdgeRepository } from './repositories/EdgeRepository.js';
-import { MetricsRepository, DbMetrics } from './repositories/MetricsRepository.js';
-import { SnapshotRepository } from './repositories/SnapshotRepository.js';
-import { Graph } from '../graph/graph.js';
-
-export function loadGraphFromSnapshot(snapshotId: number): Graph {
-    const db = getDb();
-    const pageRepo = new PageRepository(db);
-    const edgeRepo = new EdgeRepository(db);
-    const metricsRepo = new MetricsRepository(db);
-    const snapshotRepo = new SnapshotRepository(db);
-
-    const pages = pageRepo.getPagesIteratorBySnapshot(snapshotId);
-    const metrics = metricsRepo.getMetricsIterator(snapshotId);
-    const snapshot = snapshotRepo.getSnapshot(snapshotId);
-    const metricsMap = new Map<number, DbMetrics>();
-    for (const m of metrics) {
-        metricsMap.set(m.page_id, m);
-    }
-
-    const graph = new Graph();
-    let pagesFetched = 0;
-    let pagesCached = 0;
-    let pagesSkipped = 0;
-
-    if (snapshot) {
-        graph.limitReached = !!snapshot.limit_reached;
-    }
-    const idMap = new Map<number, string>();
-
-    for (const p of pages) {
-        idMap.set(p.id, p.normalized_url);
-        graph.addNode(p.normalized_url, p.depth, p.http_status || 0);
-
-        const m = metricsMap.get(p.id);
-        if (m) {
-            const isProcessed = m.crawl_status === 'fetched' ||
-                m.crawl_status === 'fetched_error' ||
-                m.crawl_status === 'network_error' ||
-                m.crawl_status === 'failed_after_retries' ||
-                m.crawl_status === 'blocked_by_robots';
-
-            if (isProcessed) pagesFetched++;
-            else if (m.crawl_status === 'cached') pagesCached++;
-            else if (m.crawl_status === 'skipped') pagesSkipped++;
-        }
-
-        let incrementalStatus: 'new' | 'changed' | 'unchanged' | undefined;
-        if (p.first_seen_snapshot_id === snapshotId) {
-            incrementalStatus = 'new';
-        } else if (m?.crawl_status === 'cached') {
-            incrementalStatus = 'unchanged';
-        } else if (m?.crawl_status === 'fetched') {
-            incrementalStatus = 'changed';
-        }
-
-        graph.updateNodeData(p.normalized_url, {
-            canonical: p.canonical_url || undefined,
-            contentHash: p.content_hash || undefined,
-            simhash: p.simhash || undefined,
-            etag: p.etag || undefined,
-            lastModified: p.last_modified || undefined,
-            html: p.html || undefined,
-            soft404Score: p.soft404_score || undefined,
-            noindex: !!p.noindex,
-            nofollow: !!p.nofollow,
-            incrementalStatus,
-            securityError: p.security_error || undefined,
-            retries: p.retries || undefined,
-            bytesReceived: p.bytes_received || undefined,
-            redirectChain: p.redirect_chain ? JSON.parse(p.redirect_chain) : undefined,
-            crawlTrapFlag: !!p.crawl_trap_flag,
-            crawlTrapRisk: p.crawl_trap_risk || undefined,
-            trapType: p.trap_type || undefined,
-            // Metrics
-            pageRank: m?.pagerank ?? undefined,
-            pageRankScore: m?.pagerank_score ?? m?.pagerank ?? undefined,
-            authorityScore: m?.authority_score ?? undefined,
-            hubScore: m?.hub_score ?? undefined,
-            linkRole: m?.link_role ?? undefined,
-            // Duplicate info
-            duplicateClusterId: m?.duplicate_cluster_id ?? undefined,
-            duplicateType: m?.duplicate_type ?? undefined,
-            isClusterPrimary: m?.is_cluster_primary ? true : undefined,
-            // Additional metrics
-            crawlStatus: m?.crawl_status || undefined,
-            wordCount: m?.word_count != null ? m.word_count : undefined,
-            thinContentScore: m?.thin_content_score != null ? m.thin_content_score : undefined,
-            externalLinkRatio: m?.external_link_ratio != null ? m.external_link_ratio : undefined,
-            orphanScore: m?.orphan_score != null ? m.orphan_score : undefined,
-        });
-    }
-
-    const edges = edgeRepo.getEdgesIteratorBySnapshot(snapshotId);
-
-    for (const e of edges) {
-        const source = idMap.get(e.source_page_id);
-        const target = idMap.get(e.target_page_id);
-        if (source && target) {
-            graph.addEdge(source, target, e.weight || 1.0);
-        }
-    }
-
-    // Load duplicate clusters
-    const dupClusters = db.prepare('SELECT * FROM duplicate_clusters WHERE snapshot_id = ?').all(snapshotId) as any[];
-    graph.duplicateClusters = dupClusters.map(c => ({
-        id: c.id,
-        type: c.type,
-        size: c.size,
-        representative: c.representative,
-        severity: c.severity
-    }));
-
-    // Load content clusters
-    const contentClusters = db.prepare('SELECT * FROM content_clusters WHERE snapshot_id = ?').all(snapshotId) as any[];
-    graph.contentClusters = contentClusters.map(c => ({
-        id: c.id,
-        count: c.count,
-        primaryUrl: c.primary_url,
-        risk: c.risk,
-        sharedPathPrefix: c.shared_path_prefix || undefined
-    }));
-
-    // Set session stats
-    graph.sessionStats = {
-        pagesFetched,
-        pagesCached,
-        pagesSkipped,
-        totalFound: idMap.size
-    };
-
-    return graph;
-}

package/src/db/index.ts

@@ -1,75 +0,0 @@
-import Database from 'better-sqlite3';
-import path from 'node:path';
-import fs from 'node:fs';
-import os from 'node:os';
-import { initSchema } from './schema.js';
-
-let dbInstance: Database.Database | null = null;
-
-export * from './repositories/SiteRepository.js';
-export * from './repositories/SnapshotRepository.js';
-export { initSchema } from './schema.js';
-
-export function getDbPath(): string {
-  if (process.env.NODE_ENV === 'test') {
-    return ':memory:';
-  }
-  if (process.env.CRAWLITH_DB_PATH) {
-    return process.env.CRAWLITH_DB_PATH;
-  }
-  const homeDir = os.homedir();
-  const crawlithDir = path.join(homeDir, '.crawlith');
-  if (!fs.existsSync(crawlithDir)) {
-    fs.mkdirSync(crawlithDir, { recursive: true });
-    // Set permissions to 700 (user only)
-    fs.chmodSync(crawlithDir, 0o700);
-  }
-  return path.join(crawlithDir, 'crawlith.db');
-}
-
-export function getDb(): Database.Database {
-  if (dbInstance) {
-    return dbInstance;
-  }
-
-  const dbPath = getDbPath();
-  const db = new Database(dbPath);
-
-  // Hardening & Performance Configuration
-  db.pragma('journal_mode = WAL');
-  db.pragma('synchronous = NORMAL');
-  db.pragma('foreign_keys = ON');
-  db.pragma('temp_store = MEMORY');
-  db.pragma('mmap_size = 30000000000');
-  db.pragma('cache_size = -20000');
-  db.pragma('busy_timeout = 5000');
-
-  // Security controls
-  // Ensure file permissions are 600 (user read/write only)
-  try {
-    fs.chmodSync(dbPath, 0o600);
-  } catch (_e) {
-    // might fail on first creation if file doesn't exist yet, but better-sqlite3 creates it
-    // so we can try again or ignore if it's new
-  }
-
-  // Integrity check on startup
-  const integrity = db.pragma('integrity_check', { simple: true });
-  if (integrity !== 'ok') {
-    // Reverted to console.warn to avoid breaking change
-    console.warn('Database integrity check failed:', integrity);
-  }
-
-  // Initialize schema
-  initSchema(db);
-
-  dbInstance = db;
-  return db;
-}
-
-export function closeDb() {
-  if (dbInstance) {
-    dbInstance.close();
-    dbInstance = null;
-  }
-}

package/src/db/repositories/EdgeRepository.ts

@@ -1,43 +0,0 @@
-import { Database } from 'better-sqlite3';
-
-export interface Edge {
-  id: number;
-  snapshot_id: number;
-  source_page_id: number;
-  target_page_id: number;
-  weight: number;
-  rel: 'nofollow' | 'sponsored' | 'ugc' | 'internal' | 'external' | 'unknown';
-}
-
-export class EdgeRepository {
-  private insertStmt;
-
-  constructor(private db: Database) {
-    this.insertStmt = this.db.prepare(`
-      INSERT INTO edges (snapshot_id, source_page_id, target_page_id, weight, rel)
-      VALUES (?, ?, ?, ?, ?)
-    `);
-  }
-
-  insertEdge(snapshotId: number, sourcePageId: number, targetPageId: number, weight: number = 1.0, rel: string = 'internal') {
-    this.insertStmt.run(snapshotId, sourcePageId, targetPageId, weight, rel);
-  }
-
-  insertEdges(edges: { snapshot_id: number; source_page_id: number; target_page_id: number; weight: number; rel: string }[]) {
-    if (edges.length === 0) return;
-    const tx = this.db.transaction((edgesBatch) => {
-      for (const edge of edgesBatch) {
-        this.insertStmt.run(edge.snapshot_id, edge.source_page_id, edge.target_page_id, edge.weight, edge.rel);
-      }
-    });
-    tx(edges);
-  }
-
-  getEdgesBySnapshot(snapshotId: number): Edge[] {
-    return this.db.prepare('SELECT * FROM edges WHERE snapshot_id = ?').all(snapshotId) as Edge[];
-  }
-
-  getEdgesIteratorBySnapshot(snapshotId: number): IterableIterator<Edge> {
-    return this.db.prepare('SELECT * FROM edges WHERE snapshot_id = ?').iterate(snapshotId) as IterableIterator<Edge>;
-  }
-}

package/src/db/repositories/MetricsRepository.ts

@@ -1,63 +0,0 @@
-import { Database } from 'better-sqlite3';
-
-export interface DbMetrics {
-  snapshot_id: number;
-  page_id: number;
-  authority_score: number | null;
-  hub_score: number | null;
-  pagerank: number | null;
-  pagerank_score: number | null;
-  link_role: 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral' | null;
-  crawl_status: string | null;
-  word_count: number | null;
-  thin_content_score: number | null;
-  external_link_ratio: number | null;
-  orphan_score: number | null;
-  duplicate_cluster_id: string | null;
-  duplicate_type: 'exact' | 'near' | 'template_heavy' | 'none' | null;
-  is_cluster_primary: number;
-}
-
-export class MetricsRepository {
-  private insertStmt;
-  private getByPageStmt;
-
-  constructor(private db: Database) {
-    this.getByPageStmt = this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ? AND page_id = ?');
-    this.insertStmt = this.db.prepare(`
-      INSERT OR REPLACE INTO metrics (
-        snapshot_id, page_id, authority_score, hub_score, pagerank, pagerank_score,
-        link_role, crawl_status, word_count, thin_content_score, external_link_ratio,
-        orphan_score, duplicate_cluster_id, duplicate_type, is_cluster_primary
-      ) VALUES (
-        @snapshot_id, @page_id, @authority_score, @hub_score, @pagerank, @pagerank_score,
-        @link_role, @crawl_status, @word_count, @thin_content_score, @external_link_ratio,
-        @orphan_score, @duplicate_cluster_id, @duplicate_type, @is_cluster_primary
-      )
-    `);
-  }
-
-  insertMetrics(metrics: DbMetrics) {
-    this.insertStmt.run(metrics);
-  }
-
-  getMetrics(snapshotId: number): DbMetrics[] {
-    return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ?').all(snapshotId) as DbMetrics[];
-  }
-
-  getMetricsIterator(snapshotId: number): IterableIterator<DbMetrics> {
-    return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ?').iterate(snapshotId) as IterableIterator<DbMetrics>;
-  }
-
-  getMetricsForPage(snapshotId: number, pageId: number): DbMetrics | undefined {
-    return this.getByPageStmt.get(snapshotId, pageId) as DbMetrics | undefined;
-  }
-
-  insertMany(metricsList: DbMetrics[]) {
-    const insert = this.insertStmt;
-    const tx = this.db.transaction((items: DbMetrics[]) => {
-      for (const item of items) insert.run(item);
-    });
-    tx(metricsList);
-  }
-}