@crawlith/core 0.1.0 → 0.1.1

package/dist/core/security/ipGuard.js

@@ -1,6 +1,7 @@
 import * as dns from 'dns';
 import * as net from 'net';
 import { promisify } from 'util';
+import { Agent } from 'undici';
 const resolve4 = promisify(dns.resolve4);
 const resolve6 = promisify(dns.resolve6);
 export class IPGuard {
@@ -43,6 +44,14 @@ export class IPGuard {
             // fe80::/10 (Link Local)
             if ((firstWord & 0xffc0) === 0xfe80)
                 return true;
+            // IPv4-mapped IPv6: ::ffff:0:0/96
+            if (expanded.startsWith('0000:0000:0000:0000:0000:ffff:')) {
+                const parts = expanded.split(':');
+                const p7 = parseInt(parts[6], 16);
+                const p8 = parseInt(parts[7], 16);
+                const ip4 = `${(p7 >> 8) & 255}.${p7 & 255}.${(p8 >> 8) & 255}.${p8 & 255}`;
+                return IPGuard.isInternal(ip4);
+            }
             return false;
         }
         return true; // Unknown format, block it for safety
@@ -67,12 +76,71 @@ export class IPGuard {
             return false;
         }
     }
+    /**
+     * Custom lookup function for undici that validates the resolved IP.
+     * Prevents DNS Rebinding attacks by checking the IP immediately before connection.
+     */
+    static secureLookup(hostname, options, callback) {
+        dns.lookup(hostname, options, (err, address, family) => {
+            if (err) {
+                return callback(err, address, family);
+            }
+            const checkIP = (ip) => {
+                if (IPGuard.isInternal(ip)) {
+                    return new Error(`Blocked internal IP: ${ip}`);
+                }
+                return null;
+            };
+            if (typeof address === 'string') {
+                const error = checkIP(address);
+                if (error) {
+                    // Return a custom error that undici will propagate
+                    const blockedError = new Error(`Blocked internal IP: ${address}`);
+                    blockedError.code = 'EBLOCKED';
+                    return callback(blockedError, address, family);
+                }
+            }
+            else if (Array.isArray(address)) {
+                // Handle array of addresses (if options.all is true)
+                for (const addr of address) {
+                    const error = checkIP(addr.address);
+                    if (error) {
+                        const blockedError = new Error(`Blocked internal IP: ${addr.address}`);
+                        blockedError.code = 'EBLOCKED';
+                        return callback(blockedError, address, family);
+                    }
+                }
+            }
+            callback(null, address, family);
+        });
+    }
+    /**
+     * Returns an undici Agent configured with secure DNS lookup.
+     */
+    static getSecureDispatcher() {
+        return new Agent({
+            connect: {
+                lookup: IPGuard.secureLookup
+            }
+        });
+    }
     static expandIPv6(ip) {
         if (ip === '::')
             return '0000:0000:0000:0000:0000:0000:0000:0000';
-        let full = ip;
-        if (ip.includes('::')) {
-            const parts = ip.split('::');
+        let normalizedIp = ip;
+        if (ip.includes('.')) {
+            const lastColonIndex = ip.lastIndexOf(':');
+            const lastPart = ip.substring(lastColonIndex + 1);
+            if (net.isIPv4(lastPart)) {
+                const parts = lastPart.split('.').map(Number);
+                const hex1 = ((parts[0] << 8) | parts[1]).toString(16);
+                const hex2 = ((parts[2] << 8) | parts[3]).toString(16);
+                normalizedIp = ip.substring(0, lastColonIndex + 1) + hex1 + ':' + hex2;
+            }
+        }
+        let full = normalizedIp;
+        if (normalizedIp.includes('::')) {
+            const parts = normalizedIp.split('::');
             const left = parts[0].split(':').filter(x => x !== '');
             const right = parts[1].split(':').filter(x => x !== '');
             const missing = 8 - (left.length + right.length);

package/dist/crawler/crawl.d.ts

@@ -1,22 +1,4 @@
-import { Graph } from '../graph/graph.js';
-export interface CrawlOptions {
-    limit: number;
-    depth: number;
-    concurrency?: number;
-    ignoreRobots?: boolean;
-    stripQuery?: boolean;
-    previousGraph?: Graph;
-    sitemap?: string;
-    debug?: boolean;
-    detectSoft404?: boolean;
-    detectTraps?: boolean;
-    rate?: number;
-    maxBytes?: number;
-    allowedDomains?: string[];
-    deniedDomains?: string[];
-    includeSubdomains?: boolean;
-    proxyUrl?: string;
-    maxRedirects?: number;
-    userAgent?: string;
-}
-export declare function crawl(startUrl: string, options: CrawlOptions): Promise<number>;
+import { CrawlOptions } from './crawler.js';
+import { EngineContext } from '../events.js';
+export { CrawlOptions };
+export declare function crawl(startUrl: string, options: CrawlOptions, context?: EngineContext): Promise<number>;

package/dist/crawler/crawl.js

@@ -1,336 +1,5 @@
-import { request } from 'undici';
-import pLimit from 'p-limit';
-import chalk from 'chalk';
-import robotsParser from 'robots-parser';
-import { Fetcher } from './fetcher.js';
-import { Parser } from './parser.js';
-import { Sitemap } from './sitemap.js';
-import { normalizeUrl } from './normalize.js';
-import { TrapDetector } from './trap.js';
-import { ScopeManager } from '../core/scope/scopeManager.js';
-import { getDb } from '../db/index.js';
-import { SiteRepository } from '../db/repositories/SiteRepository.js';
-import { SnapshotRepository } from '../db/repositories/SnapshotRepository.js';
-import { PageRepository } from '../db/repositories/PageRepository.js';
-import { EdgeRepository } from '../db/repositories/EdgeRepository.js';
-import { MetricsRepository } from '../db/repositories/MetricsRepository.js';
-import { analyzeContent, calculateThinContentScore } from '../analysis/content.js';
-import { analyzeLinks } from '../analysis/links.js';
-export async function crawl(startUrl, options) {
-    const visited = new Set();
-    const concurrency = Math.min(options.concurrency || 2, 10);
-    const limitConcurrency = pLimit(concurrency);
-    const trapDetector = new TrapDetector();
-    const db = getDb();
-    const siteRepo = new SiteRepository(db);
-    const snapshotRepo = new SnapshotRepository(db);
-    const pageRepo = new PageRepository(db);
-    const edgeRepo = new EdgeRepository(db);
-    const metricsRepo = new MetricsRepository(db);
-    const rootUrl = normalizeUrl(startUrl, '', { stripQuery: options.stripQuery });
-    if (!rootUrl)
-        throw new Error('Invalid start URL');
-    const urlObj = new URL(rootUrl);
-    const domain = urlObj.hostname.replace('www.', '');
-    const site = siteRepo.firstOrCreateSite(domain);
-    const siteId = site.id;
-    const snapshotId = snapshotRepo.createSnapshot(siteId, options.previousGraph ? 'incremental' : 'full');
-    const rootOrigin = urlObj.origin;
-    // DB Helper
-    const savePageToDb = (url, depth, status, data = {}) => {
-        try {
-            const existing = pageRepo.getPage(siteId, url);
-            const isSameSnapshot = existing?.last_seen_snapshot_id === snapshotId;
-            return pageRepo.upsertAndGetId({
-                site_id: siteId,
-                normalized_url: url,
-                depth: isSameSnapshot ? existing.depth : depth,
-                http_status: status,
-                first_seen_snapshot_id: existing ? existing.first_seen_snapshot_id : snapshotId,
-                last_seen_snapshot_id: snapshotId,
-                canonical_url: data.canonical !== undefined ? data.canonical : existing?.canonical_url,
-                content_hash: data.contentHash !== undefined ? data.contentHash : existing?.content_hash,
-                simhash: data.simhash !== undefined ? data.simhash : existing?.simhash,
-                etag: data.etag !== undefined ? data.etag : existing?.etag,
-                last_modified: data.lastModified !== undefined ? data.lastModified : existing?.last_modified,
-                html: data.html !== undefined ? data.html : existing?.html,
-                soft404_score: data.soft404Score !== undefined ? data.soft404Score : existing?.soft404_score,
-                noindex: data.noindex !== undefined ? (data.noindex ? 1 : 0) : existing?.noindex,
-                nofollow: data.nofollow !== undefined ? (data.nofollow ? 1 : 0) : existing?.nofollow,
-                security_error: data.securityError !== undefined ? data.securityError : existing?.security_error,
-                retries: data.retries !== undefined ? data.retries : existing?.retries
-            });
-        }
-        catch (e) {
-            console.error(`Failed to save page ${url}:`, e);
-            return null;
-        }
-    };
-    const saveEdgeToDb = (sourceUrl, targetUrl, weight = 1.0, rel = 'internal') => {
-        try {
-            const sourceId = pageRepo.getIdByUrl(siteId, sourceUrl);
-            const targetId = pageRepo.getIdByUrl(siteId, targetUrl);
-            if (sourceId && targetId) {
-                edgeRepo.insertEdge(snapshotId, sourceId, targetId, weight, rel);
-            }
-        }
-        catch (e) {
-            console.error(`Failed to save edge ${sourceUrl} -> ${targetUrl}:`, e);
-        }
-    };
-    // Initialize Modules
-    const scopeManager = new ScopeManager({
-        allowedDomains: options.allowedDomains || [],
-        deniedDomains: options.deniedDomains || [],
-        includeSubdomains: options.includeSubdomains || false,
-        rootUrl: startUrl
-    });
-    const fetcher = new Fetcher({
-        rate: options.rate,
-        proxyUrl: options.proxyUrl,
-        scopeManager,
-        maxRedirects: options.maxRedirects,
-        userAgent: options.userAgent
-    });
-    const parser = new Parser();
-    const sitemapFetcher = new Sitemap();
-    // Handle robots.txt
-    let robots = null;
-    if (!options.ignoreRobots) {
-        try {
-            const robotsUrl = new URL('/robots.txt', rootOrigin).toString();
-            const res = await request(robotsUrl, {
-                maxRedirections: 3,
-                headers: { 'User-Agent': 'crawlith/1.0' },
-                headersTimeout: 5000,
-                bodyTimeout: 5000
-            });
-            if (res.statusCode >= 200 && res.statusCode < 300) {
-                const txt = await res.body.text();
-                robots = robotsParser(robotsUrl, txt);
-            }
-            else {
-                await res.body.dump();
-            }
-        }
-        catch {
-            console.warn('Failed to fetch robots.txt, proceeding...');
-        }
-    }
-    // Queue Setup
-    const queue = [];
-    const uniqueQueue = new Set();
-    const addToQueue = (u, d) => {
-        if (scopeManager.isUrlEligible(u) !== 'allowed')
-            return;
-        if (!uniqueQueue.has(u)) {
-            uniqueQueue.add(u);
-            queue.push({ url: u, depth: d });
-        }
-    };
-    // Seed from Sitemap
-    if (options.sitemap) {
-        try {
-            const sitemapUrl = options.sitemap === 'true' ? new URL('/sitemap.xml', rootOrigin).toString() : options.sitemap;
-            if (sitemapUrl.startsWith('http')) {
-                console.log(`Fetching sitemap: ${sitemapUrl}`);
-                const sitemapUrls = await sitemapFetcher.fetch(sitemapUrl);
-                for (const u of sitemapUrls) {
-                    const normalized = normalizeUrl(u, '', options);
-                    if (normalized)
-                        addToQueue(normalized, 0);
-                }
-            }
-        }
-        catch (e) {
-            console.warn('Sitemap fetch failed', e);
-        }
-    }
-    // Seed from startUrl
-    addToQueue(rootUrl, 0);
-    let pagesCrawled = 0;
-    let active = 0;
-    let reachedLimit = false;
-    const maxDepthInCrawl = Math.min(options.depth, 10);
-    const shouldEnqueue = (url, depth) => {
-        if (visited.has(url))
-            return false;
-        if (uniqueQueue.has(url))
-            return false;
-        if (depth > maxDepthInCrawl)
-            return false;
-        if (scopeManager.isUrlEligible(url) !== 'allowed')
-            return false;
-        if (options.detectTraps) {
-            const trap = trapDetector.checkTrap(url, depth);
-            if (trap.risk > 0.8)
-                return false;
-        }
-        return true;
-    };
-    return new Promise((resolve) => {
-        const checkDone = () => {
-            if (queue.length === 0 && active === 0) {
-                snapshotRepo.updateSnapshotStatus(snapshotId, 'completed', {
-                    limit_reached: reachedLimit ? 1 : 0
-                });
-                resolve(snapshotId);
-                return true;
-            }
-            return false;
-        };
-        const next = () => {
-            if (checkDone())
-                return;
-            if (pagesCrawled >= options.limit) {
-                reachedLimit = true;
-                if (active === 0) {
-                    snapshotRepo.updateSnapshotStatus(snapshotId, 'completed', {
-                        limit_reached: 1
-                    });
-                    resolve(snapshotId);
-                }
-                return;
-            }
-            while (queue.length > 0 && active < concurrency && pagesCrawled < options.limit) {
-                const item = queue.shift();
-                if (visited.has(item.url))
-                    continue;
-                if (robots && !robots.isAllowed(item.url, 'crawlith'))
-                    continue;
-                active++;
-                pagesCrawled++;
-                visited.add(item.url);
-                limitConcurrency(() => processPage(item)).finally(() => {
-                    active--;
-                    next();
-                });
-            }
-        };
-        const processPage = async (item) => {
-            const { url, depth } = item;
-            if (scopeManager.isUrlEligible(url) !== 'allowed') {
-                savePageToDb(url, depth, 0, { securityError: 'blocked_by_domain_filter' });
-                return;
-            }
-            const existingInDb = pageRepo.getPage(siteId, url);
-            savePageToDb(url, depth, 0);
-            try {
-                const res = await fetcher.fetch(url, {
-                    etag: existingInDb?.etag || undefined,
-                    lastModified: existingInDb?.last_modified || undefined,
-                    maxBytes: options.maxBytes,
-                    crawlDelay: robots ? robots.getCrawlDelay('crawlith') : undefined
-                });
-                if (options.debug) {
-                    console.log(`${chalk.gray(`[D:${depth}]`)} ${res.status} ${chalk.blue(url)}`);
-                }
-                if (res.status === 304) {
-                    savePageToDb(url, depth, 304);
-                    metricsRepo.insertMetrics({
-                        snapshot_id: snapshotId,
-                        page_id: existingInDb.id,
-                        authority_score: null,
-                        hub_score: null,
-                        pagerank: null,
-                        pagerank_score: null,
-                        link_role: null,
-                        crawl_status: 'cached',
-                        word_count: null,
-                        thin_content_score: null,
-                        external_link_ratio: null,
-                        orphan_score: null,
-                        duplicate_cluster_id: null,
-                        duplicate_type: null,
-                        is_cluster_primary: 0
-                    });
-                    return;
-                }
-                const chain = res.redirectChain;
-                for (const step of chain) {
-                    const source = normalizeUrl(step.url, '', options);
-                    const target = normalizeUrl(step.target, '', options);
-                    if (source && target) {
-                        savePageToDb(source, depth, step.status);
-                        savePageToDb(target, depth, 0);
-                        saveEdgeToDb(source, target);
-                    }
-                }
-                const finalUrl = normalizeUrl(res.finalUrl, '', options);
-                if (!finalUrl)
-                    return;
-                const isStringStatus = typeof res.status === 'string';
-                if (isStringStatus || (typeof res.status === 'number' && res.status >= 300)) {
-                    savePageToDb(finalUrl, depth, typeof res.status === 'number' ? res.status : 0, {
-                        securityError: isStringStatus ? res.status : undefined,
-                        retries: res.retries
-                    });
-                    return;
-                }
-                if (res.status === 200) {
-                    const contentTypeHeader = res.headers['content-type'];
-                    const contentType = Array.isArray(contentTypeHeader) ? contentTypeHeader[0] : (contentTypeHeader || '');
-                    if (!contentType || !contentType.toLowerCase().includes('text/html')) {
-                        savePageToDb(finalUrl, depth, res.status);
-                        return;
-                    }
-                    savePageToDb(finalUrl, depth, res.status);
-                    const parseResult = parser.parse(res.body, finalUrl, res.status);
-                    const pageId = savePageToDb(finalUrl, depth, res.status, {
-                        html: parseResult.html,
-                        canonical: parseResult.canonical || undefined,
-                        noindex: parseResult.noindex,
-                        nofollow: parseResult.nofollow,
-                        contentHash: parseResult.contentHash,
-                        simhash: parseResult.simhash,
-                        soft404Score: parseResult.soft404Score,
-                        etag: res.etag,
-                        lastModified: res.lastModified,
-                        retries: res.retries
-                    });
-                    if (pageId) {
-                        try {
-                            const contentAnalysis = analyzeContent(parseResult.html);
-                            const linkAnalysis = analyzeLinks(parseResult.html, finalUrl, rootOrigin);
-                            const thinScore = calculateThinContentScore(contentAnalysis, 0);
-                            metricsRepo.insertMetrics({
-                                snapshot_id: snapshotId,
-                                page_id: pageId,
-                                authority_score: null,
-                                hub_score: null,
-                                pagerank: null,
-                                pagerank_score: null,
-                                link_role: null,
-                                crawl_status: 'fetched',
-                                word_count: contentAnalysis.wordCount,
-                                thin_content_score: thinScore,
-                                external_link_ratio: linkAnalysis.externalRatio,
-                                orphan_score: null,
-                                duplicate_cluster_id: null,
-                                duplicate_type: null,
-                                is_cluster_primary: 0
-                            });
-                        }
-                        catch (e) {
-                            console.error(`Error calculating per-page metrics for ${finalUrl}:`, e);
-                        }
-                    }
-                    for (const linkItem of parseResult.links) {
-                        const normalizedLink = normalizeUrl(linkItem.url, '', options);
-                        if (normalizedLink && normalizedLink !== finalUrl) {
-                            savePageToDb(normalizedLink, depth + 1, 0);
-                            saveEdgeToDb(finalUrl, normalizedLink, 1.0, 'internal');
-                            if (shouldEnqueue(normalizedLink, depth + 1)) {
-                                addToQueue(normalizedLink, depth + 1);
-                            }
-                        }
-                    }
-                }
-            }
-            catch (e) {
-                console.error(`Error processing ${url}:`, e);
-            }
-        };
-        next();
-    });
+import { Crawler } from './crawler.js';
+export async function crawl(startUrl, options, context) {
+    const crawler = new Crawler(startUrl, options, context);
+    return crawler.run();
 }

package/dist/crawler/crawler.d.ts

@@ -0,0 +1,75 @@
+import { Graph } from '../graph/graph.js';
+import { EngineContext } from '../events.js';
+export interface CrawlOptions {
+    limit: number;
+    depth: number;
+    concurrency?: number;
+    ignoreRobots?: boolean;
+    stripQuery?: boolean;
+    previousGraph?: Graph;
+    sitemap?: string;
+    debug?: boolean;
+    detectSoft404?: boolean;
+    detectTraps?: boolean;
+    rate?: number;
+    maxBytes?: number;
+    allowedDomains?: string[];
+    deniedDomains?: string[];
+    includeSubdomains?: boolean;
+    proxyUrl?: string;
+    maxRedirects?: number;
+    userAgent?: string;
+    snapshotType?: 'full' | 'partial' | 'incremental';
+}
+export declare class Crawler {
+    private startUrl;
+    private options;
+    private context;
+    private visited;
+    private uniqueQueue;
+    private queue;
+    private active;
+    private pagesCrawled;
+    private reachedLimit;
+    private maxDepthInCrawl;
+    private concurrency;
+    private limitConcurrency;
+    private siteRepo;
+    private snapshotRepo;
+    private pageRepo;
+    private edgeRepo;
+    private metricsRepo;
+    private siteId;
+    private snapshotId;
+    private rootOrigin;
+    private discoveryDepths;
+    private pageBuffer;
+    private edgeBuffer;
+    private metricsBuffer;
+    private scopeManager;
+    private fetcher;
+    private parser;
+    private sitemapFetcher;
+    private trapDetector;
+    private robots;
+    constructor(startUrl: string, options: CrawlOptions, context?: EngineContext);
+    initialize(): Promise<void>;
+    setupModules(): void;
+    fetchRobots(): Promise<void>;
+    shouldEnqueue(url: string, depth: number): boolean;
+    addToQueue(u: string, d: number): void;
+    seedQueue(): Promise<void>;
+    private bufferPage;
+    private flushPages;
+    private bufferEdge;
+    private flushEdges;
+    private bufferMetrics;
+    private flushMetrics;
+    flushAll(): Promise<void>;
+    private fetchPage;
+    private handleCachedResponse;
+    private handleRedirects;
+    private handleSuccessResponse;
+    private processPage;
+    run(): Promise<number>;
+}