@crawlith/core 0.1.0 → 0.1.2

package/dist/crawler/parser.js

@@ -6,7 +6,7 @@ export class Parser {
     /**
      * Parses HTML content to extract metadata and links.
      */
-    parse(html, baseUrl, status) {
+    parse(html, baseUrl, _status) {
         const $ = cheerio.load(html);
         // 1. Robots Meta
         let noindex = false;
@@ -97,51 +97,6 @@ export class Parser {
         const uniqueTokens = new Set(tokens);
         const uniqueTokenRatio = tokens.length > 0 ? (uniqueTokens.size / tokens.length) : 0;
         const simhash = SimHash.generate(tokens).toString();
-        // 5. Soft 404 Detection
-        let soft404Score = 0;
-        const soft404Signals = [];
-        if (status === 200) {
-            const title = $('title').text().toLowerCase();
-            const h1Text = $('h1').first().text().toLowerCase();
-            const bodyText = cleanText.toLowerCase();
-            const errorPatterns = ['404', 'not found', 'error', 'doesn\'t exist', 'unavailable', 'invalid'];
-            // Pattern checks
-            for (const pattern of errorPatterns) {
-                if (title.includes(pattern)) {
-                    soft404Score += 0.4;
-                    soft404Signals.push(`title_pattern_${pattern.replace(/\s+/g, '_')}`);
-                    break;
-                }
-            }
-            for (const pattern of errorPatterns) {
-                if (h1Text.includes(pattern)) {
-                    soft404Score += 0.3;
-                    soft404Signals.push(`h1_pattern_${pattern.replace(/\s+/g, '_')}`);
-                    break;
-                }
-            }
-            if (bodyText.includes('page not found') || bodyText.includes('404 error')) {
-                soft404Score += 0.2;
-                soft404Signals.push('body_error_phrase');
-            }
-            // Content length check (Word count approximation)
-            const words = cleanText.split(/\s+/).filter(w => w.length > 0);
-            if (words.length < 50) {
-                soft404Score += 0.3;
-                soft404Signals.push('very_low_word_count');
-            }
-            else if (words.length < 150) {
-                soft404Score += 0.1;
-                soft404Signals.push('low_word_count');
-            }
-            // Link count check
-            if (links.size === 0) {
-                soft404Score += 0.2;
-                soft404Signals.push('no_outbound_links');
-            }
-            // Cap at 1.0
-            soft404Score = Math.min(1.0, soft404Score);
-        }
         return {
             links: Array.from(links.entries()).map(([url, weight]) => ({ url, weight })),
             html: html, // pass raw HTML for analysis
@@ -150,9 +105,7 @@ export class Parser {
             nofollow,
             contentHash,
             simhash,
-            uniqueTokenRatio,
-            soft404Score,
-            soft404Signals
+            uniqueTokenRatio
         };
     }
 }

package/dist/crawler/resolver.d.ts

@@ -0,0 +1,11 @@
+import { Fetcher } from './fetcher.js';
+import { Site } from '../db/repositories/SiteRepository.js';
+export interface ResolvedUrl {
+    url: string;
+    site: Site;
+}
+export declare class UrlResolver {
+    private siteRepo;
+    constructor();
+    resolve(inputUrl: string, fetcher: Fetcher): Promise<ResolvedUrl>;
+}

package/dist/crawler/resolver.js

@@ -0,0 +1,67 @@
+import { SiteRepository } from '../db/repositories/SiteRepository.js';
+import { getDb } from '../db/index.js';
+export class UrlResolver {
+    siteRepo;
+    constructor() {
+        this.siteRepo = new SiteRepository(getDb());
+    }
+    async resolve(inputUrl, fetcher) {
+        const hasProtocol = inputUrl.startsWith('http://') || inputUrl.startsWith('https://');
+        const workingUrl = hasProtocol ? inputUrl : `https://${inputUrl}`;
+        let hostname;
+        try {
+            hostname = new URL(workingUrl).hostname;
+        }
+        catch {
+            throw new Error(`Invalid URL or domain: ${inputUrl}`);
+        }
+        const domain = hostname.replace(/^www\./, '');
+        let site = this.siteRepo.firstOrCreateSite(domain);
+        // If protocol was omitted, we use our discovery logic or stored preference
+        if (!hasProtocol) {
+            if (site.ssl !== null && site.preferred_url) {
+                return {
+                    url: site.preferred_url,
+                    site
+                };
+            }
+            // No protocol provided and no stored preference: Probe HTTPS first
+            try {
+                const res = await fetcher.fetch(`https://${hostname}/`);
+                if (typeof res.status === 'number' && res.status >= 200 && res.status < 400) {
+                    const isSsl = res.finalUrl.startsWith('https:') ? 1 : 0;
+                    this.siteRepo.updateSitePreference(site.id, { preferred_url: res.finalUrl, ssl: isSsl });
+                    // Refresh site object
+                    site = this.siteRepo.getSiteById(site.id);
+                    return { url: res.finalUrl, site };
+                }
+            }
+            catch {
+                // Fallback to HTTP
+            }
+            // Try HTTP
+            try {
+                const res = await fetcher.fetch(`http://${hostname}/`);
+                if (typeof res.status === 'number' && res.status >= 200 && res.status < 400) {
+                    const isSsl = res.finalUrl.startsWith('https:') ? 1 : 0;
+                    this.siteRepo.updateSitePreference(site.id, { preferred_url: res.finalUrl, ssl: isSsl });
+                    site = this.siteRepo.getSiteById(site.id);
+                    return { url: res.finalUrl, site };
+                }
+            }
+            catch {
+                // If both fail, we still default to the provided input as https
+                return { url: workingUrl, site };
+            }
+        }
+        // Protocol was provided, we just return it but ensure site is in sync if it's the first time
+        if (site.ssl === null) {
+            this.siteRepo.updateSitePreference(site.id, {
+                preferred_url: inputUrl,
+                ssl: inputUrl.startsWith('https:') ? 1 : 0
+            });
+            site = this.siteRepo.getSiteById(site.id);
+        }
+        return { url: inputUrl, site };
+    }
+}

package/dist/crawler/sitemap.d.ts

@@ -1,4 +1,10 @@
+import { EngineContext } from '../events.js';
+import { Fetcher } from './fetcher.js';
 export declare class Sitemap {
+    private context?;
+    private fetcher?;
+    private userAgent;
+    constructor(context?: EngineContext | undefined, fetcher?: Fetcher | undefined, userAgent?: string);
     /**
      * Fetches and parses a sitemap (or sitemap index) to extract URLs.
      * Recursively handles sitemap indexes with loop detection and depth limits.

package/dist/crawler/sitemap.js

@@ -1,7 +1,19 @@
-import { request } from 'undici';
 import * as cheerio from 'cheerio';
+import pLimit from 'p-limit';
 import { normalizeUrl } from './normalize.js';
+import { DEFAULTS } from '../constants.js';
 export class Sitemap {
+    context;
+    fetcher;
+    userAgent = DEFAULTS.USER_AGENT;
+    constructor(context, fetcher, userAgent) {
+        this.context = context;
+        this.fetcher = fetcher;
+        if (userAgent)
+            this.userAgent = userAgent;
+        else if (fetcher)
+            this.userAgent = fetcher.userAgent;
+    }
     /**
      * Fetches and parses a sitemap (or sitemap index) to extract URLs.
      * Recursively handles sitemap indexes with loop detection and depth limits.
@@ -20,14 +32,16 @@ export class Sitemap {
         if (visited.size > 50)
             return;
         try {
-            const res = await request(url, {
-                maxRedirections: 3,
-                headers: { 'User-Agent': 'crawlith/1.0' },
-                headersTimeout: 10000,
-                bodyTimeout: 10000
-            });
-            if (res.statusCode >= 200 && res.statusCode < 300) {
-                const xml = await res.body.text();
+            const res = this.fetcher
+                ? await this.fetcher.fetch(url, { maxBytes: 5000000 })
+                : await (async () => {
+                    const { request } = await import('undici');
+                    const r = await request(url, { headers: { 'User-Agent': this.userAgent } });
+                    const b = await r.body.text();
+                    return { status: r.statusCode, body: b };
+                })();
+            if (typeof res.status === 'number' && res.status >= 200 && res.status < 300) {
+                const xml = res.body;
                 // Basic validation: must verify it looks like XML
                 if (!xml.trim().startsWith('<'))
                     return;
@@ -41,10 +55,9 @@ export class Sitemap {
                         if (loc)
                             childSitemaps.push(loc);
                     });
-                    // Process children sequentially to avoid massive concurrency spike
-                    for (const childUrl of childSitemaps) {
-                        await this.processSitemap(childUrl, visited, urls);
-                    }
+                    // Process children concurrently but with a limit to avoid massive concurrency spike
+                    const limit = pLimit(10);
+                    await Promise.all(childSitemaps.map(childUrl => limit(() => this.processSitemap(childUrl, visited, urls))));
                 }
                 else {
                     // It's a URL Set
@@ -59,12 +72,9 @@ export class Sitemap {
                     });
                 }
             }
-            else {
-                await res.body.dump();
-            }
         }
         catch (e) {
-            console.warn(`Failed to fetch sitemap ${url}:`, e);
+            this.context?.emit({ type: 'warn', message: `Failed to fetch sitemap ${url} (${String(e)})`, context: e });
         }
     }
 }

package/dist/crawler/trap.d.ts

@@ -16,7 +16,11 @@ export declare class TrapDetector {
     /**
      * Checks if a URL represents a potential crawl trap.
      */
-    checkTrap(rawUrl: string, _depth: number): TrapResult;
+    checkTrap(rawUrl: string, _depth: number, isInternal?: boolean): TrapResult;
+    /**
+     * Iterates over all nodes in the graph and flags potential traps.
+     */
+    analyze(graph: any): void;
     /**
      * Resets internal state (useful for multi-crawl sessions if needed)
      */

package/dist/crawler/trap.js

@@ -1,7 +1,7 @@
 export class TrapDetector {
     pathCounters = new Map();
     paginationCounters = new Map();
-    sessionParams = new Set(['sid', 'session', 'phpsessid', 'sessid', 'token']);
+    sessionParams = new Set(['sid', 'session', 'phpsessid', 'sessid', 'token', 'intended']);
     // Configurable thresholds
     PARAM_EXPLOSION_THRESHOLD = 30;
     PAGINATION_THRESHOLD = 50;
@@ -14,7 +14,12 @@ export class TrapDetector {
     /**
      * Checks if a URL represents a potential crawl trap.
      */
-    checkTrap(rawUrl, _depth) {
+    checkTrap(rawUrl, _depth, isInternal = true) {
+        // If it's not internal (e.g., social sharing links), we don't flag it as a trap
+        // that affects our crawl health, even though technically it might have many params.
+        if (!isInternal) {
+            return { risk: 0, type: null };
+        }
         let risk = 0;
         let type = null;
         try {
@@ -68,6 +73,22 @@ export class TrapDetector {
         }
         return { risk, type };
     }
+    /**
+     * Iterates over all nodes in the graph and flags potential traps.
+     */
+    analyze(graph) {
+        const nodes = graph.getNodes();
+        for (const node of nodes) {
+            if (node.status === 200 || node.status === 0) {
+                const res = this.checkTrap(node.url, node.depth || 0, !!node.isInternal);
+                if (res.risk > 0.4) {
+                    node.crawlTrapFlag = true;
+                    node.crawlTrapRisk = res.risk;
+                    node.trapType = res.type;
+                }
+            }
+        }
+    }
     /**
      * Resets internal state (useful for multi-crawl sessions if needed)
      */

package/dist/db/CrawlithDB.d.ts

@@ -0,0 +1,110 @@
+import Database from 'better-sqlite3';
+import type { CrawlithPlugin } from '../plugin-system/plugin-types.js';
+export declare class CrawlithDB {
+    private db;
+    private statements;
+    private registry;
+    /**
+     * @internal
+     * Dangerous: Returns the raw better-sqlite3 instance.
+     * Core only. Plugins must never use this.
+     */
+    unsafeGetRawDb(): Database.Database;
+    private _pluginName?;
+    private _snapshotId?;
+    /** Whether live fallback is allowed (from --live flag). Core-controlled. */
+    private _live;
+    /** Whether this plugin makes network calls. Core-controlled via plugin.storage.fetchMode. */
+    private _fetchMode;
+    constructor(dbPath: string);
+    /**
+     * Schema API
+     */
+    get schema(): {
+        define: (columns: Record<string, string>) => void;
+    };
+    /**
+     * Fluent Data API (URL-scoped rows)
+     */
+    get data(): {
+        save: <T>(input: {
+            url: string;
+            data: T;
+        }) => void;
+        find: <T>(url: string, options?: {
+            maxAge?: string | number;
+            global?: boolean;
+        }) => T | null;
+        all: <T>() => T[];
+        /**
+         * Cache-first with live fallback. Core-enforced pattern:
+         * 1. If cached data exists → return it (always, regardless of age)
+         * 2. If no cache + fetchMode='network' + live=false → return null (skip)
+         * 3. If no cache + (fetchMode='local' OR live=true) → call fetchFn, save, return
+         *
+         * Plugin authors NEVER touch ctx.live — the core injects it via scope().
+         */
+        getOrFetch: <T>(url: string, fetchFn: () => Promise<T>) => Promise<T | null>;
+    };
+    /**
+     * Report API (Global snapshot summary)
+     */
+    get report(): {
+        save: (summary: any, optionalScores?: {
+            totalScore?: number;
+            scoreCount?: number;
+            scoreWeightSum?: number;
+            scoreCalculatedAt?: string;
+        }) => void;
+        find: <T>() => T | null;
+    };
+    initialize(): void;
+    /**
+     * Create a scoped instance for a specific plugin.
+     * Also bakes in live + fetchMode so getOrFetch() can enforce the protocol
+     * without exposing those controls to the plugin author.
+     */
+    scope(pluginName: string, snapshotId?: number | string, options?: {
+        live?: boolean;
+        fetchMode?: 'local' | 'network';
+    }): CrawlithDB;
+    registerPluginDataSchema(pluginNameOrColumns: string | Record<string, string>, extraColumns?: Record<string, string>): void;
+    /** @deprecated Use registerPluginDataSchema */
+    registerPluginMigration(pluginName: string, migrationSQL: string): void;
+    getPageIdByUrl(snapshotId: number | string, url: string): number | null;
+    insertPluginReport(input: {
+        snapshotId?: number | string;
+        pluginName?: string;
+        summary: unknown;
+        totalScore?: number;
+        scoreCount?: number;
+        scoreWeightSum?: number;
+        scoreCalculatedAt?: string;
+    }): void;
+    insertPluginRow<T>(input: {
+        tableName?: string;
+        snapshotId?: number | string;
+        url: string;
+        data: T;
+    }): void;
+    getPluginReport(snapshotId?: number | string, pluginName?: string): unknown | null;
+    getPluginRows<T>(tableName?: string, snapshotId?: number | string): T[];
+    getPluginRow<T>(tableNameOrUrl: string, snapshotId?: number | string, url?: string, options?: {
+        maxAge?: string | number;
+        global?: boolean;
+    }): T | null;
+    private _parseDuration;
+    private _parseRow;
+    deleteSnapshotPlugins(snapshotId: number | string): void;
+    private _getOrFetch;
+    aggregateScoreProviders(snapshotId: number | string, plugins: CrawlithPlugin[]): void;
+    runInTransaction(fn: () => void): void;
+    private _resolveTableName;
+    /** Converts a plugin name to its canonical SQLite table name, sanitizing invalid characters. */
+    private _toTableName;
+    close(): void;
+    private _isMigrationExecuted;
+    private _assertSnapshotExists;
+    private _assertTableRegistered;
+    private _assertOwnership;
+}