@crawlith/core 0.1.1 → 0.1.2

package/dist/crawler/sitemap.d.ts

@@ -1,7 +1,10 @@
 import { EngineContext } from '../events.js';
+import { Fetcher } from './fetcher.js';
 export declare class Sitemap {
     private context?;
-    constructor(context?: EngineContext | undefined);
+    private fetcher?;
+    private userAgent;
+    constructor(context?: EngineContext | undefined, fetcher?: Fetcher | undefined, userAgent?: string);
     /**
      * Fetches and parses a sitemap (or sitemap index) to extract URLs.
      * Recursively handles sitemap indexes with loop detection and depth limits.

package/dist/crawler/sitemap.js

@@ -1,10 +1,18 @@
-import { request } from 'undici';
 import * as cheerio from 'cheerio';
+import pLimit from 'p-limit';
 import { normalizeUrl } from './normalize.js';
+import { DEFAULTS } from '../constants.js';
 export class Sitemap {
     context;
-    constructor(context) {
+    fetcher;
+    userAgent = DEFAULTS.USER_AGENT;
+    constructor(context, fetcher, userAgent) {
         this.context = context;
+        this.fetcher = fetcher;
+        if (userAgent)
+            this.userAgent = userAgent;
+        else if (fetcher)
+            this.userAgent = fetcher.userAgent;
     }
     /**
      * Fetches and parses a sitemap (or sitemap index) to extract URLs.
@@ -24,14 +32,16 @@ export class Sitemap {
         if (visited.size > 50)
             return;
         try {
-            const res = await request(url, {
-                maxRedirections: 3,
-                headers: { 'User-Agent': 'crawlith/1.0' },
-                headersTimeout: 10000,
-                bodyTimeout: 10000
-            });
-            if (res.statusCode >= 200 && res.statusCode < 300) {
-                const xml = await res.body.text();
+            const res = this.fetcher
+                ? await this.fetcher.fetch(url, { maxBytes: 5000000 })
+                : await (async () => {
+                    const { request } = await import('undici');
+                    const r = await request(url, { headers: { 'User-Agent': this.userAgent } });
+                    const b = await r.body.text();
+                    return { status: r.statusCode, body: b };
+                })();
+            if (typeof res.status === 'number' && res.status >= 200 && res.status < 300) {
+                const xml = res.body;
                 // Basic validation: must verify it looks like XML
                 if (!xml.trim().startsWith('<'))
                     return;
@@ -45,10 +55,9 @@ export class Sitemap {
                         if (loc)
                             childSitemaps.push(loc);
                     });
-                    // Process children sequentially to avoid massive concurrency spike
-                    for (const childUrl of childSitemaps) {
-                        await this.processSitemap(childUrl, visited, urls);
-                    }
+                    // Process children concurrently but with a limit to avoid massive concurrency spike
+                    const limit = pLimit(10);
+                    await Promise.all(childSitemaps.map(childUrl => limit(() => this.processSitemap(childUrl, visited, urls))));
                 }
                 else {
                     // It's a URL Set
@@ -63,12 +72,9 @@ export class Sitemap {
                     });
                 }
             }
-            else {
-                await res.body.dump();
-            }
         }
         catch (e) {
-            this.context?.emit({ type: 'warn', message: `Failed to fetch sitemap ${url}`, context: e });
+            this.context?.emit({ type: 'warn', message: `Failed to fetch sitemap ${url} (${String(e)})`, context: e });
         }
     }
 }

package/dist/crawler/trap.d.ts

@@ -16,7 +16,11 @@ export declare class TrapDetector {
     /**
      * Checks if a URL represents a potential crawl trap.
      */
-    checkTrap(rawUrl: string, _depth: number): TrapResult;
+    checkTrap(rawUrl: string, _depth: number, isInternal?: boolean): TrapResult;
+    /**
+     * Iterates over all nodes in the graph and flags potential traps.
+     */
+    analyze(graph: any): void;
     /**
      * Resets internal state (useful for multi-crawl sessions if needed)
      */

package/dist/crawler/trap.js

@@ -1,7 +1,7 @@
 export class TrapDetector {
     pathCounters = new Map();
     paginationCounters = new Map();
-    sessionParams = new Set(['sid', 'session', 'phpsessid', 'sessid', 'token']);
+    sessionParams = new Set(['sid', 'session', 'phpsessid', 'sessid', 'token', 'intended']);
     // Configurable thresholds
     PARAM_EXPLOSION_THRESHOLD = 30;
     PAGINATION_THRESHOLD = 50;
@@ -14,7 +14,12 @@ export class TrapDetector {
     /**
      * Checks if a URL represents a potential crawl trap.
      */
-    checkTrap(rawUrl, _depth) {
+    checkTrap(rawUrl, _depth, isInternal = true) {
+        // If it's not internal (e.g., social sharing links), we don't flag it as a trap
+        // that affects our crawl health, even though technically it might have many params.
+        if (!isInternal) {
+            return { risk: 0, type: null };
+        }
         let risk = 0;
         let type = null;
         try {
@@ -68,6 +73,22 @@ export class TrapDetector {
         }
         return { risk, type };
     }
+    /**
+     * Iterates over all nodes in the graph and flags potential traps.
+     */
+    analyze(graph) {
+        const nodes = graph.getNodes();
+        for (const node of nodes) {
+            if (node.status === 200 || node.status === 0) {
+                const res = this.checkTrap(node.url, node.depth || 0, !!node.isInternal);
+                if (res.risk > 0.4) {
+                    node.crawlTrapFlag = true;
+                    node.crawlTrapRisk = res.risk;
+                    node.trapType = res.type;
+                }
+            }
+        }
+    }
     /**
      * Resets internal state (useful for multi-crawl sessions if needed)
      */

package/dist/db/CrawlithDB.d.ts

@@ -0,0 +1,110 @@
+import Database from 'better-sqlite3';
+import type { CrawlithPlugin } from '../plugin-system/plugin-types.js';
+export declare class CrawlithDB {
+    private db;
+    private statements;
+    private registry;
+    /**
+     * @internal
+     * Dangerous: Returns the raw better-sqlite3 instance.
+     * Core only. Plugins must never use this.
+     */
+    unsafeGetRawDb(): Database.Database;
+    private _pluginName?;
+    private _snapshotId?;
+    /** Whether live fallback is allowed (from --live flag). Core-controlled. */
+    private _live;
+    /** Whether this plugin makes network calls. Core-controlled via plugin.storage.fetchMode. */
+    private _fetchMode;
+    constructor(dbPath: string);
+    /**
+     * Schema API
+     */
+    get schema(): {
+        define: (columns: Record<string, string>) => void;
+    };
+    /**
+     * Fluent Data API (URL-scoped rows)
+     */
+    get data(): {
+        save: <T>(input: {
+            url: string;
+            data: T;
+        }) => void;
+        find: <T>(url: string, options?: {
+            maxAge?: string | number;
+            global?: boolean;
+        }) => T | null;
+        all: <T>() => T[];
+        /**
+         * Cache-first with live fallback. Core-enforced pattern:
+         * 1. If cached data exists → return it (always, regardless of age)
+         * 2. If no cache + fetchMode='network' + live=false → return null (skip)
+         * 3. If no cache + (fetchMode='local' OR live=true) → call fetchFn, save, return
+         *
+         * Plugin authors NEVER touch ctx.live — the core injects it via scope().
+         */
+        getOrFetch: <T>(url: string, fetchFn: () => Promise<T>) => Promise<T | null>;
+    };
+    /**
+     * Report API (Global snapshot summary)
+     */
+    get report(): {
+        save: (summary: any, optionalScores?: {
+            totalScore?: number;
+            scoreCount?: number;
+            scoreWeightSum?: number;
+            scoreCalculatedAt?: string;
+        }) => void;
+        find: <T>() => T | null;
+    };
+    initialize(): void;
+    /**
+     * Create a scoped instance for a specific plugin.
+     * Also bakes in live + fetchMode so getOrFetch() can enforce the protocol
+     * without exposing those controls to the plugin author.
+     */
+    scope(pluginName: string, snapshotId?: number | string, options?: {
+        live?: boolean;
+        fetchMode?: 'local' | 'network';
+    }): CrawlithDB;
+    registerPluginDataSchema(pluginNameOrColumns: string | Record<string, string>, extraColumns?: Record<string, string>): void;
+    /** @deprecated Use registerPluginDataSchema */
+    registerPluginMigration(pluginName: string, migrationSQL: string): void;
+    getPageIdByUrl(snapshotId: number | string, url: string): number | null;
+    insertPluginReport(input: {
+        snapshotId?: number | string;
+        pluginName?: string;
+        summary: unknown;
+        totalScore?: number;
+        scoreCount?: number;
+        scoreWeightSum?: number;
+        scoreCalculatedAt?: string;
+    }): void;
+    insertPluginRow<T>(input: {
+        tableName?: string;
+        snapshotId?: number | string;
+        url: string;
+        data: T;
+    }): void;
+    getPluginReport(snapshotId?: number | string, pluginName?: string): unknown | null;
+    getPluginRows<T>(tableName?: string, snapshotId?: number | string): T[];
+    getPluginRow<T>(tableNameOrUrl: string, snapshotId?: number | string, url?: string, options?: {
+        maxAge?: string | number;
+        global?: boolean;
+    }): T | null;
+    private _parseDuration;
+    private _parseRow;
+    deleteSnapshotPlugins(snapshotId: number | string): void;
+    private _getOrFetch;
+    aggregateScoreProviders(snapshotId: number | string, plugins: CrawlithPlugin[]): void;
+    runInTransaction(fn: () => void): void;
+    private _resolveTableName;
+    /** Converts a plugin name to its canonical SQLite table name, sanitizing invalid characters. */
+    private _toTableName;
+    close(): void;
+    private _isMigrationExecuted;
+    private _assertSnapshotExists;
+    private _assertTableRegistered;
+    private _assertOwnership;
+}