@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/crawl-checkpoint.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Crawl checkpoint system for resume capability.
+ * Saves progress to a JSON file so interrupted crawls can continue.
+ */
+export interface CrawlCheckpoint {
+    /** Unique crawl job ID (hash of start URL + options) */
+    jobId: string;
+    /** Starting URL */
+    startUrl: string;
+    /** URLs already crawled (with their results) */
+    completed: Map<string, {
+        status: number;
+        contentLength: number;
+        timestamp: number;
+    }>;
+    /** URLs queued but not yet crawled */
+    pending: string[];
+    /** URLs discovered but not yet queued */
+    discovered: string[];
+    /** Crawl options (serialized) */
+    options: Record<string, any>;
+    /** When crawl started */
+    startedAt: number;
+    /** Last checkpoint time */
+    lastCheckpoint: number;
+    /** Total pages target */
+    maxPages: number;
+}
+/**
+ * Generate a deterministic job ID from URL + options.
+ */
+export declare function generateJobId(url: string, options?: Record<string, any>): string;
+/**
+ * Save a checkpoint to disk.
+ */
+export declare function saveCheckpoint(checkpoint: CrawlCheckpoint): void;
+/**
+ * Load a checkpoint from disk.
+ */
+export declare function loadCheckpoint(jobId: string): CrawlCheckpoint | null;
+/**
+ * Delete a checkpoint (crawl completed or abandoned).
+ */
+export declare function deleteCheckpoint(jobId: string): void;
+/**
+ * List all active checkpoints.
+ */
+export declare function listCheckpoints(): Array<{
+    jobId: string;
+    startUrl: string;
+    completed: number;
+    pending: number;
+    lastCheckpoint: number;
+}>;

package/dist/core/crawl-checkpoint.js

@@ -0,0 +1,104 @@
+/**
+ * Crawl checkpoint system for resume capability.
+ * Saves progress to a JSON file so interrupted crawls can continue.
+ */
+import { existsSync, readFileSync, writeFileSync, mkdirSync, unlinkSync, readdirSync } from 'fs';
+import { join } from 'path';
+import { createHash } from 'crypto';
+const CHECKPOINT_DIR = join(process.env.HOME || '/tmp', '.webpeel', 'checkpoints');
+/**
+ * Generate a deterministic job ID from URL + options.
+ */
+export function generateJobId(url, options = {}) {
+    const key = JSON.stringify({
+        url,
+        maxPages: options.maxPages,
+        maxDepth: options.maxDepth,
+        includes: options.includes,
+        excludes: options.excludes,
+    });
+    return createHash('sha256').update(key).digest('hex').slice(0, 16);
+}
+/**
+ * Get the checkpoint file path for a job.
+ */
+function getCheckpointPath(jobId) {
+    return join(CHECKPOINT_DIR, `${jobId}.json`);
+}
+/**
+ * Save a checkpoint to disk.
+ */
+export function saveCheckpoint(checkpoint) {
+    try {
+        mkdirSync(CHECKPOINT_DIR, { recursive: true });
+        const data = {
+            ...checkpoint,
+            completed: Object.fromEntries(checkpoint.completed),
+            lastCheckpoint: Date.now(),
+        };
+        writeFileSync(getCheckpointPath(checkpoint.jobId), JSON.stringify(data, null, 2));
+    }
+    catch (e) {
+        if (process.env.DEBUG) {
+            console.debug('[webpeel]', 'Failed to save checkpoint:', e instanceof Error ? e.message : e);
+        }
+    }
+}
+/**
+ * Load a checkpoint from disk.
+ */
+export function loadCheckpoint(jobId) {
+    const path = getCheckpointPath(jobId);
+    if (!existsSync(path))
+        return null;
+    try {
+        const raw = JSON.parse(readFileSync(path, 'utf-8'));
+        return {
+            ...raw,
+            completed: new Map(Object.entries(raw.completed || {})),
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Delete a checkpoint (crawl completed or abandoned).
+ */
+export function deleteCheckpoint(jobId) {
+    const path = getCheckpointPath(jobId);
+    try {
+        if (existsSync(path)) {
+            unlinkSync(path);
+        }
+    }
+    catch { /* ignore */ }
+}
+/**
+ * List all active checkpoints.
+ */
+export function listCheckpoints() {
+    try {
+        if (!existsSync(CHECKPOINT_DIR))
+            return [];
+        const files = readdirSync(CHECKPOINT_DIR).filter((f) => f.endsWith('.json'));
+        return files.map(f => {
+            try {
+                const raw = JSON.parse(readFileSync(join(CHECKPOINT_DIR, f), 'utf-8'));
+                return {
+                    jobId: raw.jobId,
+                    startUrl: raw.startUrl,
+                    completed: Object.keys(raw.completed || {}).length,
+                    pending: (raw.pending || []).length,
+                    lastCheckpoint: raw.lastCheckpoint,
+                };
+            }
+            catch {
+                return null;
+            }
+        }).filter(Boolean);
+    }
+    catch {
+        return [];
+    }
+}

package/dist/core/crawler.d.ts

@@ -0,0 +1,84 @@
+/**
+ * Web crawler functionality
+ * Crawls a starting URL and follows links matching specified patterns
+ */
+import type { PeelOptions } from '../types.js';
+export interface CrawlOptions extends Omit<PeelOptions, 'format'> {
+    /** Maximum number of pages to crawl (default: 10, max: tier-dependent) */
+    maxPages?: number;
+    /** Tier for determining the max pages cap (default: 'free') */
+    tier?: string;
+    /** Maximum depth to crawl (default: 2, max: 5) */
+    maxDepth?: number;
+    /** Only crawl URLs from these domains (default: same domain as starting URL) */
+    allowedDomains?: string[];
+    /** Exclude URLs matching these patterns (regex strings) */
+    excludePatterns?: string[];
+    /** Respect robots.txt (default: true) */
+    respectRobotsTxt?: boolean;
+    /** Rate limit between requests in milliseconds (default: 1000ms = 1 req/sec) */
+    rateLimitMs?: number;
+    /** Try sitemap.xml first to discover URLs (default: false) */
+    sitemapFirst?: boolean;
+    /** Crawl strategy: breadth-first or depth-first (default: 'bfs') */
+    strategy?: 'bfs' | 'dfs';
+    /** Skip duplicate content using fingerprinting (default: true) */
+    deduplication?: boolean;
+    /** Only crawl URLs matching these regex patterns */
+    includePatterns?: string[];
+    /** Progress callback called after each page */
+    onProgress?: (status: CrawlProgress) => void;
+    /** Per-page callback — receives the full result as soon as a page completes */
+    onPage?: (result: CrawlResult) => void;
+    /** Resume an interrupted crawl from its last checkpoint */
+    resume?: boolean;
+}
+export interface CrawlProgress {
+    crawled: number;
+    queued: number;
+    failed: number;
+    currentUrl: string;
+    elapsed: number;
+}
+export interface CrawlResult {
+    /** URL of the crawled page */
+    url: string;
+    /** Page title */
+    title: string;
+    /** Markdown content */
+    markdown: string;
+    /** Token count for this page's content */
+    tokens: number;
+    /** All links found on this page (absolute URLs) */
+    links: string[];
+    /** Depth level (0 = starting URL) */
+    depth: number;
+    /** Parent URL that linked to this page (null for starting URL) */
+    parent: string | null;
+    /** Time elapsed fetching this page (ms) */
+    elapsed: number;
+    /** Error message if page failed to fetch */
+    error?: string;
+    /** Content fingerprint for deduplication */
+    fingerprint?: string;
+}
+/**
+ * Crawl a website starting from a URL
+ *
+ * @param startUrl - Starting URL to crawl from
+ * @param options - Crawl options
+ * @returns Array of crawl results
+ *
+ * @example
+ * ```typescript
+ * import { crawl } from 'webpeel';
+ *
+ * const results = await crawl('https://example.com', {
+ *   maxPages: 20,
+ *   maxDepth: 2,
+ * });
+ *
+ * console.log(`Crawled ${results.length} pages`);
+ * ```
+ */
+export declare function crawl(startUrl: string, options?: CrawlOptions): Promise<CrawlResult[]>;

package/dist/core/crawler.js

@@ -0,0 +1,349 @@
+/**
+ * Web crawler functionality
+ * Crawls a starting URL and follows links matching specified patterns
+ */
+import { peel } from '../index.js';
+import { fetch as undiciFetch } from 'undici';
+import { createHash } from 'crypto';
+import { discoverSitemap } from './sitemap.js';
+import { generateJobId, loadCheckpoint, saveCheckpoint, deleteCheckpoint, } from './crawl-checkpoint.js';
+import { createLogger } from './logger.js';
+const log = createLogger('crawler');
+/** Safely compile a user-supplied regex pattern. Rejects patterns longer than 200 chars
+ *  and wraps compilation in a try-catch to prevent invalid regex crashes. */
+function safeRegex(pattern) {
+    if (pattern.length > 200) {
+        throw new Error(`Regex pattern too long (${pattern.length} chars, max 200)`);
+    }
+    try {
+        return new RegExp(pattern);
+    }
+    catch {
+        throw new Error(`Invalid regex pattern: ${pattern}`);
+    }
+}
+/** Maximum pages allowed per tier */
+const TIER_MAX_PAGES = {
+    free: 10,
+    starter: 25,
+    pro: 50,
+    enterprise: 100,
+    max: 100,
+    admin: 10000,
+};
+/**
+ * Parse robots.txt and return disallowed paths for User-agent: *
+ */
+async function fetchRobotsTxt(domain) {
+    const robotsUrl = `https://${domain}/robots.txt`;
+    try {
+        const response = await undiciFetch(robotsUrl, {
+            headers: {
+                'User-Agent': 'WebPeel/0.3.1 (+https://webpeel.dev)',
+            },
+            signal: AbortSignal.timeout(5000), // 5 second timeout
+        });
+        if (!response.ok) {
+            // If robots.txt doesn't exist, allow everything
+            return { disallowedPaths: [] };
+        }
+        const text = await response.text();
+        const lines = text.split('\n');
+        const disallowedPaths = [];
+        let crawlDelay;
+        let relevantSection = false;
+        for (const line of lines) {
+            const trimmed = line.trim();
+            // Check for User-agent: *
+            if (trimmed.toLowerCase().startsWith('user-agent:')) {
+                const agent = trimmed.substring('user-agent:'.length).trim();
+                relevantSection = agent === '*';
+                continue;
+            }
+            if (!relevantSection)
+                continue;
+            // Parse Disallow directives
+            if (trimmed.toLowerCase().startsWith('disallow:')) {
+                const path = trimmed.substring('disallow:'.length).trim();
+                if (path) {
+                    disallowedPaths.push(path);
+                }
+            }
+            // Parse Crawl-delay directive
+            if (trimmed.toLowerCase().startsWith('crawl-delay:')) {
+                const delay = parseInt(trimmed.substring('crawl-delay:'.length).trim());
+                if (!isNaN(delay)) {
+                    crawlDelay = delay * 1000; // Convert to milliseconds
+                }
+            }
+        }
+        return { disallowedPaths, crawlDelay };
+    }
+    catch {
+        // If we can't fetch robots.txt, allow everything
+        return { disallowedPaths: [] };
+    }
+}
+/**
+ * Check if a URL is allowed by robots.txt rules
+ */
+function isAllowedByRobots(url, rules) {
+    const urlObj = new URL(url);
+    const path = urlObj.pathname;
+    for (const disallowed of rules.disallowedPaths) {
+        // Simple prefix matching (proper robots.txt parsing would handle wildcards)
+        if (path.startsWith(disallowed)) {
+            return false;
+        }
+    }
+    return true;
+}
+/**
+ * Crawl a website starting from a URL
+ *
+ * @param startUrl - Starting URL to crawl from
+ * @param options - Crawl options
+ * @returns Array of crawl results
+ *
+ * @example
+ * ```typescript
+ * import { crawl } from 'webpeel';
+ *
+ * const results = await crawl('https://example.com', {
+ *   maxPages: 20,
+ *   maxDepth: 2,
+ * });
+ *
+ * console.log(`Crawled ${results.length} pages`);
+ * ```
+ */
+export async function crawl(startUrl, options = {}) {
+    const { maxPages = 10, tier, maxDepth = 2, allowedDomains, excludePatterns = [], respectRobotsTxt = true, rateLimitMs = 500, sitemapFirst = false, strategy = 'bfs', deduplication = true, includePatterns = [], resume = false, onProgress, onPage, ...peelOptions } = options;
+    const crawlStartTime = Date.now();
+    // Validate limits
+    const tierMaxPages = TIER_MAX_PAGES[tier || 'free'] ?? TIER_MAX_PAGES.free;
+    const validatedMaxPages = Math.min(Math.max(maxPages, 1), tierMaxPages);
+    const validatedMaxDepth = Math.min(Math.max(maxDepth, 1), 5);
+    const validatedRateLimit = Math.max(rateLimitMs, 100); // Min 100ms between requests
+    // Parse starting URL
+    const startUrlObj = new URL(startUrl);
+    const startDomain = startUrlObj.hostname;
+    // Default: only crawl same domain as starting URL
+    const validatedAllowedDomains = allowedDomains && allowedDomains.length > 0
+        ? allowedDomains
+        : [startDomain];
+    // Compile exclude patterns (with timeout protection against ReDoS)
+    const excludeRegexes = excludePatterns.map(pattern => safeRegex(pattern));
+    // Compile include patterns (with timeout protection against ReDoS)
+    const includeRegexes = includePatterns.map(pattern => safeRegex(pattern));
+    // Fetch robots.txt if needed
+    let robotsRules = { disallowedPaths: [] };
+    if (respectRobotsTxt) {
+        robotsRules = await fetchRobotsTxt(startDomain);
+        // Use crawl-delay from robots.txt if it's larger than our rate limit
+        if (robotsRules.crawlDelay && robotsRules.crawlDelay > validatedRateLimit) {
+            log.info(`Using Crawl-delay from robots.txt: ${robotsRules.crawlDelay}ms`);
+        }
+    }
+    const effectiveRateLimit = robotsRules.crawlDelay || validatedRateLimit;
+    // Checkpoint: generate a deterministic job ID for this crawl
+    const crawlOptionsForCheckpoint = {
+        maxPages: validatedMaxPages,
+        maxDepth: validatedMaxDepth,
+        includes: includePatterns,
+        excludes: excludePatterns,
+    };
+    const jobId = generateJobId(startUrl, crawlOptionsForCheckpoint);
+    // Load existing checkpoint if resume is requested
+    const checkpoint = resume ? loadCheckpoint(jobId) : null;
+    if (checkpoint) {
+        log.info(`Resuming crawl from checkpoint: ${checkpoint.completed.size} pages already crawled`);
+    }
+    // State tracking
+    const results = [];
+    const visited = new Set();
+    const contentFingerprints = new Set();
+    let failedCount = 0;
+    // If resuming, restore visited/results from checkpoint
+    if (checkpoint) {
+        for (const [url] of checkpoint.completed) {
+            visited.add(url);
+        }
+    }
+    const queue = [];
+    // If resuming with pending URLs, restore queue; otherwise start from scratch
+    if (checkpoint && checkpoint.pending.length > 0) {
+        for (const pendingUrl of checkpoint.pending) {
+            queue.push({ url: pendingUrl, depth: 1, parent: startUrl });
+        }
+    }
+    else {
+        queue.push({ url: startUrl, depth: 0, parent: null });
+    }
+    // Sitemap-first: Discover URLs from sitemap before crawling
+    if (sitemapFirst) {
+        try {
+            const sitemap = await discoverSitemap(startDomain, { timeout: 10000, maxUrls: validatedMaxPages });
+            for (const entry of sitemap.urls) {
+                const entryUrl = entry.url;
+                try {
+                    const entryUrlObj = new URL(entryUrl);
+                    if (validatedAllowedDomains.includes(entryUrlObj.hostname)) {
+                        queue.push({ url: entryUrl, depth: 1, parent: startUrl });
+                    }
+                }
+                catch { /* skip invalid URLs */ }
+            }
+        }
+        catch { /* skip sitemap errors */ }
+    }
+    while (queue.length > 0 && results.length < validatedMaxPages) {
+        // Use DFS (stack) or BFS (queue) strategy
+        const item = strategy === 'dfs' ? queue.pop() : queue.shift();
+        const { url, depth, parent } = item;
+        // Skip if already visited
+        if (visited.has(url))
+            continue;
+        visited.add(url);
+        // Skip if depth exceeded
+        if (depth > validatedMaxDepth)
+            continue;
+        // Validate URL
+        let urlObj;
+        try {
+            urlObj = new URL(url);
+        }
+        catch {
+            continue; // Skip invalid URLs
+        }
+        // Check if domain is allowed
+        if (!validatedAllowedDomains.includes(urlObj.hostname)) {
+            continue;
+        }
+        // Check exclude patterns
+        if (excludeRegexes.some(regex => regex.test(url))) {
+            continue;
+        }
+        // Check include patterns
+        if (includeRegexes.length > 0 && !includeRegexes.some(regex => regex.test(url))) {
+            continue;
+        }
+        // Check robots.txt
+        if (respectRobotsTxt && !isAllowedByRobots(url, robotsRules)) {
+            log.debug(`Skipping ${url} (disallowed by robots.txt)`);
+            continue;
+        }
+        // Fetch the page
+        try {
+            const result = await peel(url, {
+                ...peelOptions,
+                format: 'markdown',
+            });
+            // Deduplication: compute content fingerprint
+            let fingerprint;
+            if (deduplication) {
+                fingerprint = createHash('sha256').update(result.content).digest('hex');
+                if (contentFingerprints.has(fingerprint)) {
+                    // Skip duplicate content
+                    continue;
+                }
+                contentFingerprints.add(fingerprint);
+            }
+            const crawlResult = {
+                url: result.url,
+                title: result.title,
+                markdown: result.content,
+                tokens: result.tokens ?? 0,
+                links: result.links,
+                depth,
+                parent,
+                elapsed: result.elapsed,
+            };
+            if (fingerprint) {
+                crawlResult.fingerprint = fingerprint;
+            }
+            results.push(crawlResult);
+            // Save checkpoint every 5 pages
+            if (results.length % 5 === 0) {
+                saveCheckpoint({
+                    jobId,
+                    startUrl,
+                    completed: new Map(results
+                        .filter(r => !r.error)
+                        .map(r => [r.url, { status: 200, contentLength: r.markdown.length, timestamp: Date.now() }])),
+                    pending: queue.map(q => q.url),
+                    discovered: [],
+                    options: crawlOptionsForCheckpoint,
+                    startedAt: crawlStartTime,
+                    lastCheckpoint: Date.now(),
+                    maxPages: validatedMaxPages,
+                });
+            }
+            // Call per-page callback with full result
+            if (onPage) {
+                onPage(crawlResult);
+            }
+            // Call progress callback
+            if (onProgress) {
+                onProgress({
+                    crawled: results.length,
+                    queued: queue.length,
+                    failed: failedCount,
+                    currentUrl: url,
+                    elapsed: Date.now() - crawlStartTime,
+                });
+            }
+            // Add discovered links to queue
+            if (depth < validatedMaxDepth) {
+                for (const link of result.links) {
+                    if (!visited.has(link)) {
+                        queue.push({
+                            url: link,
+                            depth: depth + 1,
+                            parent: url,
+                        });
+                    }
+                }
+            }
+            // Rate limiting
+            if (results.length < validatedMaxPages) {
+                await new Promise(resolve => setTimeout(resolve, effectiveRateLimit));
+            }
+        }
+        catch (error) {
+            // Log error and continue
+            failedCount++;
+            const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+            log.error(`Failed to fetch ${url}: ${errorMessage}`);
+            const errorResult = {
+                url,
+                title: '',
+                markdown: '',
+                tokens: 0,
+                links: [],
+                depth,
+                parent,
+                elapsed: 0,
+                error: errorMessage,
+            };
+            results.push(errorResult);
+            // Call per-page callback with error result
+            if (onPage) {
+                onPage(errorResult);
+            }
+            // Call progress callback even for failed pages
+            if (onProgress) {
+                onProgress({
+                    crawled: results.length,
+                    queued: queue.length,
+                    failed: failedCount,
+                    currentUrl: url,
+                    elapsed: Date.now() - crawlStartTime,
+                });
+            }
+        }
+    }
+    // Crawl complete — clean up checkpoint
+    deleteCheckpoint(jobId);
+    return results;
+}

package/dist/core/cross-verify.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Cross-source verification — search multiple engines, compare results,
+ * compute consensus/confidence scores.
+ */
+import type { WebSearchResult } from './search-provider.js';
+export interface CrossVerifyResult {
+    query: string;
+    sources: Array<{
+        engine: string;
+        resultCount: number;
+        topResults: WebSearchResult[];
+    }>;
+    consensus: Array<{
+        url: string;
+        title: string;
+        appearsIn: string[];
+        agreementScore: number;
+        averagePosition: number;
+    }>;
+    confidence: number;
+    totalSources: number;
+    elapsed: number;
+}
+export declare function crossVerifySearch(query: string, options?: {
+    engines?: string[];
+    count?: number;
+}): Promise<CrossVerifyResult>;