@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/research.js

@@ -0,0 +1,270 @@
+/**
+ * WebPeel Deep Research Agent
+ *
+ * Autonomously searches the web, fetches top sources, filters content with
+ * BM25, optionally follows promising links, and synthesizes a comprehensive
+ * report using an LLM.
+ *
+ * Design principle: orchestrate existing modules (peel, bm25-filter,
+ * llm-extract) — don't reinvent anything.
+ */
+// Regex for markdown links: [title](url) — supports https:// and protocol-relative //
+const LINK_REGEX = /\[([^\]]*)\]\(((?:https?:)?\/\/[^)]+)\)/g;
+/**
+ * Resolve a DDG redirect URL to the actual destination.
+ * DDG HTML search uses `//duckduckgo.com/l/?uddg=https%3A%2F%2Factual-url&rut=...`
+ */
+function resolveDdgRedirect(rawUrl) {
+    try {
+        // Normalize protocol-relative URLs
+        const normalised = rawUrl.startsWith('//') ? `https:${rawUrl}` : rawUrl;
+        const parsed = new URL(normalised);
+        if (parsed.hostname.includes('duckduckgo.com') && parsed.pathname === '/l/') {
+            const target = parsed.searchParams.get('uddg');
+            if (target)
+                return target;
+        }
+        // Not a DDG redirect — return as-is if it's a real http(s) URL
+        if (normalised.startsWith('http://') || normalised.startsWith('https://')) {
+            return normalised;
+        }
+        return null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Extract unique non-DDG links from markdown content.
+ * Handles DDG redirect URLs by extracting the actual destination from the `uddg` param.
+ */
+function extractLinks(markdown, visitedUrls) {
+    const found = [];
+    const regex = new RegExp(LINK_REGEX.source, 'g');
+    let match;
+    while ((match = regex.exec(markdown)) !== null) {
+        const [, title, rawUrl] = match;
+        if (!rawUrl)
+            continue;
+        const resolvedUrl = resolveDdgRedirect(rawUrl);
+        if (!resolvedUrl)
+            continue;
+        if (resolvedUrl.includes('duckduckgo.com'))
+            continue;
+        if (visitedUrls.has(resolvedUrl))
+            continue;
+        found.push({ title: title || '', url: resolvedUrl });
+        visitedUrls.add(resolvedUrl);
+    }
+    return found;
+}
+/**
+ * Conduct autonomous multi-step web research on a topic.
+ */
+export async function research(options) {
+    const { query, maxSources = 5, maxDepth = 1, timeout = 60000, outputFormat = 'report', onProgress, } = options;
+    const startTime = Date.now();
+    const sources = [];
+    const visitedUrls = new Set();
+    // Lazy imports so users who don't call research() don't pay the cost
+    const { peel } = await import('../index.js');
+    const { filterByRelevance, computeRelevanceScore } = await import('./bm25-filter.js');
+    // -------------------------------------------------------------------------
+    // Phase 1: Search
+    // -------------------------------------------------------------------------
+    onProgress?.({ phase: 'searching', message: `Searching for: ${query}` });
+    let searchUrls = [];
+    try {
+        // Use the search provider abstraction instead of directly scraping DDG.
+        // This picks the best available provider (Serper > Brave > DDG with fallbacks).
+        const { getBestSearchProvider } = await import('./search-provider.js');
+        const { provider, apiKey } = getBestSearchProvider();
+        const searchResults = await provider.searchWeb(query, {
+            count: Math.min(maxSources * 2, 10), // Fetch extra for filtering
+            apiKey,
+        });
+        if (searchResults.length > 0) {
+            searchUrls = searchResults.map(r => ({ title: r.title, url: r.url }));
+        }
+        else {
+            // Fallback: scrape DDG directly via peel (works locally)
+            const searchResult = await peel(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`, {
+                format: 'markdown',
+                timeout: 10000,
+            });
+            searchUrls = extractLinks(searchResult.content, visitedUrls);
+        }
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'search failed:', e instanceof Error ? e.message : e);
+    }
+    // -------------------------------------------------------------------------
+    // Phase 2: Fetch top sources in parallel
+    // -------------------------------------------------------------------------
+    const sourcesToFetch = searchUrls.slice(0, maxSources);
+    onProgress?.({
+        phase: 'fetching',
+        message: `Fetching ${sourcesToFetch.length} sources`,
+        sourcesFound: searchUrls.length,
+    });
+    const fetchPromises = sourcesToFetch.map(async ({ title, url }) => {
+        try {
+            // Guard: bail if we've burned more than 70% of the time budget
+            if (Date.now() - startTime > timeout * 0.7)
+                return null;
+            const result = await peel(url, {
+                format: 'markdown',
+                timeout: 15000,
+                budget: 3000,
+            });
+            // Phase 3 (inline): BM25 filter content to query + compute relevance
+            const filtered = filterByRelevance(result.content, { query });
+            const relevance = computeRelevanceScore(result.content, query);
+            return {
+                url,
+                title: result.title || title || url,
+                findings: filtered.content.slice(0, 4000),
+                relevance,
+            };
+        }
+        catch {
+            return null;
+        }
+    });
+    const fetchResults = await Promise.allSettled(fetchPromises);
+    for (const r of fetchResults) {
+        if (r.status === 'fulfilled' && r.value) {
+            sources.push(r.value);
+        }
+    }
+    // Sort by relevance (descending)
+    sources.sort((a, b) => b.relevance - a.relevance);
+    onProgress?.({
+        phase: 'extracting',
+        message: `Extracted content from ${sources.length} sources`,
+        sourcesFetched: sources.length,
+    });
+    // -------------------------------------------------------------------------
+    // Phase 4: Follow promising links (only when maxDepth > 1)
+    // -------------------------------------------------------------------------
+    if (maxDepth > 1 && sources.length > 0 && Date.now() - startTime < timeout * 0.5) {
+        onProgress?.({ phase: 'following', message: 'Following promising links for deeper research' });
+        const topSources = sources.slice(0, 2);
+        for (const source of topSources) {
+            const linkedUrls = extractLinks(source.findings, visitedUrls).slice(0, 2);
+            for (const { url: followUrl } of linkedUrls) {
+                if (Date.now() - startTime > timeout * 0.7)
+                    break;
+                try {
+                    const followResult = await peel(followUrl, {
+                        format: 'markdown',
+                        timeout: 10000,
+                        budget: 2000,
+                    });
+                    const filtered = filterByRelevance(followResult.content, { query });
+                    const followRelevance = computeRelevanceScore(followResult.content, query);
+                    sources.push({
+                        url: followUrl,
+                        title: followResult.title || followUrl,
+                        findings: filtered.content.slice(0, 3000),
+                        // Slightly lower weight for follow-up links
+                        relevance: followRelevance * 0.8,
+                    });
+                }
+                catch (e) {
+                    if (process.env.DEBUG)
+                        console.debug('[webpeel]', 'followup fetch failed:', e instanceof Error ? e.message : e);
+                }
+            }
+        }
+        sources.sort((a, b) => b.relevance - a.relevance);
+    }
+    // -------------------------------------------------------------------------
+    // Phase 5: Synthesize
+    // -------------------------------------------------------------------------
+    let report = '';
+    let tokensUsed;
+    let cost;
+    const apiKey = options.apiKey ?? process.env.OPENAI_API_KEY;
+    if (outputFormat === 'report' && apiKey && sources.length > 0) {
+        onProgress?.({ phase: 'synthesizing', message: 'Synthesizing research report' });
+        const model = options.model ?? 'gpt-4o-mini';
+        const baseUrl = options.baseUrl ?? 'https://api.openai.com/v1';
+        const sourceSummaries = sources
+            .slice(0, 8)
+            .map((s, i) => `Source ${i + 1}: ${s.title}\nURL: ${s.url}\nRelevance: ${Math.round(s.relevance * 100)}%\n\n${s.findings.slice(0, 2000)}`)
+            .join('\n\n---\n\n');
+        const synthPrompt = `Based on the following web research sources, write a comprehensive research report answering this question:
+
+"${query}"
+
+Sources:
+${sourceSummaries}
+
+Instructions:
+1. Synthesize information from ALL relevant sources
+2. Include specific data points, numbers, and facts
+3. Cite sources using [Source N] format
+4. If sources disagree, note the conflicting information
+5. End with a "Sources" section listing all URLs
+6. Write in markdown format
+7. Be thorough but concise`;
+        try {
+            const response = await fetch(`${baseUrl}/chat/completions`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    Authorization: `Bearer ${apiKey}`,
+                },
+                body: JSON.stringify({
+                    model,
+                    messages: [
+                        {
+                            role: 'system',
+                            content: 'You are a research analyst. Produce well-structured, factual research reports based on the provided web sources. Always cite your sources.',
+                        },
+                        { role: 'user', content: synthPrompt },
+                    ],
+                    temperature: 0.3,
+                    max_tokens: 4000,
+                }),
+                signal: AbortSignal.timeout(30000),
+            });
+            if (response.ok) {
+                const data = (await response.json());
+                report = data.choices?.[0]?.message?.content ?? '';
+                tokensUsed = {
+                    input: data.usage?.prompt_tokens ?? 0,
+                    output: data.usage?.completion_tokens ?? 0,
+                };
+                const { estimateCost } = await import('./llm-extract.js');
+                cost = estimateCost(model, tokensUsed.input, tokensUsed.output);
+            }
+        }
+        catch (e) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', 'llm synthesis failed:', e instanceof Error ? e.message : e);
+        }
+    }
+    // If synthesis wasn't attempted or failed, produce raw sources report
+    if (!report) {
+        report = sources
+            .map((s, i) => `## Source ${i + 1}: ${s.title}\n**URL:** ${s.url}\n**Relevance:** ${Math.round(s.relevance * 100)}%\n\n${s.findings.slice(0, 2000)}`)
+            .join('\n\n---\n\n');
+    }
+    return {
+        report,
+        sources: sources.map(s => ({
+            url: s.url,
+            title: s.title,
+            findings: s.findings.slice(0, 500),
+            relevance: s.relevance,
+        })),
+        totalSourcesFound: searchUrls.length,
+        sourcesConsulted: sources.length,
+        elapsed: Date.now() - startTime,
+        tokensUsed,
+        cost,
+    };
+}

package/dist/core/retry.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Smart retry with exponential backoff and jitter.
+ * Inspired by Crawl4AI's RateLimiter — the cleanest implementation found.
+ *
+ * Features:
+ * - Exponential backoff with ±25% jitter (prevents thundering herd)
+ * - Per-domain delay tracking (optional)
+ * - Success reduces delay by 25% (gradual recovery)
+ * - Configurable retry predicate
+ */
+export interface RetryOptions {
+    /** Max retry attempts (default: 3) */
+    maxRetries?: number;
+    /** Initial delay in ms (default: 1000) */
+    baseDelayMs?: number;
+    /** Max delay cap in ms (default: 30000) */
+    maxDelayMs?: number;
+    /** Add ±25% jitter (default: true) */
+    jitter?: boolean;
+    /** Custom predicate: should we retry this error? (default: isRetryable) */
+    retryOn?: (error: Error, attempt: number) => boolean;
+    /** Called before each retry (for logging/metrics) */
+    onRetry?: (error: Error, attempt: number, delayMs: number) => void;
+    /** Label for logging */
+    label?: string;
+}
+/**
+ * Execute a function with retry logic.
+ * Throws the last error if all retries are exhausted.
+ */
+export declare function withRetry<T>(fn: () => Promise<T>, options?: RetryOptions): Promise<T>;
+/**
+ * Per-domain rate state tracker.
+ * Adapts delay per target domain based on success/failure patterns.
+ * Useful for avoiding rate limits on target sites.
+ */
+export declare class DomainRateLimiter {
+    private domains;
+    private baseDelay;
+    private maxDelay;
+    private rateLimitCodes;
+    constructor(options?: {
+        baseDelay?: number;
+        maxDelay?: number;
+        rateLimitCodes?: number[];
+    });
+    /** Get the hostname from a URL */
+    private getDomain;
+    /** Wait if needed before making a request to this domain */
+    throttle(url: string): Promise<void>;
+    /** Record a response status for adaptive delay */
+    recordResult(url: string, statusCode: number): void;
+    /** Get current state for diagnostics */
+    getStats(): Record<string, {
+        delay: number;
+        failCount: number;
+    }>;
+}
+/** Singleton domain rate limiter for the fetch layer */
+export declare const domainLimiter: DomainRateLimiter;

package/dist/core/retry.js

@@ -0,0 +1,119 @@
+/**
+ * Smart retry with exponential backoff and jitter.
+ * Inspired by Crawl4AI's RateLimiter — the cleanest implementation found.
+ *
+ * Features:
+ * - Exponential backoff with ±25% jitter (prevents thundering herd)
+ * - Per-domain delay tracking (optional)
+ * - Success reduces delay by 25% (gradual recovery)
+ * - Configurable retry predicate
+ */
+import { isRetryable } from '../errors.js';
+import { createLogger } from './logger.js';
+const log = createLogger('retry');
+/**
+ * Execute a function with retry logic.
+ * Throws the last error if all retries are exhausted.
+ */
+export async function withRetry(fn, options = {}) {
+    const { maxRetries = 3, baseDelayMs = 1000, maxDelayMs = 30_000, jitter = true, retryOn = (err) => isRetryable(err), onRetry, label = 'operation', } = options;
+    let delay = baseDelayMs;
+    let lastError;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            const result = await fn();
+            // If we succeeded after retries, log it
+            if (attempt > 0) {
+                log.info(`${label} succeeded after ${attempt} retries`);
+            }
+            return result;
+        }
+        catch (err) {
+            lastError = err instanceof Error ? err : new Error(String(err));
+            // Last attempt or non-retryable — throw immediately
+            if (attempt === maxRetries || !retryOn(lastError, attempt)) {
+                throw lastError;
+            }
+            // Calculate delay with exponential backoff and jitter
+            const jitterFactor = jitter ? (0.75 + Math.random() * 0.5) : 1;
+            const actualDelay = Math.min(delay * jitterFactor, maxDelayMs);
+            if (onRetry) {
+                onRetry(lastError, attempt + 1, actualDelay);
+            }
+            log.info(`${label} attempt ${attempt + 1} failed: ${lastError.message}. Retrying in ${Math.round(actualDelay)}ms`);
+            await new Promise(r => setTimeout(r, actualDelay));
+            // Exponential increase for next attempt
+            delay = Math.min(delay * 2, maxDelayMs);
+        }
+    }
+    throw lastError ?? new Error('Retry exhausted with no error');
+}
+/**
+ * Per-domain rate state tracker.
+ * Adapts delay per target domain based on success/failure patterns.
+ * Useful for avoiding rate limits on target sites.
+ */
+export class DomainRateLimiter {
+    domains = new Map();
+    baseDelay;
+    maxDelay;
+    rateLimitCodes;
+    constructor(options = {}) {
+        this.baseDelay = options.baseDelay ?? 1000;
+        this.maxDelay = options.maxDelay ?? 60_000;
+        this.rateLimitCodes = options.rateLimitCodes ?? [429, 503];
+    }
+    /** Get the hostname from a URL */
+    getDomain(url) {
+        try {
+            return new URL(url).hostname;
+        }
+        catch {
+            return url;
+        }
+    }
+    /** Wait if needed before making a request to this domain */
+    async throttle(url) {
+        const domain = this.getDomain(url);
+        const state = this.domains.get(domain);
+        if (!state)
+            return;
+        const elapsed = Date.now() - state.lastHit;
+        const waitTime = Math.max(0, state.delay - elapsed);
+        if (waitTime > 0) {
+            await new Promise(r => setTimeout(r, waitTime));
+        }
+        state.lastHit = Date.now();
+    }
+    /** Record a response status for adaptive delay */
+    recordResult(url, statusCode) {
+        const domain = this.getDomain(url);
+        let state = this.domains.get(domain);
+        if (!state) {
+            state = { delay: 0, failCount: 0, lastHit: Date.now() };
+            this.domains.set(domain, state);
+        }
+        if (this.rateLimitCodes.includes(statusCode)) {
+            state.failCount++;
+            // Exponential backoff with ±25% jitter
+            const jitter = 0.75 + Math.random() * 0.5;
+            state.delay = Math.min((state.delay || this.baseDelay) * 2 * jitter, this.maxDelay);
+            log.warn(`Domain ${domain} rate limited (${statusCode}). Delay: ${Math.round(state.delay)}ms`);
+        }
+        else {
+            // Gradual recovery on success
+            state.delay = Math.max(0, state.delay * 0.75);
+            state.failCount = 0;
+        }
+    }
+    /** Get current state for diagnostics */
+    getStats() {
+        const stats = {};
+        for (const [domain, state] of this.domains) {
+            stats[domain] = { delay: Math.round(state.delay), failCount: state.failCount };
+        }
+        return stats;
+    }
+}
+/** Singleton domain rate limiter for the fetch layer */
+export const domainLimiter = new DomainRateLimiter();

package/dist/core/safe-browsing.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Domain safety check using Google Safe Browsing Lookup API v4.
+ * Free: 10,000 lookups/day.
+ * Falls back to a local blocklist when no API key is configured.
+ * Also checks community threat feeds (URLhaus, PhishTank, OpenPhish) for known threats.
+ */
+import { type ThreatFeedResult } from './threat-feeds.js';
+export type { ThreatFeedResult };
+export interface SafeBrowsingResult {
+    safe: boolean;
+    threats: string[];
+    source: 'google-api' | 'local-blocklist' | 'unchecked';
+    /** Community threat feed check result (URLhaus, PhishTank, OpenPhish) */
+    threatFeeds?: ThreatFeedResult;
+}
+/**
+ * Check URL safety.
+ *
+ * Flow:
+ * 1. If SAFE_BROWSING_API_KEY (or passed apiKey) is set, race Google API vs 2s timeout.
+ *    Falls back to local blocklist on timeout or error.
+ * 2. Run community threat feeds (URLhaus, PhishTank, OpenPhish) in parallel.
+ * 3. Without an API key, use local heuristic blocklist only.
+ *
+ * If ANY source flags the URL as unsafe, the overall result is safe: false.
+ *
+ * @param url    The URL to check
+ * @param apiKey Google Safe Browsing API key (optional). Falls back to SAFE_BROWSING_API_KEY env var.
+ */
+export declare function checkUrlSafety(url: string, apiKey?: string): Promise<SafeBrowsingResult>;

package/dist/core/safe-browsing.js

@@ -0,0 +1,206 @@
+/**
+ * Domain safety check using Google Safe Browsing Lookup API v4.
+ * Free: 10,000 lookups/day.
+ * Falls back to a local blocklist when no API key is configured.
+ * Also checks community threat feeds (URLhaus, PhishTank, OpenPhish) for known threats.
+ */
+import { checkThreatFeeds } from './threat-feeds.js';
+// Known brands commonly impersonated in phishing
+const KNOWN_BRANDS = [
+    'amazon', 'google', 'facebook', 'apple', 'microsoft', 'paypal', 'netflix',
+    'instagram', 'twitter', 'linkedin', 'dropbox', 'chase', 'wellsfargo', 'bankofamerica',
+    'citibank', 'hsbc', 'ebay', 'walmart', 'target', 'bestbuy', 'fedex', 'ups', 'usps',
+    'irs', 'dmv', 'gov', 'yahoo', 'outlook', 'hotmail',
+];
+// TLDs heavily abused for phishing/malware (free-domain registrars)
+const SUSPICIOUS_TLDS = new Set(['.tk', '.ml', '.ga', '.cf', '.gq', '.top', '.click', '.loan', '.win', '.xyz', '.club', '.work']);
+// Private/reserved IPv4 ranges (safe for local dev)
+const PRIVATE_IP_RANGES = [
+    /^127\.\d+\.\d+\.\d+$/, // loopback
+    /^10\.\d+\.\d+\.\d+$/, // RFC 1918
+    /^192\.168\.\d+\.\d+$/, // RFC 1918
+    /^172\.(1[6-9]|2\d|3[01])\.\d+\.\d+$/, // RFC 1918
+    /^169\.254\.\d+\.\d+$/, // link-local
+    /^::1$/, // IPv6 loopback
+    /^fc00:/, // IPv6 private
+    /^fd[0-9a-f]{2}:/i, // IPv6 ULA
+];
+function isPrivateIp(host) {
+    return PRIVATE_IP_RANGES.some((re) => re.test(host));
+}
+function isIpAddress(host) {
+    // IPv4
+    if (/^\d{1,3}(\.\d{1,3}){3}$/.test(host))
+        return true;
+    // IPv6 (bare or bracketed)
+    if (/^\[?[0-9a-fA-F:]+\]?$/.test(host))
+        return true;
+    return false;
+}
+/**
+ * Local heuristic blocklist — catches common attack patterns without an API key.
+ */
+function checkLocalBlocklist(url) {
+    const threats = [];
+    // 1. Data URIs — always suspicious
+    if (/^data:/i.test(url.trim())) {
+        threats.push('DATA_URI');
+        return { safe: false, threats, source: 'local-blocklist' };
+    }
+    let parsed = null;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        // Unparseable URL — flag as suspicious
+        threats.push('INVALID_URL');
+        return { safe: false, threats, source: 'local-blocklist' };
+    }
+    const { hostname, username, password } = parsed;
+    // 2. @ sign trick: http://google.com@evil.com/login → username = 'google.com'
+    if (username || password) {
+        threats.push('URL_CREDENTIALS_TRICK');
+        return { safe: false, threats, source: 'local-blocklist' };
+    }
+    // 3. Punycode homograph attacks (xn-- internationalized domains)
+    if (/\bxn--/i.test(hostname)) {
+        // Allow legitimate IDN TLDs (e.g. .xn--p1ai = .рф)
+        const parts = hostname.split('.');
+        const hasPunycodeLabel = parts.slice(0, -1).some((p) => /^xn--/i.test(p));
+        if (hasPunycodeLabel) {
+            threats.push('PUNYCODE_HOMOGRAPH');
+        }
+    }
+    // 4. IP-only URLs pointing to non-private ranges
+    if (isIpAddress(hostname)) {
+        const bare = hostname.replace(/^\[|\]$/g, ''); // strip brackets from IPv6
+        if (!isPrivateIp(bare)) {
+            threats.push('SUSPICIOUS_IP');
+        }
+        if (threats.length > 0)
+            return { safe: false, threats, source: 'local-blocklist' };
+        return { safe: true, threats: [], source: 'local-blocklist' };
+    }
+    const lowerHost = hostname.toLowerCase();
+    // Remove www prefix for analysis
+    const hostNoWww = lowerHost.replace(/^www\./, '');
+    const parts = hostNoWww.split('.');
+    const tld = parts.length >= 2 ? '.' + parts[parts.length - 1] : '';
+    const sld = parts.length >= 2 ? parts[parts.length - 2] : '';
+    // 5. Known-bad TLDs combined with brand names (amazon-login.tk)
+    if (SUSPICIOUS_TLDS.has(tld)) {
+        const containsBrand = KNOWN_BRANDS.some((brand) => hostNoWww.includes(brand));
+        if (containsBrand) {
+            threats.push('PHISHING');
+        }
+    }
+    // 6. Excessive hyphens in SLD (amaz0n-login-verify-account.com)
+    const hyphenCount = (sld.match(/-/g) || []).length;
+    if (hyphenCount >= 3) {
+        threats.push('EXCESSIVE_HYPHENS');
+    }
+    // 7. Brand name in subdomain combined with suspicious TLD
+    if (SUSPICIOUS_TLDS.has(tld)) {
+        const subdomains = parts.slice(0, -2).join('.');
+        const subHasBrand = KNOWN_BRANDS.some((brand) => subdomains.includes(brand));
+        if (subHasBrand && !threats.includes('PHISHING')) {
+            threats.push('PHISHING');
+        }
+    }
+    // 8. Excessive subdomains: login.secure.verify.account.bank.xyz.com
+    if (parts.length > 5) {
+        threats.push('EXCESSIVE_SUBDOMAINS');
+    }
+    if (threats.length > 0) {
+        return { safe: false, threats, source: 'local-blocklist' };
+    }
+    return { safe: true, threats: [], source: 'local-blocklist' };
+}
+/**
+ * Check a URL against the Google Safe Browsing Lookup API v4.
+ * Returns null on any error (network timeout, bad key, etc.) so caller can fall back.
+ */
+async function checkGoogleSafeBrowsing(url, apiKey) {
+    const endpoint = `https://safebrowsing.googleapis.com/v4/threatMatches:find?key=${encodeURIComponent(apiKey)}`;
+    const body = {
+        client: { clientId: 'webpeel', clientVersion: '1.0.0' },
+        threatInfo: {
+            threatTypes: ['MALWARE', 'SOCIAL_ENGINEERING', 'UNWANTED_SOFTWARE', 'POTENTIALLY_HARMFUL_APPLICATION'],
+            platformTypes: ['ANY_PLATFORM'],
+            threatEntryTypes: ['URL'],
+            threatEntries: [{ url }],
+        },
+    };
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 2000);
+    try {
+        const resp = await fetch(endpoint, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify(body),
+            signal: controller.signal,
+        });
+        clearTimeout(timeoutId);
+        if (!resp.ok)
+            return null;
+        const data = await resp.json();
+        if (!data.matches || data.matches.length === 0) {
+            return { safe: true, threats: [], source: 'google-api' };
+        }
+        const threats = [...new Set(data.matches.map((m) => m.threatType))];
+        return { safe: false, threats, source: 'google-api' };
+    }
+    catch {
+        clearTimeout(timeoutId);
+        return null;
+    }
+}
+/**
+ * Check URL safety.
+ *
+ * Flow:
+ * 1. If SAFE_BROWSING_API_KEY (or passed apiKey) is set, race Google API vs 2s timeout.
+ *    Falls back to local blocklist on timeout or error.
+ * 2. Run community threat feeds (URLhaus, PhishTank, OpenPhish) in parallel.
+ * 3. Without an API key, use local heuristic blocklist only.
+ *
+ * If ANY source flags the URL as unsafe, the overall result is safe: false.
+ *
+ * @param url    The URL to check
+ * @param apiKey Google Safe Browsing API key (optional). Falls back to SAFE_BROWSING_API_KEY env var.
+ */
+export async function checkUrlSafety(url, apiKey) {
+    const key = apiKey ?? process.env.SAFE_BROWSING_API_KEY;
+    // Run Google Safe Browsing + community threat feeds in parallel
+    const [baseResult, threatFeedsResult] = await Promise.all([
+        (async () => {
+            if (key) {
+                // Race: Google API with 2s timeout, fallback to local
+                const timeoutResult = checkLocalBlocklist(url);
+                const googleResult = await Promise.race([
+                    checkGoogleSafeBrowsing(url, key),
+                    new Promise((resolve) => setTimeout(() => resolve(null), 2000)),
+                ]);
+                if (googleResult !== null)
+                    return googleResult;
+                // API timed out or errored — use local blocklist result
+                return timeoutResult;
+            }
+            // No API key — local blocklist only
+            return checkLocalBlocklist(url);
+        })(),
+        checkThreatFeeds(url),
+    ]);
+    // Merge: if threat feeds found threats, combine into final result
+    const allThreats = [
+        ...baseResult.threats,
+        ...threatFeedsResult.threats,
+    ];
+    const isUnsafe = !baseResult.safe || !threatFeedsResult.safe;
+    return {
+        safe: !isUnsafe,
+        threats: [...new Set(allThreats)],
+        source: baseResult.source,
+        threatFeeds: threatFeedsResult,
+    };
+}