@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/dns-cache.js

@@ -0,0 +1,198 @@
+/**
+ * DNS Pre-Resolution Cache
+ *
+ * Warms a local Map<hostname, ip[]> on startup for the top ~50 popular domains
+ * and exposes a custom lookup function compatible with undici's Agent `connect.lookup`.
+ */
+import dns from 'node:dns';
+import net from 'node:net';
+const DNS_CACHE_TTL_MS = 30 * 60 * 1000; // 30 minutes
+const DNS_WARMUP_DOMAINS = [
+    'github.com',
+    'www.github.com',
+    'raw.githubusercontent.com',
+    'api.github.com',
+    'wikipedia.org',
+    'en.wikipedia.org',
+    'news.ycombinator.com',
+    'stackoverflow.com',
+    'www.stackoverflow.com',
+    'developer.mozilla.org',
+    'react.dev',
+    'nextjs.org',
+    'vercel.com',
+    'tailwindcss.com',
+    'supabase.com',
+    'npmjs.com',
+    'www.npmjs.com',
+    'reddit.com',
+    'www.reddit.com',
+    'www.cloudflare.com',
+    'medium.com',
+    'linkedin.com',
+    'www.linkedin.com',
+    'www.bloomberg.com',
+    'www.glassdoor.com',
+    'arxiv.org',
+    'www.sec.gov',
+    'w3.org',
+    'www.w3.org',
+    'tools.ietf.org',
+    'unicode.org',
+    'www.bbc.com',
+    'bbc.co.uk',
+    'stripe.com',
+    'docs.stripe.com',
+    'vuejs.org',
+    'angular.io',
+    'www.washingtonpost.com',
+    'www.theguardian.com',
+    'techcrunch.com',
+    'www.wired.com',
+    'arstechnica.com',
+    'docs.google.com',
+    'drive.google.com',
+    'www.notion.so',
+    'www.producthunt.com',
+    'www.crunchbase.com',
+    'news.google.com',
+    'www.youtube.com',
+    'example.com',
+    'httpbin.org',
+    'docs.python.org',
+    'nodejs.org',
+    'openai.com',
+    'anthropic.com',
+    'x.com',
+    'twitter.com',
+    'www.nytimes.com',
+    'www.wsj.com',
+    'www.reuters.com',
+    'www.theverge.com',
+    'www.cnn.com',
+    'www.amazon.com',
+    'www.apple.com',
+    'www.microsoft.com',
+];
+const dnsCache = new Map();
+let warmupStarted = false;
+let roundRobinCursor = 0;
+function normalizeHostname(hostname) {
+    return hostname.trim().toLowerCase();
+}
+function pruneIfExpired(hostname) {
+    const entry = dnsCache.get(hostname);
+    if (!entry)
+        return;
+    if (entry.expiresAt <= Date.now()) {
+        dnsCache.delete(hostname);
+    }
+}
+export function getCachedDns(hostname) {
+    const normalized = normalizeHostname(hostname);
+    pruneIfExpired(normalized);
+    const entry = dnsCache.get(normalized);
+    if (!entry || entry.ips.length === 0)
+        return null;
+    return [...entry.ips];
+}
+function setCachedDns(hostname, ips) {
+    if (ips.length === 0)
+        return;
+    const normalized = normalizeHostname(hostname);
+    dnsCache.set(normalized, {
+        ips: [...new Set(ips)],
+        expiresAt: Date.now() + DNS_CACHE_TTL_MS,
+    });
+}
+export async function resolveAndCache(hostname) {
+    const normalized = normalizeHostname(hostname);
+    const cached = getCachedDns(normalized);
+    if (cached)
+        return cached;
+    try {
+        const ips = await dns.promises.resolve4(normalized);
+        if (ips.length > 0)
+            setCachedDns(normalized, ips);
+        return ips;
+    }
+    catch {
+        return [];
+    }
+}
+function selectCachedIp(ips) {
+    if (ips.length === 1)
+        return ips[0];
+    const selected = ips[roundRobinCursor % ips.length];
+    roundRobinCursor = (roundRobinCursor + 1) % Number.MAX_SAFE_INTEGER;
+    return selected;
+}
+/**
+ * Custom lookup function compatible with undici's Agent `connect.lookup`.
+ *
+ * undici passes `{ hints: 1024, all: true }` — so when `all` is true the
+ * callback must receive `(err, entries: { address, family }[])`.
+ * When `all` is false (or absent), the callback is `(err, address, family)`.
+ */
+export function cachedLookup(hostname, options, callback) {
+    // If hostname is already an IP, return immediately
+    const ipFamily = net.isIP(hostname);
+    if (ipFamily === 4 || ipFamily === 6) {
+        if (options?.all) {
+            callback(null, [{ address: hostname, family: ipFamily }]);
+        }
+        else {
+            callback(null, hostname, ipFamily);
+        }
+        return;
+    }
+    // Only use cache for IPv4 lookups (family 0 or 4)
+    const requestedFamily = typeof options?.family === 'number' ? options.family : 0;
+    if (requestedFamily !== 6) {
+        const cachedIps = getCachedDns(hostname);
+        if (cachedIps && cachedIps.length > 0) {
+            if (options?.all) {
+                callback(null, cachedIps.map(ip => ({ address: ip, family: 4 })));
+            }
+            else {
+                callback(null, selectCachedIp(cachedIps), 4);
+            }
+            return;
+        }
+        // Async resolve, fall back to native lookup on failure
+        void resolveAndCache(hostname)
+            .then((resolvedIps) => {
+            if (resolvedIps.length > 0) {
+                if (options?.all) {
+                    callback(null, resolvedIps.map(ip => ({ address: ip, family: 4 })));
+                }
+                else {
+                    callback(null, selectCachedIp(resolvedIps), 4);
+                }
+            }
+            else {
+                dns.lookup(hostname, options, callback);
+            }
+        })
+            .catch(() => {
+            dns.lookup(hostname, options, callback);
+        });
+        return;
+    }
+    // IPv6 requested — fall through to native lookup
+    dns.lookup(hostname, options, callback);
+}
+export async function warmupDnsCache(domains = DNS_WARMUP_DOMAINS) {
+    await Promise.allSettled(domains.map((d) => resolveAndCache(d)));
+}
+export function startDnsWarmup() {
+    if (warmupStarted)
+        return;
+    warmupStarted = true;
+    void warmupDnsCache().catch(() => {
+        // Best-effort only.
+    });
+}
+export function clearDnsCache() {
+    dnsCache.clear();
+}

package/dist/core/documents.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Document (PDF/DOCX) parsing utilities.
+ *
+ * Keeps binary/document parsing separate from the HTML scraping pipeline.
+ */
+export type DocumentFormat = 'markdown' | 'text' | 'html';
+export interface DocumentExtractionResult {
+    content: string;
+    metadata: {
+        title: string;
+        contentType: string;
+        wordCount: number;
+        [key: string]: any;
+    };
+}
+export declare function normalizeContentType(contentTypeHeader: string | undefined | null): string;
+export declare function isPdfContentType(contentTypeHeader: string | undefined | null): boolean;
+export declare function isDocxContentType(contentTypeHeader: string | undefined | null): boolean;
+export declare function extractDocumentToFormat(buffer: Buffer, options?: {
+    url?: string;
+    contentType?: string;
+    format?: DocumentFormat;
+}): Promise<DocumentExtractionResult>;

package/dist/core/documents.js

@@ -0,0 +1,123 @@
+/**
+ * Document (PDF/DOCX) parsing utilities.
+ *
+ * Keeps binary/document parsing separate from the HTML scraping pipeline.
+ */
+import { htmlToMarkdown, htmlToText } from './markdown.js';
+import { extractPdf } from './pdf.js';
+export function normalizeContentType(contentTypeHeader) {
+    if (!contentTypeHeader)
+        return '';
+    return contentTypeHeader.split(';')[0]?.trim().toLowerCase() || '';
+}
+export function isPdfContentType(contentTypeHeader) {
+    const ct = normalizeContentType(contentTypeHeader);
+    return ct === 'application/pdf' || ct.endsWith('+pdf');
+}
+export function isDocxContentType(contentTypeHeader) {
+    const ct = normalizeContentType(contentTypeHeader);
+    return ct === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
+}
+function basenameFromUrl(url) {
+    if (!url)
+        return '';
+    try {
+        const u = new URL(url);
+        const last = u.pathname.split('/').filter(Boolean).pop() || '';
+        return decodeURIComponent(last);
+    }
+    catch {
+        return '';
+    }
+}
+function stripExtension(name) {
+    return name.replace(/\.(pdf|docx)$/i, '');
+}
+function escapeHtml(text) {
+    return text
+        .replace(/&/g, '&')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&#39;');
+}
+function countWords(text) {
+    const words = text
+        .replace(/\s+/g, ' ')
+        .trim()
+        .split(' ')
+        .filter(Boolean);
+    return words.length;
+}
+function normalizePlainText(text) {
+    // pdf-parse returns lots of line breaks; keep paragraphs but reduce noise.
+    return text
+        .replace(/\r\n/g, '\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .replace(/[ \t]+/g, ' ')
+        .trim();
+}
+export async function extractDocumentToFormat(buffer, options = {}) {
+    const { url, contentType, format = 'markdown' } = options;
+    const normalized = normalizeContentType(contentType);
+    const urlLower = (url || '').toLowerCase();
+    const isPdf = isPdfContentType(normalized) || urlLower.endsWith('.pdf');
+    const isDocx = isDocxContentType(normalized) || urlLower.endsWith('.docx');
+    if (isPdf) {
+        const pdf = await extractPdf(buffer);
+        const text = normalizePlainText(pdf.text || '');
+        const fallbackTitle = stripExtension(basenameFromUrl(url)) || 'PDF Document';
+        const title = pdf.metadata?.title || fallbackTitle;
+        const wordCount = countWords(text);
+        let content;
+        if (format === 'html') {
+            content = `<pre>${escapeHtml(text)}</pre>`;
+        }
+        else {
+            // markdown + text: return readable plain text.
+            content = text;
+        }
+        return {
+            content,
+            metadata: {
+                title,
+                contentType: normalized || 'application/pdf',
+                wordCount,
+                pages: pdf.pages,
+                ...pdf.metadata,
+            },
+        };
+    }
+    if (isDocx) {
+        // Mammoth returns clean semantic HTML.
+        const mammothMod = await import('mammoth');
+        const mammoth = mammothMod.default || mammothMod;
+        const result = await mammoth.convertToHtml({ buffer });
+        const html = (result?.value || '').trim();
+        const fallbackTitle = stripExtension(basenameFromUrl(url)) || 'Word Document';
+        const title = fallbackTitle;
+        // Word count should be based on plain text, not markdown formatting.
+        const plainText = htmlToText(html);
+        const wordCount = countWords(plainText);
+        let content;
+        if (format === 'html') {
+            content = html;
+        }
+        else if (format === 'text') {
+            content = plainText;
+        }
+        else {
+            content = htmlToMarkdown(html);
+        }
+        return {
+            content,
+            metadata: {
+                title,
+                contentType: normalized || 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+                wordCount,
+                messages: result?.messages || [],
+            },
+        };
+    }
+    throw new Error(`Unsupported document type: ${normalized || contentType || 'unknown'}`);
+}

package/dist/core/domain-memory.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Adaptive Domain Learning — Store and Reuse Successful Extraction Patterns
+ *
+ * Remembers which extraction method works best per domain and reuses that
+ * knowledge on subsequent fetches. Backed by an in-memory LRU cache with
+ * optional Postgres persistence (TODO).
+ */
+export interface DomainMemoryEntry {
+    domain: string;
+    /** Best method that succeeded: 'simple' | 'browser' | 'stealth' | 'cloaked' | 'domain-api' */
+    bestMethod: string;
+    /** Average response time in ms for the best method */
+    avgResponseMs: number;
+    /** Success count for the best method */
+    successCount: number;
+    /** Last successful fetch timestamp */
+    lastSuccess: number;
+    /** Content quality score (0-1) from the best method */
+    avgQuality: number;
+    /** Whether this domain requires JavaScript rendering */
+    requiresJs: boolean;
+    /** Whether this domain has anti-bot protection */
+    hasAntibot: boolean;
+    /** Total fetch attempts across all methods */
+    totalAttempts: number;
+}
+/**
+ * Record a fetch result for a domain.
+ * Call this after every successful fetch to build up domain knowledge.
+ */
+export declare function recordFetchResult(url: string, result: {
+    method: string;
+    responseMs: number;
+    quality: number;
+    wasBlocked: boolean;
+    hadJavascript: boolean;
+}): void;
+/**
+ * Get the recommended method for a domain based on past experience.
+ * Returns null if no history exists.
+ */
+export declare function getRecommendedMethod(url: string): {
+    method: string;
+    confidence: number;
+    avgResponseMs: number;
+    requiresJs: boolean;
+} | null;
+/**
+ * Get full domain memory entry.
+ */
+export declare function getDomainMemory(domain: string): DomainMemoryEntry | null;
+/**
+ * Get stats about the domain memory cache.
+ */
+export declare function getDomainMemoryStats(): {
+    totalDomains: number;
+    topDomains: Array<{
+        domain: string;
+        bestMethod: string;
+        successCount: number;
+    }>;
+};
+/** Persist current in-memory cache to Postgres. */
+export declare function syncToPostgres(): Promise<void>;
+/** Load domain memory from Postgres into the in-memory cache on startup. */
+export declare function loadFromPostgres(): Promise<void>;

package/dist/core/domain-memory.js

@@ -0,0 +1,163 @@
+/**
+ * Adaptive Domain Learning — Store and Reuse Successful Extraction Patterns
+ *
+ * Remembers which extraction method works best per domain and reuses that
+ * knowledge on subsequent fetches. Backed by an in-memory LRU cache with
+ * optional Postgres persistence (TODO).
+ */
+import { LRUCache } from 'lru-cache';
+// ---------------------------------------------------------------------------
+// Cache
+// ---------------------------------------------------------------------------
+/** In-memory cache: 5 000 domains, 1-hour TTL */
+const memoryCache = new LRUCache({
+    max: 5000,
+    ttl: 3_600_000,
+});
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Extract and normalise the hostname from a URL string.
+ * Strips leading "www." so that www.example.com and example.com share an entry.
+ */
+function normaliseDomain(url) {
+    try {
+        const hostname = new URL(url).hostname.toLowerCase();
+        return hostname.replace(/^www\./, '');
+    }
+    catch {
+        // If the URL is already just a hostname (e.g. "example.com"), use as-is.
+        return url.toLowerCase().replace(/^www\./, '');
+    }
+}
+/** Methods that imply JS rendering was needed. */
+const JS_METHODS = new Set(['browser', 'stealth']);
+/** Methods that imply anti-bot protection. */
+const ANTIBOT_METHODS = new Set(['stealth', 'cloaked']);
+// ---------------------------------------------------------------------------
+// Core functions
+// ---------------------------------------------------------------------------
+/**
+ * Record a fetch result for a domain.
+ * Call this after every successful fetch to build up domain knowledge.
+ */
+export function recordFetchResult(url, result) {
+    const domain = normaliseDomain(url);
+    const existing = memoryCache.get(domain);
+    if (existing) {
+        // Decide whether the incoming method should become the new bestMethod.
+        // We upgrade if the incoming quality is strictly higher.
+        const isBetterMethod = result.quality > existing.avgQuality;
+        if (isBetterMethod) {
+            existing.bestMethod = result.method;
+        }
+        // Rolling averages: (old * count + new) / (count + 1)
+        const count = existing.successCount;
+        existing.avgResponseMs =
+            (existing.avgResponseMs * count + result.responseMs) / (count + 1);
+        existing.avgQuality =
+            (existing.avgQuality * count + result.quality) / (count + 1);
+        existing.successCount += 1;
+        existing.totalAttempts += 1;
+        existing.lastSuccess = Date.now();
+        // Accumulate flags — once set they stay set.
+        if (JS_METHODS.has(result.method) || result.hadJavascript) {
+            existing.requiresJs = true;
+        }
+        if (ANTIBOT_METHODS.has(result.method) || result.wasBlocked) {
+            existing.hasAntibot = true;
+        }
+        memoryCache.set(domain, existing);
+    }
+    else {
+        // Brand-new entry
+        const entry = {
+            domain,
+            bestMethod: result.method,
+            avgResponseMs: result.responseMs,
+            successCount: 1,
+            lastSuccess: Date.now(),
+            avgQuality: result.quality,
+            requiresJs: JS_METHODS.has(result.method) || result.hadJavascript,
+            hasAntibot: ANTIBOT_METHODS.has(result.method) || result.wasBlocked,
+            totalAttempts: 1,
+        };
+        memoryCache.set(domain, entry);
+    }
+}
+/**
+ * Get the recommended method for a domain based on past experience.
+ * Returns null if no history exists.
+ */
+export function getRecommendedMethod(url) {
+    const domain = normaliseDomain(url);
+    const entry = memoryCache.get(domain);
+    if (!entry)
+        return null;
+    // Confidence ramp: 1 → 0.3, 5+ → 0.8, 10+ → 0.95
+    let confidence;
+    if (entry.successCount >= 10) {
+        confidence = 0.95;
+    }
+    else if (entry.successCount >= 5) {
+        // Linear interpolation between 0.8 and 0.95 for 5..9
+        confidence = 0.8 + ((entry.successCount - 5) / 5) * 0.15;
+    }
+    else if (entry.successCount >= 2) {
+        // Linear interpolation between 0.3 and 0.8 for 1..4
+        confidence = 0.3 + ((entry.successCount - 1) / 4) * 0.5;
+    }
+    else {
+        confidence = 0.3;
+    }
+    return {
+        method: entry.bestMethod,
+        confidence,
+        avgResponseMs: entry.avgResponseMs,
+        requiresJs: entry.requiresJs,
+    };
+}
+/**
+ * Get full domain memory entry.
+ */
+export function getDomainMemory(domain) {
+    // Accept both raw domain and full URL.
+    const key = normaliseDomain(domain);
+    return memoryCache.get(key) ?? null;
+}
+/**
+ * Get stats about the domain memory cache.
+ */
+export function getDomainMemoryStats() {
+    const entries = [];
+    // LRUCache v11 supports for..of iteration
+    for (const [, value] of memoryCache.entries()) {
+        if (value)
+            entries.push(value);
+    }
+    // Sort by successCount descending, take top 20
+    entries.sort((a, b) => b.successCount - a.successCount);
+    const topDomains = entries.slice(0, 20).map((e) => ({
+        domain: e.domain,
+        bestMethod: e.bestMethod,
+        successCount: e.successCount,
+    }));
+    return {
+        totalDomains: memoryCache.size,
+        topDomains,
+    };
+}
+// ---------------------------------------------------------------------------
+// Postgres sync stubs (wire later)
+// ---------------------------------------------------------------------------
+/** Persist current in-memory cache to Postgres. */
+export async function syncToPostgres() {
+    // TODO: INSERT/UPSERT all entries from memoryCache into a
+    // `domain_memory` table keyed on `domain`.
+}
+/** Load domain memory from Postgres into the in-memory cache on startup. */
+export async function loadFromPostgres() {
+    // TODO: SELECT * FROM domain_memory and populate memoryCache
+    // with each row, respecting the LRU max-size limit.
+}

package/dist/core/domain-verify.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Active domain verification — runtime TLS, HTTP header, and DNS signals.
+ *
+ * Runs during the fetch pipeline for sites that are NOT already in the known
+ * official/established lists.  All network operations have a hard 3-second
+ * timeout and fail-open (any error → null for that section).
+ *
+ * Scoring adds bonus points (0–80) on top of the static source-credibility score.
+ */
+export interface DomainVerification {
+    tls: {
+        valid: boolean;
+        issuer: string;
+        daysRemaining: number;
+        ev: boolean;
+    } | null;
+    headers: {
+        hsts: boolean;
+        csp: boolean;
+        xFrameOptions: boolean;
+        server: string;
+        poweredBy: string | null;
+    };
+    dns: {
+        hasMx: boolean;
+        hasDmarc: boolean;
+        hasSpf: boolean;
+        nameservers: string[];
+    } | null;
+    signals: string[];
+    warnings: string[];
+    verificationScore: number;
+}
+/**
+ * Perform active domain verification (TLS + HTTP headers + DNS).
+ *
+ * @param url   Full URL to verify (e.g. "https://stripe.com")
+ * @param existingHeaders Optional pre-fetched HTTP response headers (avoids a HEAD request)
+ */
+export declare function verifyDomain(url: string, existingHeaders?: Record<string, string>): Promise<DomainVerification>;