@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/ee/extractors/kayak-cars.js

@@ -0,0 +1,270 @@
+// ---------------------------------------------------------------------------
+// Kayak Car Rental extractor
+// ---------------------------------------------------------------------------
+export async function kayakCarRentalExtractor(_html, url) {
+    if (!url.includes('/cars/'))
+        return null;
+    // Rental company homepage URLs
+    const rentalCompanyUrls = {
+        'Hertz': 'https://www.hertz.com',
+        'Budget': 'https://www.budget.com',
+        'Avis': 'https://www.avis.com',
+        'Enterprise': 'https://www.enterprise.com',
+        'National': 'https://www.nationalcar.com',
+        'Alamo': 'https://www.alamo.com',
+        'Dollar': 'https://www.dollar.com',
+        'Thrifty': 'https://www.thrifty.com',
+        'Sixt': 'https://www.sixt.com',
+        'Fox': 'https://www.foxrentacar.com',
+        'Payless': 'https://www.paylesscar.com',
+        'Turn': 'https://www.turn.com',
+        'EconomyBookings': 'https://www.economybookings.com',
+        'Priceline': 'https://www.priceline.com',
+        'Expedia': 'https://www.expedia.com',
+        'Turo': 'https://www.turo.com',
+        'KAYAK': 'https://www.kayak.com',
+        'Booking.com': 'https://www.booking.com',
+        'DiscoverCars': 'https://www.discovercars.com',
+        'RentalCars': 'https://www.rentalcars.com',
+        'Car Rental 8': 'https://www.carrental8.com',
+        'Hotwire': 'https://www.hotwire.com',
+    };
+    function getCompanyUrl(company) {
+        return rentalCompanyUrls[company] || `https://www.kayak.com`;
+    }
+    // Parse dates from URL: /cars/Location/YYYY-MM-DD/YYYY-MM-DD
+    let numDays = 1;
+    let pickupDate = '';
+    let dropoffDate = '';
+    let locationName = '';
+    const dateMatch = url.match(/\/cars\/([^/]+)\/(\d{4}-\d{2}-\d{2})\/(\d{4}-\d{2}-\d{2})/);
+    if (dateMatch) {
+        locationName = decodeURIComponent(dateMatch[1]);
+        pickupDate = dateMatch[2];
+        dropoffDate = dateMatch[3];
+        const pickup = new Date(pickupDate);
+        const dropoff = new Date(dropoffDate);
+        numDays = Math.max(1, Math.round((dropoff.getTime() - pickup.getTime()) / (1000 * 60 * 60 * 24)));
+    }
+    // Format date range for display (e.g. "Apr 1–3")
+    function formatDateRange(from, to) {
+        if (!from || !to)
+            return '';
+        const fromDate = new Date(from + 'T12:00:00');
+        const toDate = new Date(to + 'T12:00:00');
+        const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
+        const fromMonth = months[fromDate.getUTCMonth()];
+        const toMonth = months[toDate.getUTCMonth()];
+        const fromDay = fromDate.getUTCDate();
+        const toDay = toDate.getUTCDate();
+        if (fromMonth === toMonth)
+            return `${fromMonth} ${fromDay}–${toDay}`;
+        return `${fromMonth} ${fromDay}–${toMonth} ${toDay}`;
+    }
+    // Process content: strip HTML if needed
+    let text = _html;
+    if (text.includes('<!DOCTYPE') || text.includes('<html')) {
+        text = text
+            .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+            .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+            .replace(/<[^>]+>/g, '\n')
+            .replace(/&amp;/g, '&')
+            .replace(/&lt;/g, '<')
+            .replace(/&gt;/g, '>')
+            .replace(/&#\d+;/g, '')
+            .replace(/\n{2,}/g, '\n');
+    }
+    const lines = text.split('\n').map(l => l.trim()).filter(Boolean);
+    const KNOWN_COMPANIES = ['Hertz', 'Budget', 'Avis', 'Enterprise', 'National', 'Alamo', 'Dollar', 'Thrifty', 'Sixt', 'Fox', 'Payless', 'Turn', 'EconomyBookings', 'Priceline', 'Expedia', 'Turo', 'KAYAK', 'Booking.com', 'DiscoverCars', 'RentalCars', 'Car Rental 8', 'Hotwire'];
+    const listings = [];
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        // Detect: "or similar {Class}" — this signals a standard car rental listing
+        // The car name is the line BEFORE "or similar"
+        const orSimilarMatch = line.match(/^or similar\s+(.+)$/);
+        if (orSimilarMatch) {
+            const carClass = orSimilarMatch[1].trim();
+            const carName = i > 0 ? lines[i - 1] : '';
+            if (!carName || carName.length > 60)
+                continue;
+            // Look ahead for: pickup location, rating, company, price
+            let location = '';
+            let distanceFromCenter = '';
+            let rating = null;
+            let company = '';
+            let totalPrice = 0;
+            let cancellation = '';
+            for (let j = i + 1; j < Math.min(i + 30, lines.length); j++) {
+                const l = lines[j];
+                // Pickup location
+                if (!location && l.startsWith('Pick-up')) {
+                    const locMatch = l.match(/Pick-up (?:city|airport):\s*(.+)/);
+                    if (locMatch)
+                        location = locMatch[1].trim();
+                    continue;
+                }
+                // Distance from center
+                if (!distanceFromCenter) {
+                    const distM = l.match(/^([\d.]+)\s+mi\s+from\s+city\s+center/);
+                    if (distM) {
+                        distanceFromCenter = `${distM[1]} mi from city center`;
+                        continue;
+                    }
+                }
+                // Rating (number like "9.2", "8.5", "7.2")
+                if (rating === null) {
+                    const ratingM = l.match(/^(\d+\.\d+)$/);
+                    if (ratingM) {
+                        rating = parseFloat(ratingM[1]);
+                        continue;
+                    }
+                }
+                // Company from "X offer from {Company}" or "{Company}" line
+                if (!company) {
+                    const offerMatch = l.match(/offer from (.+)$/);
+                    if (offerMatch) {
+                        company = offerMatch[1].trim();
+                        continue;
+                    }
+                    // Also detect company name standalone
+                    for (const c of KNOWN_COMPANIES) {
+                        if (l === c) {
+                            company = c;
+                            break;
+                        }
+                    }
+                    if (company)
+                        continue;
+                }
+                // Cancellation policy
+                if (!cancellation && (l.includes('Free cancellation') || l.includes('No free cancellation'))) {
+                    cancellation = l;
+                    continue;
+                }
+                // Price — "$NNN" followed by "Total"
+                const priceM = l.match(/^\$(\d[\d,]*)$/);
+                if (priceM) {
+                    const nextLine = lines[j + 1] || '';
+                    if (nextLine === 'Total' || nextLine.includes('Total')) {
+                        totalPrice = parseInt(priceM[1].replace(',', ''));
+                        break;
+                    }
+                }
+                // Also catch price on same line
+                const inlinePriceM = l.match(/\$(\d[\d,]*)\s*Total/);
+                if (inlinePriceM) {
+                    totalPrice = parseInt(inlinePriceM[1].replace(',', ''));
+                    break;
+                }
+                // Stop if we hit another car listing marker
+                if (l.match(/^or similar\s/) || l === 'Show more results')
+                    break;
+            }
+            if (carName && totalPrice > 0) {
+                const normalizedClass = carClass.replace('Full size', 'Full-size');
+                listings.push({
+                    name: carName,
+                    carClass: normalizedClass,
+                    totalPrice,
+                    perDayPrice: Math.round(totalPrice / numDays),
+                    company: company || 'Unknown',
+                    location: location || 'See booking',
+                    distanceFromCenter,
+                    rating,
+                    cancellation,
+                    isTuro: false,
+                });
+            }
+        }
+    }
+    // Deduplicate: first prefer listings with real company info over "Unknown"
+    // Key by name+price; keep the one with best data
+    const byKey = new Map();
+    for (const c of listings) {
+        const key = `${c.name.toLowerCase()}-${c.totalPrice}`;
+        const existing = byKey.get(key);
+        if (!existing) {
+            byKey.set(key, c);
+        }
+        else {
+            // Prefer non-Unknown company, or same company with more info
+            if (existing.company === 'Unknown' && c.company !== 'Unknown') {
+                byKey.set(key, c);
+            }
+        }
+    }
+    const unique = Array.from(byKey.values());
+    if (unique.length === 0)
+        return null;
+    // Filter out Unknown company entries if the total found from page suggests more results exist
+    // Also filter them only if they have no location info (these are likely ad/promo extractions)
+    const knownCompanyListings = unique.filter(c => c.company !== 'Unknown');
+    const finalListings = knownCompanyListings.length > 0 ? knownCompanyListings : unique;
+    // Sort by price
+    finalListings.sort((a, b) => a.totalPrice - b.totalPrice);
+    // Get total count from page if mentioned
+    let totalFound = unique.length;
+    for (const l of lines) {
+        const m = l.match(/^(\d+)\s+results?$/);
+        if (m) {
+            totalFound = parseInt(m[1]);
+            break;
+        }
+        const m2 = l.match(/(\d+)\s+cars?\s+found/);
+        if (m2) {
+            totalFound = parseInt(m2[1]);
+            break;
+        }
+    }
+    // Format location name nicely (e.g. "Punta-Gorda,FL-c34451" → "Punta Gorda, FL")
+    function formatLocation(loc) {
+        return loc
+            .replace(/-c\d+$/, '') // remove trailing "-c12345"
+            .replace(/-/g, ' ') // hyphens to spaces
+            .replace(/,(\S)/g, ', $1'); // ensure space after comma
+    }
+    const dateRange = formatDateRange(pickupDate, dropoffDate);
+    const displayLocation = formatLocation(locationName);
+    const daysLabel = numDays === 1 ? '1 day' : `${numDays} days`;
+    const md = [
+        `# 🚗 Car Rentals — ${displayLocation} · ${dateRange} (${daysLabel})`,
+        '',
+        `*${totalFound} cars found · Source: [Kayak](${url})*`,
+        `*Free cancellation available on most rentals*`,
+        '',
+    ];
+    for (let idx = 0; idx < finalListings.length; idx++) {
+        const c = finalListings[idx];
+        md.push(`## ${idx + 1}. ${c.name} (${c.carClass}) — $${c.totalPrice} total · $${c.perDayPrice}/day`);
+        if (c.distanceFromCenter) {
+            md.push(`📍 ${c.distanceFromCenter}`);
+        }
+        else if (c.location && c.location !== 'See booking') {
+            md.push(`📍 ${c.location}`);
+        }
+        const ratingStr = c.rating !== null ? ` · Rating: ${c.rating}` : '';
+        md.push(`🏪 via ${c.company}${ratingStr}`);
+        if (c.cancellation)
+            md.push(`✅ ${c.cancellation}`);
+        md.push(`🔍 [See price on Kayak](${url})`);
+        md.push(`🛒 [Book on ${c.company}](${getCompanyUrl(c.company)})`);
+        md.push('');
+    }
+    md.push('---');
+    md.push(`📌 *Prices verified via [Kayak](${url}). Click "See price" to confirm current rate, then book with the rental company.*`);
+    return {
+        domain: 'kayak.com/cars',
+        type: 'car-rental',
+        structured: {
+            cars: finalListings,
+            location: displayLocation,
+            pickupDate,
+            dropoffDate,
+            numDays,
+            totalFound,
+            source: 'Kayak',
+            sourceUrl: url,
+        },
+        cleanContent: md.join('\n'),
+    };
+}

package/dist/ee/extractors/linkedin.d.ts

@@ -0,0 +1,2 @@
+import type { DomainExtractResult } from './types.js';
+export declare function linkedinExtractor(html: string, url: string): Promise<DomainExtractResult | null>;

package/dist/ee/extractors/linkedin.js

@@ -0,0 +1,113 @@
+import { tryParseJson } from './shared.js';
+// ---------------------------------------------------------------------------
+// 17. LinkedIn extractor
+// ---------------------------------------------------------------------------
+export async function linkedinExtractor(html, url) {
+    try {
+        const { load } = await import('cheerio');
+        const $ = load(html);
+        // Detect page type from URL first
+        const urlObj = new URL(url);
+        const pathParts = urlObj.pathname.split('/').filter(Boolean);
+        const pageType = pathParts[0] === 'company' ? 'company'
+            : pathParts[0] === 'in' ? 'profile'
+                : pathParts[0] === 'jobs' ? 'job'
+                    : 'page';
+        // Detect if we're on the authwall (LinkedIn redirects unauthenticated requests)
+        const isAuthwall = html.includes('authwall') || html.includes('Join LinkedIn') || html.includes('Sign in') && !html.includes('linkedin.com/in/');
+        // --- Try parsing meta tags / JSON-LD from the HTML ---
+        let jsonLd = null;
+        $('script[type="application/ld+json"]').each((_, el) => {
+            if (jsonLd)
+                return;
+            const raw = $(el).html() || '';
+            const parsed = tryParseJson(raw);
+            if (parsed?.['@type'] === 'Person' || parsed?.['@type'] === 'Organization')
+                jsonLd = parsed;
+        });
+        const ogTitle = $('meta[property="og:title"]').attr('content') || '';
+        const ogDescription = $('meta[property="og:description"]').attr('content') || '';
+        const ogImage = $('meta[property="og:image"]').attr('content') || '';
+        const metaDescription = $('meta[name="description"]').attr('content') || '';
+        let name = jsonLd?.name || ogTitle.replace(/ \| LinkedIn$/, '').replace(/Sign Up \| LinkedIn$/, '').trim() || '';
+        // When on authwall, discard authwall-specific meta data
+        let headline = isAuthwall ? (jsonLd?.jobTitle || '') : (jsonLd?.jobTitle || metaDescription?.split('|')?.[0]?.trim() || ogDescription || '');
+        let description = isAuthwall ? (jsonLd?.description || '') : (jsonLd?.description || ogDescription || '');
+        let location = $('[class*="location"]').first().text().trim() || jsonLd?.address?.addressLocality || '';
+        // --- If authwall or no useful data, try direct HTTPS fetch with minimal headers ---
+        // LinkedIn returns rich og: meta tags when fetched with a plain browser UA (no Sec-Fetch-* noise)
+        if (!name || isAuthwall || name.toLowerCase().includes('sign up') || name.toLowerCase().includes('linkedin')) {
+            try {
+                const { default: httpsLI } = await import('https');
+                const { gunzip } = await import('zlib');
+                const linkedInHtml = await new Promise((resolve, reject) => {
+                    const req = httpsLI.request({
+                        hostname: 'www.linkedin.com',
+                        path: urlObj.pathname,
+                        method: 'GET',
+                        headers: {
+                            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                            'Accept': 'text/html,application/xhtml+xml',
+                            'Accept-Language': 'en-US,en;q=0.9',
+                            'Accept-Encoding': 'gzip, deflate',
+                        },
+                    }, (res) => {
+                        if (res.statusCode && res.statusCode >= 400) {
+                            reject(new Error(`HTTP ${res.statusCode}`));
+                            res.resume();
+                            return;
+                        }
+                        const chunks = [];
+                        res.on('data', (chunk) => chunks.push(chunk));
+                        res.on('end', () => {
+                            const buf = Buffer.concat(chunks);
+                            const enc = res.headers['content-encoding'] || '';
+                            if (enc === 'gzip') {
+                                gunzip(buf, (err, decoded) => err ? reject(err) : resolve(decoded.toString('utf8')));
+                            }
+                            else {
+                                resolve(buf.toString('utf8'));
+                            }
+                        });
+                    });
+                    req.on('error', reject);
+                    setTimeout(() => req.destroy(new Error('timeout')), 10000);
+                    req.end();
+                });
+                if (linkedInHtml) {
+                    const $li = load(linkedInHtml);
+                    const liOgTitle = $li('meta[property="og:title"]').attr('content') || '';
+                    const liOgDesc = $li('meta[property="og:description"]').attr('content') || '';
+                    // Only use if it has real profile data (not authwall)
+                    if (liOgTitle && !liOgTitle.toLowerCase().includes('sign up') && !liOgTitle.toLowerCase().includes('join linkedin')) {
+                        // "Name - Headline | LinkedIn" or "Name | LinkedIn"
+                        const titleParts = liOgTitle.replace(/ \| LinkedIn$/, '').split(/\s*[-–]\s*/);
+                        if (titleParts[0])
+                            name = titleParts[0].trim();
+                        if (titleParts[1])
+                            headline = titleParts[1].trim();
+                        if (liOgDesc)
+                            description = liOgDesc;
+                    }
+                }
+            }
+            catch { /* direct fetch optional */ }
+        }
+        if (!name)
+            return null;
+        const structured = {
+            name, headline, description, location, pageType,
+            image: ogImage, url,
+        };
+        const typeLine = pageType === 'company' ? '🏢' : pageType === 'profile' ? '👤' : '🔗';
+        const locationLine = location ? `\n📍 ${location}` : '';
+        const headlineLine = headline && headline !== name ? `\n*${headline}*` : '';
+        const descriptionLine = description ? `\n\n${description}` : '';
+        const authNote = '\n\n⚠️ Full LinkedIn profiles require authentication. Use /v1/session to log in first.';
+        const cleanContent = `# ${typeLine} ${name} — LinkedIn${headlineLine}${locationLine}${descriptionLine}${authNote}`;
+        return { domain: 'linkedin.com', type: pageType, structured, cleanContent };
+    }
+    catch {
+        return null;
+    }
+}

package/dist/ee/extractors/medium.d.ts

@@ -0,0 +1,2 @@
+import type { DomainExtractResult } from './types.js';
+export declare function mediumExtractor(html: string, url: string): Promise<DomainExtractResult | null>;

package/dist/ee/extractors/medium.js

@@ -0,0 +1,130 @@
+import { tryParseJson } from './shared.js';
+// ---------------------------------------------------------------------------
+// 13. Medium Articles extractor
+// ---------------------------------------------------------------------------
+export async function mediumExtractor(html, url) {
+    try {
+        const { load } = await import('cheerio');
+        const $ = load(html);
+        // JSON-LD
+        let jsonLdData = null;
+        $('script[type="application/ld+json"]').each((_, el) => {
+            if (jsonLdData)
+                return;
+            const raw = $(el).html() || '';
+            const parsed = tryParseJson(raw);
+            if (parsed?.['@type'] === 'NewsArticle' || parsed?.['@type'] === 'Article')
+                jsonLdData = parsed;
+        });
+        const title = jsonLdData?.headline ||
+            $('meta[property="og:title"]').attr('content') ||
+            $('h1').first().text().trim() || '';
+        if (!title)
+            return null;
+        const author = jsonLdData?.author?.name ||
+            $('meta[name="author"]').attr('content') ||
+            $('[data-testid="authorName"]').text().trim() ||
+            $('a[rel="author"]').first().text().trim() || '';
+        const publishDate = jsonLdData?.datePublished ||
+            $('meta[property="article:published_time"]').attr('content') || '';
+        const readingTime = $('[data-testid="storyReadTime"]').text().trim() ||
+            $('span').filter((_, el) => $(el).text().includes('min read')).first().text().trim() || '';
+        const description = jsonLdData?.description ||
+            $('meta[property="og:description"]').attr('content') || '';
+        // Publication name — subdomain (towardsdatascience.medium.com), meta tags, or breadcrumb
+        let publication = '';
+        try {
+            const urlObj2 = new URL(url);
+            const hostname = urlObj2.hostname;
+            if (hostname !== 'medium.com' && hostname !== 'www.medium.com' && hostname.endsWith('.medium.com')) {
+                publication = hostname.replace('.medium.com', '').replace(/-/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase());
+            }
+        }
+        catch { /* ignore */ }
+        if (!publication) {
+            publication = $('[data-testid="publicationName"]').text().trim() ||
+                $('a[data-testid="publicationName"]').text().trim() ||
+                $('meta[property="article:section"]').attr('content') ||
+                $('a[href*="/tag/"]').first().text().trim() || '';
+        }
+        // Author bio — usually shown in an author card or bio section
+        const authorBio = $('[data-testid="authorBio"]').text().trim() ||
+            $('p[class*="bio"]').first().text().trim() ||
+            $('[aria-label="authorBio"]').text().trim() || '';
+        // Clap count — Medium shows clap button with count
+        let clapCount = '';
+        $('button[data-testid="storyClaps"], button[aria-label*="clap"]').each((_, el) => {
+            const txt = $(el).text().trim();
+            if (txt && /\d/.test(txt)) {
+                clapCount = txt;
+                return false;
+            }
+        });
+        if (!clapCount) {
+            // Fallback: find spans that look like clap counts (e.g., "2.4K")
+            $('span').filter((_, el) => {
+                const label = $(el).closest('[aria-label]').attr('aria-label') || '';
+                return label.toLowerCase().includes('clap');
+            }).each((_, el) => {
+                const txt = $(el).text().trim();
+                if (txt && /\d/.test(txt)) {
+                    clapCount = txt;
+                    return false;
+                }
+            });
+        }
+        // Extract article body — Medium puts content in <article> or section
+        let articleBody = '';
+        const articleEl = $('article').first();
+        if (articleEl.length) {
+            // Remove nav, aside, buttons, author-card, footer sections
+            articleEl.find('nav, aside, button, [data-testid="navbar"], footer, [data-testid="authorCard"]').remove();
+            // Get paragraphs and headings
+            const parts = [];
+            articleEl.find('h1, h2, h3, h4, p, blockquote, pre, li, figure figcaption').each((_, el) => {
+                const tag = el.name;
+                const text = $(el).text().trim();
+                if (!text || text.length < 5)
+                    return;
+                if (tag === 'h1' || tag === 'h2')
+                    parts.push(`## ${text}`);
+                else if (tag === 'h3' || tag === 'h4')
+                    parts.push(`### ${text}`);
+                else if (tag === 'blockquote')
+                    parts.push(`> ${text}`);
+                else if (tag === 'pre')
+                    parts.push('```\n' + text + '\n```');
+                else if (tag === 'figcaption')
+                    parts.push(`*${text}*`);
+                else
+                    parts.push(text);
+            });
+            articleBody = parts.join('\n\n');
+        }
+        // Fallback to og:description if no body
+        const contentBody = articleBody || description;
+        const structured = {
+            title,
+            author,
+            authorBio,
+            publishDate,
+            readingTime,
+            description,
+            publication,
+            clapCount,
+            url,
+        };
+        const authorLine = author ? `\n**Author:** ${author}` : '';
+        const bioLine = authorBio ? `\n**Author Bio:** ${authorBio}` : '';
+        const dateLine = publishDate ? `\n**Published:** ${publishDate.split('T')[0]}` : '';
+        const timeLine = readingTime ? `\n**Reading time:** ${readingTime}` : '';
+        const pubLine = publication ? `\n**Publication:** ${publication}` : '';
+        const clapsLine = clapCount ? `\n**Claps:** ${clapCount}` : '';
+        // No hard character cap — let the pipeline's budget/maxTokens handle truncation
+        const cleanContent = `# ${title}${authorLine}${bioLine}${dateLine}${timeLine}${pubLine}${clapsLine}\n\n${contentBody}`;
+        return { domain: 'medium.com', type: 'article', structured, cleanContent };
+    }
+    catch {
+        return null;
+    }
+}

package/dist/ee/extractors/news.d.ts

@@ -0,0 +1,4 @@
+import type { DomainExtractResult } from './types.js';
+export declare function nytimesExtractor(html: string, url: string): Promise<DomainExtractResult | null>;
+export declare function bbcExtractor(html: string, url: string): Promise<DomainExtractResult | null>;
+export declare function cnnExtractor(html: string, url: string): Promise<DomainExtractResult | null>;