@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/ee/extractors/news.js

@@ -0,0 +1,173 @@
+import { tryParseJson } from './shared.js';
+// ---------------------------------------------------------------------------
+// 24–26. News article extractor helper (NYTimes / BBC / CNN)
+// ---------------------------------------------------------------------------
+/** Shared news article extractor using Schema.org JSON-LD + HTML fallbacks. */
+async function extractNewsArticle(html, url, domain) {
+    try {
+        const { load } = await import('cheerio');
+        const $ = load(html);
+        // Try JSON-LD first
+        let jsonLd = null;
+        $('script[type="application/ld+json"]').each((_, el) => {
+            if (jsonLd)
+                return;
+            const raw = $(el).html() || '';
+            const parsed = tryParseJson(raw);
+            const candidates = Array.isArray(parsed) ? parsed : [parsed];
+            for (const item of candidates) {
+                if (item?.['@type'] === 'NewsArticle' || item?.['@type'] === 'Article' || item?.['@type'] === 'WebPage') {
+                    jsonLd = item;
+                    break;
+                }
+                if (item?.['@graph']) {
+                    const g = item['@graph'].find((n) => n?.['@type'] === 'NewsArticle' || n?.['@type'] === 'Article');
+                    if (g) {
+                        jsonLd = g;
+                        break;
+                    }
+                }
+            }
+        });
+        const ogTitle = $('meta[property="og:title"]').attr('content') || '';
+        const title = jsonLd?.headline || ogTitle || $('h1').first().text().trim() || '';
+        if (!title)
+            return null;
+        // Author
+        let author = '';
+        if (jsonLd?.author) {
+            const a = Array.isArray(jsonLd.author) ? jsonLd.author[0] : jsonLd.author;
+            author = typeof a === 'string' ? a : a?.name || '';
+        }
+        if (!author)
+            author = $('meta[name="author"]').attr('content') || $('[itemprop="author"] [itemprop="name"]').first().text().trim() || $('[data-testid="byline"]').first().text().trim() || $('[class*="author"]').first().text().trim() || '';
+        // Date
+        const publishDate = jsonLd?.datePublished || $('meta[property="article:published_time"]').attr('content') || $('time[datetime]').first().attr('datetime') || '';
+        const modifiedDate = jsonLd?.dateModified || $('meta[property="article:modified_time"]').attr('content') || '';
+        // Description / summary
+        const description = jsonLd?.description || $('meta[property="og:description"]').attr('content') || $('meta[name="description"]').attr('content') || '';
+        // Section / category
+        const section = jsonLd?.articleSection || $('meta[property="article:section"]').attr('content') || '';
+        // Keywords / tags
+        const keywords = (() => {
+            if (jsonLd?.keywords) {
+                return (Array.isArray(jsonLd.keywords) ? jsonLd.keywords : String(jsonLd.keywords).split(',')).map((k) => k.trim()).filter(Boolean);
+            }
+            const kwMeta = $('meta[name="keywords"]').attr('content') || '';
+            return kwMeta ? kwMeta.split(',').map(k => k.trim()).filter(Boolean) : [];
+        })();
+        // Article body — try various content selectors
+        let articleBody = '';
+        const contentSelectors = [
+            'article', '[data-testid="article-body"]', '.article-body', '#article-body',
+            '.story-body', '.article__body', '.entry-content', '.post-content',
+            'main article', '.content-body', '[itemprop="articleBody"]',
+        ];
+        for (const selector of contentSelectors) {
+            const el = $(selector).first();
+            if (!el.length)
+                continue;
+            el.find('script, style, nav, aside, .ad, [class*="ad-"], button, figure figcaption').remove();
+            const parts = [];
+            el.find('h1, h2, h3, h4, p, blockquote, ul, ol').each((_, node) => {
+                const tag = node.name;
+                const text = $(node).text().trim();
+                if (!text || text.length < 5)
+                    return;
+                if (tag === 'h1')
+                    return; // Skip — already have title
+                if (tag === 'h2')
+                    parts.push(`## ${text}`);
+                else if (tag === 'h3' || tag === 'h4')
+                    parts.push(`### ${text}`);
+                else if (tag === 'blockquote')
+                    parts.push(`> ${text}`);
+                else
+                    parts.push(text);
+            });
+            articleBody = parts.join('\n\n');
+            if (articleBody.length > 200)
+                break;
+        }
+        // Fallback to og:description
+        const contentBody = articleBody || description;
+        const structured = {
+            title, author, publishDate, modifiedDate,
+            description, section, keywords, url, domain,
+        };
+        const authorLine = author ? `\n**Author:** ${author}` : '';
+        const dateLine = publishDate ? `\n**Published:** ${publishDate.split('T')[0]}` : '';
+        const sectionLine = section ? `\n**Section:** ${section}` : '';
+        const tagsLine = keywords.length ? `\n**Topics:** ${keywords.slice(0, 8).join(', ')}` : '';
+        const cleanContent = `# ${title}${authorLine}${dateLine}${sectionLine}${tagsLine}\n\n${contentBody.substring(0, 10000)}`;
+        return { domain, type: 'article', structured, cleanContent };
+    }
+    catch {
+        return null;
+    }
+}
+export async function nytimesExtractor(html, url) {
+    return extractNewsArticle(html, url, 'nytimes.com');
+}
+export async function bbcExtractor(html, url) {
+    return extractNewsArticle(html, url, 'bbc.com');
+}
+export async function cnnExtractor(html, url) {
+    try {
+        const u = new URL(url);
+        // For homepage — use CNN Lite which has actual headline links
+        if (u.pathname === '/' || u.pathname === '' || u.hostname === 'lite.cnn.com') {
+            const liteResp = await fetch('https://lite.cnn.com', { headers: { 'User-Agent': 'webpeel/0.21' } });
+            if (liteResp.ok) {
+                const liteHtml = await liteResp.text();
+                const headlines = [];
+                const matches = liteHtml.matchAll(/<a[^>]+href="([^"]*)"[^>]*>([^<]+)<\/a>/g);
+                for (const m of matches) {
+                    const href = m[1].trim();
+                    const text = m[2].trim();
+                    // CNN Lite article links contain year patterns like /2026/
+                    if (/\/20\d\d\//.test(href) && text.length > 10) {
+                        const fullUrl = href.startsWith('http') ? href : `https://www.cnn.com${href}`;
+                        headlines.push(`- [${text}](${fullUrl})`);
+                    }
+                }
+                if (headlines.length > 5) {
+                    return {
+                        domain: 'cnn.com',
+                        type: 'headlines',
+                        structured: { headlines: headlines.length, source: 'cnn-lite' },
+                        cleanContent: `# 📰 CNN — Top Headlines\n\n${headlines.slice(0, 20).join('\n')}\n\n---\n*Source: CNN Lite*`,
+                    };
+                }
+            }
+        }
+        // For article pages — try CNN Lite version of the same URL
+        if (/\/20\d\d\//.test(u.pathname)) {
+            const liteUrl = `https://lite.cnn.com${u.pathname}`;
+            const liteResp = await fetch(liteUrl, { headers: { 'User-Agent': 'webpeel/0.21' } });
+            if (liteResp.ok) {
+                const liteHtml = await liteResp.text();
+                const { load } = await import('cheerio');
+                const $l = load(liteHtml);
+                const title = $l('h1').first().text().trim();
+                const paragraphs = [];
+                $l('p').each((_, el) => {
+                    const text = $l(el).text().trim();
+                    if (text.length > 20)
+                        paragraphs.push(text);
+                });
+                if (title && paragraphs.length > 0) {
+                    return {
+                        domain: 'cnn.com',
+                        type: 'article',
+                        structured: { title, paragraphs: paragraphs.length, source: 'cnn-lite' },
+                        cleanContent: `# ${title}\n\n${paragraphs.join('\n\n')}\n\n---\n*Source: CNN*`,
+                    };
+                }
+            }
+        }
+    }
+    catch { /* fall through to standard extractor */ }
+    // Fallback to standard news article extractor (works if HTML has content)
+    return extractNewsArticle(html, url, 'cnn.com');
+}

package/dist/ee/extractors/npm.d.ts

@@ -0,0 +1,2 @@
+import type { DomainExtractResult } from './types.js';
+export declare function npmExtractor(_html: string, url: string): Promise<DomainExtractResult | null>;

package/dist/ee/extractors/npm.js

@@ -0,0 +1,86 @@
+import { simpleFetch } from '../../core/fetcher.js';
+import { fetchJson } from './shared.js';
+// ---------------------------------------------------------------------------
+// 9. NPM extractor (npm registry API)
+// ---------------------------------------------------------------------------
+export async function npmExtractor(_html, url) {
+    const urlObj = new URL(url);
+    const path = urlObj.pathname;
+    // Match /package/name or /package/@scope/name
+    const packageMatch = path.match(/\/package\/((?:@[^/]+\/)?[^/]+)/);
+    if (!packageMatch)
+        return null;
+    const packageName = packageMatch[1];
+    try {
+        const apiUrl = `https://registry.npmjs.org/${encodeURIComponent(packageName)}`;
+        const data = await fetchJson(apiUrl);
+        if (!data?.name)
+            return null;
+        const latest = data['dist-tags']?.latest;
+        const latestVersion = latest ? data.versions?.[latest] : null;
+        // Get download counts
+        let downloads = null;
+        try {
+            downloads = await fetchJson(`https://api.npmjs.org/downloads/point/last-week/${encodeURIComponent(packageName)}`);
+        }
+        catch { /* optional */ }
+        const structured = {
+            title: `${data.name}@${latest || 'unknown'}`,
+            name: data.name,
+            description: data.description || '',
+            version: latest || 'unknown',
+            license: latestVersion?.license || data.license || 'N/A',
+            homepage: data.homepage || latestVersion?.homepage || null,
+            repository: typeof data.repository === 'string' ? data.repository : data.repository?.url || null,
+            author: typeof data.author === 'string' ? data.author : data.author?.name || '',
+            keywords: data.keywords || [],
+            weeklyDownloads: downloads?.downloads || 0,
+            dependencies: Object.keys(latestVersion?.dependencies || {}),
+            devDependencies: Object.keys(latestVersion?.devDependencies || {}),
+            maintainers: (data.maintainers || []).map((m) => m.name || m).slice(0, 10),
+            created: data.time?.created || undefined,
+            modified: data.time?.modified || undefined,
+        };
+        // Include README if available (some packages have it, some don't)
+        let readmeText = data.readme && data.readme.length > 10 ? data.readme.slice(0, 5000) : '';
+        // If no README in registry, try fetching from unpkg.com
+        if (!readmeText) {
+            try {
+                const unpkgUrl = `https://unpkg.com/${encodeURIComponent(packageName)}/README.md`;
+                const readmeResult = await simpleFetch(unpkgUrl, undefined, 10000);
+                if (readmeResult?.html && readmeResult.html.length > 10 && !readmeResult.html.trim().startsWith('<')) {
+                    readmeText = readmeResult.html.slice(0, 5000);
+                }
+            }
+            catch { /* README from unpkg optional */ }
+        }
+        // Add to structured data
+        structured.readme = readmeText;
+        const keywordsLine = structured.keywords.length ? `\n**Keywords:** ${structured.keywords.join(', ')}` : '';
+        // Show ALL dependencies (not capped at 15)
+        const depsLine = structured.dependencies.length
+            ? `\n**Dependencies (${structured.dependencies.length}):** ${structured.dependencies.join(', ')}`
+            : '';
+        const devDepsLine = structured.devDependencies.length
+            ? `\n**Dev Dependencies (${structured.devDependencies.length}):** ${structured.devDependencies.slice(0, 10).join(', ')}${structured.devDependencies.length > 10 ? '...' : ''}`
+            : '';
+        const repoLine = structured.repository ? `\n**Repository:** ${structured.repository.replace('git+', '').replace('.git', '')}` : '';
+        const homepageLine = structured.homepage ? `\n**Homepage:** ${structured.homepage}` : '';
+        const datesLine = structured.created ? `\n**Created:** ${structured.created?.split('T')[0] || 'N/A'} | **Last modified:** ${structured.modified?.split('T')[0] || 'N/A'}` : '';
+        const readmeSection = readmeText
+            ? `\n\n### README\n\n${readmeText}`
+            : '';
+        const cleanContent = `# 📦 ${structured.name}@${structured.version}
+
+${structured.description}
+
+**License:** ${structured.license} | **Weekly Downloads:** ${structured.weeklyDownloads?.toLocaleString() || 'N/A'}
+**Author:** ${structured.author || 'N/A'} | **Maintainers:** ${structured.maintainers.join(', ') || 'N/A'}${keywordsLine}${depsLine}${devDepsLine}${repoLine}${homepageLine}${datesLine}${readmeSection}`;
+        return { domain: 'npmjs.com', type: 'package', structured, cleanContent };
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'NPM API failed:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}

package/dist/ee/extractors/pdf.d.ts

@@ -0,0 +1,2 @@
+import type { DomainExtractResult } from './types.js';
+export declare function pdfExtractor(_html: string, url: string): Promise<DomainExtractResult | null>;

package/dist/ee/extractors/pdf.js

@@ -0,0 +1,108 @@
+// ---------------------------------------------------------------------------
+// 30. PDF extractor (URL-based detection) — downloads and extracts real text
+// ---------------------------------------------------------------------------
+const PDF_MAX_BYTES = 50 * 1024 * 1024; // 50 MB
+const PDF_TRUNCATE_CHARS = 100_000;
+export async function pdfExtractor(_html, url) {
+    try {
+        const urlObj = new URL(url);
+        const filename = urlObj.pathname.split('/').pop() || 'document.pdf';
+        const hostname = urlObj.hostname;
+        // Download the PDF
+        let buffer;
+        let finalContentType = 'application/pdf';
+        try {
+            const response = await fetch(url, {
+                headers: { 'User-Agent': 'Mozilla/5.0 (compatible; WebPeel/1.0)' },
+                signal: AbortSignal.timeout(30000),
+            });
+            if (!response.ok) {
+                if (process.env.DEBUG)
+                    console.debug('[webpeel]', `PDF download failed: HTTP ${response.status}`);
+                return null; // Let the normal pipeline handle it
+            }
+            finalContentType = response.headers.get('content-type') || 'application/pdf';
+            // Verify it's actually a PDF (content-type or URL)
+            const isPdf = finalContentType.toLowerCase().includes('pdf') || /\.pdf(\?|$|#)/i.test(url);
+            if (!isPdf)
+                return null;
+            const arrayBuffer = await response.arrayBuffer();
+            buffer = Buffer.from(arrayBuffer);
+        }
+        catch (downloadErr) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', 'PDF download error:', downloadErr instanceof Error ? downloadErr.message : downloadErr);
+            return null; // Let the normal pipeline handle it
+        }
+        // Size guard
+        if (buffer.length > PDF_MAX_BYTES) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', `PDF too large (${buffer.length} bytes), falling back to stub`);
+            return null;
+        }
+        // Extract text via pdf-parse
+        const { extractPdf } = await import('../../core/pdf.js');
+        let pdf;
+        try {
+            pdf = await extractPdf(buffer);
+        }
+        catch (parseErr) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', 'PDF parse failed:', parseErr instanceof Error ? parseErr.message : parseErr);
+            return null; // Let the normal pipeline handle it
+        }
+        // Normalize whitespace (pdf-parse emits lots of blank lines)
+        let text = (pdf.text || '')
+            .replace(/\r\n/g, '\n')
+            .replace(/\n{3,}/g, '\n\n')
+            .replace(/[ \t]+/g, ' ')
+            .trim();
+        // Truncate very large documents
+        let truncated = false;
+        if (text.length > PDF_TRUNCATE_CHARS) {
+            text = text.slice(0, PDF_TRUNCATE_CHARS);
+            truncated = true;
+        }
+        if (!text) {
+            // Scanned/image-only PDF — return a clear message rather than empty content
+            const emptyNote = `## 📄 ${filename}\n\n*This PDF appears to be a scanned document (image-only). No extractable text was found.*\n\n**Source:** ${url}`;
+            return {
+                domain: hostname,
+                type: 'pdf',
+                structured: { title: filename, url, pages: pdf.pages, contentType: finalContentType },
+                cleanContent: emptyNote,
+            };
+        }
+        // Build markdown output
+        const titleRaw = pdf.metadata?.title || '';
+        const title = titleRaw || filename.replace(/\.pdf$/i, '') || 'PDF Document';
+        const metaParts = [];
+        if (pdf.metadata?.author)
+            metaParts.push(`**Author:** ${pdf.metadata.author}`);
+        if (pdf.pages)
+            metaParts.push(`**Pages:** ${pdf.pages}`);
+        metaParts.push(`**Source:** ${url}`);
+        const header = titleRaw ? `# ${titleRaw}\n\n` : '';
+        const metaBlock = metaParts.join(' | ') + '\n\n';
+        const truncNote = truncated ? '\n\n*[Content truncated — document exceeds 100,000 characters]*' : '';
+        const cleanContent = header + metaBlock + text + truncNote;
+        return {
+            domain: hostname,
+            type: 'pdf',
+            structured: {
+                title,
+                filename,
+                url,
+                pages: pdf.pages,
+                contentType: finalContentType,
+                ...pdf.metadata,
+            },
+            cleanContent,
+        };
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'PDF extractor failed:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}

package/dist/ee/extractors/pinterest.d.ts

@@ -0,0 +1,2 @@
+import type { DomainExtractResult } from './types.js';
+export declare function pinterestExtractor(_html: string, url: string): Promise<DomainExtractResult | null>;

package/dist/ee/extractors/pinterest.js

@@ -0,0 +1,34 @@
+import { fetchJson } from './shared.js';
+// ---------------------------------------------------------------------------
+// 23. Pinterest extractor (oEmbed)
+// ---------------------------------------------------------------------------
+export async function pinterestExtractor(_html, url) {
+    try {
+        const oembedUrl = `https://www.pinterest.com/oembed/?url=${encodeURIComponent(url)}`;
+        const data = await fetchJson(oembedUrl);
+        if (!data || !data.title)
+            return null;
+        // Detect content type from URL
+        const isPinPage = /\/pin\//.test(url);
+        const isBoardPage = /\/[^/]+\/[^/]+\/?$/.test(new URL(url).pathname) && !isPinPage;
+        const contentType = isPinPage ? 'pin' : isBoardPage ? 'board' : 'profile';
+        const structured = {
+            title: data.title,
+            description: data.description || '',
+            type: contentType,
+            thumbnailUrl: data.thumbnail_url || '',
+            authorName: data.author_name || '',
+            authorUrl: data.author_url || '',
+            provider: 'Pinterest',
+        };
+        const typeEmoji = contentType === 'pin' ? '📌' : contentType === 'board' ? '📋' : '👤';
+        const descLine = structured.description ? `\n\n${structured.description}` : '';
+        const cleanContent = `## ${typeEmoji} Pinterest ${contentType}: ${structured.title}${descLine}\n\n**By:** [${structured.authorName}](${structured.authorUrl})\n**URL:** ${url}`;
+        return { domain: 'pinterest.com', type: contentType, structured, cleanContent };
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'Pinterest oEmbed failed:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}

package/dist/ee/extractors/polymarket.d.ts

@@ -0,0 +1,2 @@
+import type { DomainExtractResult } from './types.js';
+export declare function polymarketExtractor(_html: string, url: string): Promise<DomainExtractResult | null>;