@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/ee/extractors/substack.js

@@ -0,0 +1,115 @@
+import { simpleFetch } from '../../core/fetcher.js';
+import { tryParseJson } from './shared.js';
+// ---------------------------------------------------------------------------
+// 14. Substack Posts extractor
+// ---------------------------------------------------------------------------
+export async function substackExtractor(html, url) {
+    try {
+        const { load } = await import('cheerio');
+        // Handle open.substack.com/pub/{publication}/p/{slug} redirect URLs
+        // These are share links that redirect to the actual post. Redirect to the real URL.
+        const urlObj = new URL(url);
+        let workingHtml = html;
+        let workingUrl = url;
+        if (urlObj.hostname === 'open.substack.com') {
+            const openMatch = urlObj.pathname.match(/\/pub\/([^/]+)\/p\/([^/]+)/);
+            if (openMatch) {
+                const [, publication, slug] = openMatch;
+                const actualUrl = `https://${publication}.substack.com/p/${slug}`;
+                try {
+                    const fetchResult = await simpleFetch(actualUrl, undefined, 15000);
+                    if (fetchResult?.html && fetchResult.html.length > 500) {
+                        workingHtml = fetchResult.html;
+                        workingUrl = actualUrl;
+                    }
+                }
+                catch { /* fall through with original HTML */ }
+            }
+        }
+        const $ = load(workingHtml);
+        // JSON-LD
+        let jsonLdData = null;
+        $('script[type="application/ld+json"]').each((_, el) => {
+            if (jsonLdData)
+                return;
+            const raw = $(el).html() || '';
+            const parsed = tryParseJson(raw);
+            if (parsed?.['@type'] === 'NewsArticle' || parsed?.['@type'] === 'Article')
+                jsonLdData = parsed;
+        });
+        const title = jsonLdData?.headline ||
+            $('meta[property="og:title"]').attr('content') ||
+            $('h1.post-title').first().text().trim() ||
+            $('h1').first().text().trim() || '';
+        if (!title)
+            return null;
+        const author = jsonLdData?.author?.name ||
+            $('meta[name="author"]').attr('content') ||
+            $('a.author-name').first().text().trim() ||
+            $('[class*="author"]').first().text().trim() || '';
+        const publishDate = jsonLdData?.datePublished ||
+            $('meta[property="article:published_time"]').attr('content') ||
+            $('time').first().attr('datetime') || '';
+        const publication = $('meta[property="og:site_name"]').attr('content') ||
+            $('a.navbar-title-link').text().trim() || new URL(workingUrl).hostname.replace('.substack.com', '');
+        const description = jsonLdData?.description ||
+            $('meta[property="og:description"]').attr('content') || '';
+        // Article content — try multiple Substack CSS patterns
+        let articleBody = '';
+        const postContent = $('.body.markup, .post-content, article, [class*="post-content"], .available-content').first();
+        if (postContent.length) {
+            postContent.find('script, style, nav, .paywall, .subscribe-widget, .subscription-widget').remove();
+            const parts = [];
+            postContent.find('h1, h2, h3, h4, p, blockquote, pre, li').each((_, el) => {
+                const tag = el.name;
+                const text = $(el).text().trim();
+                if (!text || text.length < 3)
+                    return;
+                if (tag === 'h1' || tag === 'h2')
+                    parts.push(`## ${text}`);
+                else if (tag === 'h3' || tag === 'h4')
+                    parts.push(`### ${text}`);
+                else if (tag === 'blockquote')
+                    parts.push(`> ${text}`);
+                else if (tag === 'pre')
+                    parts.push('```\n' + text + '\n```');
+                else
+                    parts.push(text);
+            });
+            articleBody = parts.join('\n\n');
+        }
+        // If no article body found, try broader search
+        if (!articleBody) {
+            const parts = [];
+            $('main p, article p, [class*="content"] p').each((_, el) => {
+                const text = $(el).text().trim();
+                if (text && text.length > 20)
+                    parts.push(text);
+            });
+            articleBody = parts.slice(0, 20).join('\n\n');
+        }
+        const contentBody = articleBody || description;
+        // Detect if the post appears paywalled (short content with no article body)
+        const isPaywalled = !articleBody && description.length > 0;
+        const paywallNote = isPaywalled
+            ? '\n\n---\n*⚠️ This post appears to be behind a paywall. Only the preview/description is available. Full content requires a subscription.*'
+            : '';
+        const structured = {
+            title,
+            author,
+            publication,
+            publishDate,
+            description,
+            paywalled: isPaywalled,
+            url: workingUrl,
+        };
+        const authorLine = author ? `\n**Author:** ${author}` : '';
+        const pubLine = publication ? `\n**Publication:** ${publication}` : '';
+        const dateLine = publishDate ? `\n**Published:** ${publishDate.split('T')[0]}` : '';
+        const cleanContent = `# ${title}${authorLine}${pubLine}${dateLine}\n\n${contentBody.substring(0, 8000)}${paywallNote}`;
+        return { domain: 'substack.com', type: 'post', structured, cleanContent };
+    }
+    catch {
+        return null;
+    }
+}

package/dist/ee/extractors/substackroot.d.ts

@@ -0,0 +1,2 @@
+import type { DomainExtractResult } from './types.js';
+export declare function substackRootExtractor(_html: string, _url: string): Promise<DomainExtractResult | null>;

package/dist/ee/extractors/substackroot.js

@@ -0,0 +1,46 @@
+// ---------------------------------------------------------------------------
+// 32. Substack root extractor (substack.com homepage)
+// ---------------------------------------------------------------------------
+export async function substackRootExtractor(_html, _url) {
+    // The substack.com homepage is a marketing page — not useful to extract.
+    // Instead, guide users to fetch individual newsletter posts.
+    // Try fetching their public sitemap to surface some featured newsletters.
+    // Note: Substack's homepage is JS-rendered; no useful API endpoints are publicly accessible.
+    // We return a helpful guide instead of trying to scrape the homepage.
+    const structured = {
+        note: 'Substack root homepage is a JS-rendered marketing page with limited extractable content.',
+        tip: 'Fetch individual Substack posts directly for full article content.',
+        examples: [
+            'https://username.substack.com/p/article-slug',
+            'https://stratechery.com/2024/...',
+        ],
+    };
+    const cleanContent = `# 📰 Substack
+
+Substack's homepage is a JS-rendered marketing page — there's not much useful content to extract here.
+
+## ✅ What Works
+
+Individual Substack posts are **fully server-rendered** and extract cleanly. Try:
+
+- \`https://username.substack.com/p/article-title\`
+- Any specific newsletter post URL
+
+## 💡 Examples
+
+\`\`\`
+https://lethain.substack.com/p/the-art-of-staffing-eng
+https://paulgraham.com/articles.html
+\`\`\`
+
+## 📋 Finding Newsletters
+
+Browse newsletters at:
+- [substack.com/explore](https://substack.com/explore) — discover publications
+- [substack.com/leaderboard](https://substack.com/leaderboard) — top newsletters by category
+
+---
+
+*WebPeel works best with individual Substack post URLs, not the root homepage.*`;
+    return { domain: 'substack.com', type: 'homepage', structured, cleanContent };
+}

package/dist/ee/extractors/tiktok.d.ts

@@ -0,0 +1,2 @@
+import type { DomainExtractResult } from './types.js';
+export declare function tiktokExtractor(_html: string, url: string): Promise<DomainExtractResult | null>;

package/dist/ee/extractors/tiktok.js

@@ -0,0 +1,29 @@
+import { fetchJson } from './shared.js';
+// ---------------------------------------------------------------------------
+// 22. TikTok extractor (oEmbed)
+// ---------------------------------------------------------------------------
+export async function tiktokExtractor(_html, url) {
+    try {
+        // TikTok official oEmbed endpoint
+        const oembedUrl = `https://www.tiktok.com/oembed?url=${encodeURIComponent(url)}`;
+        const data = await fetchJson(oembedUrl);
+        if (!data || !data.title)
+            return null;
+        const structured = {
+            title: data.title,
+            author: data.author_name || '',
+            authorUrl: data.author_url || '',
+            thumbnailUrl: data.thumbnail_url || '',
+            thumbnailWidth: data.thumbnail_width || 0,
+            thumbnailHeight: data.thumbnail_height || 0,
+            provider: 'TikTok',
+        };
+        const cleanContent = `## 🎵 TikTok: ${structured.title}\n\n**Creator:** [${structured.author}](${structured.authorUrl})\n**URL:** ${url}`;
+        return { domain: 'tiktok.com', type: 'video', structured, cleanContent };
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'TikTok oEmbed failed:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}

package/dist/ee/extractors/tradingview.d.ts

@@ -0,0 +1,2 @@
+import type { DomainExtractResult } from './types.js';
+export declare function tradingViewExtractor(_html: string, url: string): Promise<DomainExtractResult | null>;

package/dist/ee/extractors/tradingview.js

@@ -0,0 +1,182 @@
+// ---------------------------------------------------------------------------
+// 35. TradingView extractor — stock/index data via TradingView Scanner API
+// ---------------------------------------------------------------------------
+export async function tradingViewExtractor(_html, url) {
+    const urlObj = new URL(url);
+    const path = urlObj.pathname;
+    const domain = 'tradingview.com';
+    const scannerHeaders = {
+        'Origin': 'https://www.tradingview.com',
+        'Referer': 'https://www.tradingview.com/',
+        'Content-Type': 'application/json',
+    };
+    // Helper: format price
+    const fmtPrice = (v) => {
+        if (v == null)
+            return '?';
+        if (v >= 1_000_000_000_000)
+            return `${(v / 1_000_000_000_000).toFixed(2)}T`;
+        if (v >= 1_000_000_000)
+            return `${(v / 1_000_000_000).toFixed(2)}B`;
+        if (v >= 1_000_000)
+            return `${(v / 1_000_000).toFixed(2)}M`;
+        if (v >= 1_000)
+            return `${(v / 1_000).toFixed(2)}K`;
+        return v.toFixed(2);
+    };
+    const fmtChange = (c) => {
+        if (c == null)
+            return '';
+        const sign = c >= 0 ? '+' : '';
+        return `${sign}${c.toFixed(2)}%`;
+    };
+    // --- Symbol page: /symbols/<TICKER>/ or /chart?symbol=<TICKER> ---
+    const symbolMatch = path.match(/^\/symbols\/([^/?#]+)\/?/);
+    const chartSymbolParam = urlObj.searchParams.get('symbol');
+    let ticker = symbolMatch?.[1] || chartSymbolParam || null;
+    if (ticker) {
+        ticker = ticker.toUpperCase().replace(/-/g, '');
+        try {
+            // Try symbol search to resolve exchange
+            const searchResp = await fetch(`https://symbol-search.tradingview.com/symbol_search/?text=${encodeURIComponent(ticker)}&hl=0&lang=en&type=stock,fund,crypto,futures,forex&limit=5`, {
+                headers: {
+                    'User-Agent': 'webpeel/0.21 (https://webpeel.dev)',
+                    'Origin': 'https://www.tradingview.com',
+                    'Referer': 'https://www.tradingview.com/',
+                },
+                signal: AbortSignal.timeout(10000),
+            });
+            const searchData = await searchResp.json().catch(() => []);
+            // Find exact match
+            const exactMatch = searchData.find(s => s.symbol === ticker || s.symbol.replace(/<\/?em>/g, '') === ticker);
+            const symbolInfo = exactMatch || searchData[0];
+            if (symbolInfo) {
+                const exchange = symbolInfo.source_id || symbolInfo.exchange || 'NASDAQ';
+                // Fetch quote data via scanner
+                const scannerUrl = exchange === 'CRYPTO' || exchange === 'COINBASE' || exchange === 'BINANCE'
+                    ? 'https://scanner.tradingview.com/crypto/scan'
+                    : 'https://scanner.tradingview.com/america/scan';
+                const scanBody = {
+                    filter: [{ left: 'name', operation: 'equal', right: symbolInfo.symbol?.replace(/<\/?em>/g, '') || ticker }],
+                    columns: ['name', 'description', 'close', 'open', 'high', 'low', 'volume', 'change', 'change_abs', 'market_cap_basic', 'sector', 'industry', 'country', 'currency'],
+                    range: [0, 1],
+                };
+                const scanResp = await fetch(scannerUrl, {
+                    method: 'POST',
+                    headers: { ...scannerHeaders, 'User-Agent': 'webpeel/0.21 (https://webpeel.dev)' },
+                    body: JSON.stringify(scanBody),
+                    signal: AbortSignal.timeout(10000),
+                });
+                const scanData = await scanResp.json().catch(() => null);
+                const row = scanData?.data?.[0]?.d;
+                if (row) {
+                    const [name, desc, close, open, high, low, volume, changePct, changeAbs, mktCap, sector, industry, country, currency] = row;
+                    const currStr = currency || 'USD';
+                    const mktCapStr = mktCap ? fmtPrice(mktCap) : null;
+                    const structured = {
+                        symbol: name,
+                        description: desc,
+                        price: close,
+                        open,
+                        high,
+                        low,
+                        volume,
+                        change_pct: changePct,
+                        change_abs: changeAbs,
+                        market_cap: mktCap,
+                        sector,
+                        industry,
+                        country,
+                        currency: currStr,
+                        exchange,
+                        fetchedAt: new Date().toISOString(),
+                    };
+                    const changeStr = fmtChange(changePct);
+                    const changeIcon = (changePct ?? 0) >= 0 ? '📈' : '📉';
+                    const cleanContent = `# ${changeIcon} TradingView: ${desc || name} (${name})
+
+## Quote
+- **Price:** ${close?.toFixed(2) ?? '?'} ${currStr}
+- **Change:** ${changeStr} (${changeAbs?.toFixed(2) ?? '?'} ${currStr})
+- **Open:** ${open?.toFixed(2) ?? '?'} | **High:** ${high?.toFixed(2) ?? '?'} | **Low:** ${low?.toFixed(2) ?? '?'}
+- **Volume:** ${fmtPrice(volume ?? 0)}
+${mktCapStr ? `- **Market Cap:** ${mktCapStr} ${currStr}` : ''}
+
+## Details
+${sector ? `- **Sector:** ${sector}` : ''}
+${industry ? `- **Industry:** ${industry}` : ''}
+${country ? `- **Country:** ${country}` : ''}
+- **Exchange:** ${exchange}
+
+---
+*Source: [TradingView](https://www.tradingview.com/symbols/${name}/) · Data via TradingView Scanner API*`;
+                    return { domain, type: 'symbol', structured, cleanContent };
+                }
+            }
+        }
+        catch (e) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', 'TradingView symbol fetch failed:', e instanceof Error ? e.message : e);
+        }
+    }
+    // --- Markets overview page: show major indices for root / /markets paths only ---
+    // For unrecognized paths (e.g. /ideas, /screener, /profile) return null so
+    // the pipeline falls through to browser rendering instead of showing wrong data.
+    const isRootOrMarkets = path === '/' || path === '' || path.startsWith('/markets');
+    if (!ticker && !isRootOrMarkets) {
+        return null;
+    }
+    try {
+        // Fetch major indices + top stocks
+        const scanBody = {
+            filter: [
+                { left: 'name', operation: 'in_range', right: ['SPX', 'NDX', 'DJI', 'RUT', 'VIX', 'AAPL', 'MSFT', 'NVDA', 'AMZN', 'GOOGL', 'META', 'TSLA'] },
+            ],
+            columns: ['name', 'description', 'close', 'change', 'volume', 'market_cap_basic'],
+            sort: { sortBy: 'market_cap_basic', sortOrder: 'desc' },
+            range: [0, 20],
+        };
+        const resp = await fetch('https://scanner.tradingview.com/global/scan', {
+            method: 'POST',
+            headers: { ...scannerHeaders, 'User-Agent': 'webpeel/0.21 (https://webpeel.dev)' },
+            body: JSON.stringify(scanBody),
+            signal: AbortSignal.timeout(10000),
+        });
+        const data = await resp.json().catch(() => null);
+        const rows = data?.data || [];
+        if (rows.length > 0) {
+            const tableRows = rows.map((row) => {
+                const [name, desc, close, changePct] = row.d;
+                const changeStr = changePct != null ? `${changePct >= 0 ? '+' : ''}${changePct.toFixed(2)}%` : '?%';
+                const icon = (changePct ?? 0) >= 0 ? '🟢' : '🔴';
+                return `| ${name} | ${desc} | ${close?.toFixed(2) ?? '?'} | ${icon} ${changeStr} |`;
+            }).join('\n');
+            const structured = {
+                symbols: rows.map((r) => ({
+                    symbol: r.d[0],
+                    description: r.d[1],
+                    price: r.d[2],
+                    change_pct: r.d[3],
+                })),
+                fetchedAt: new Date().toISOString(),
+            };
+            const now = new Date().toLocaleString('en-US', { timeZone: 'America/New_York', hour12: false });
+            const cleanContent = `# 📈 TradingView — Market Overview
+
+*As of ${now} ET*
+
+| Symbol | Name | Price | Change |
+|--------|------|-------|--------|
+${tableRows}
+
+---
+*Source: [TradingView](https://www.tradingview.com/markets/) · Data via TradingView Scanner API*`;
+            return { domain, type: 'markets', structured, cleanContent };
+        }
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'TradingView markets fetch failed:', e instanceof Error ? e.message : e);
+    }
+    return null;
+}

package/dist/ee/extractors/twitch.d.ts

@@ -0,0 +1,2 @@
+import type { DomainExtractResult } from './types.js';
+export declare function twitchExtractor(_html: string, url: string): Promise<DomainExtractResult | null>;

package/dist/ee/extractors/twitch.js

@@ -0,0 +1,36 @@
+import { fetchJson } from './shared.js';
+// ---------------------------------------------------------------------------
+// 27. Twitch extractor (noembed / Twitch API)
+// ---------------------------------------------------------------------------
+export async function twitchExtractor(_html, url) {
+    try {
+        // Use noembed.com for Twitch clips and channel pages
+        const noembedUrl = `https://noembed.com/embed?url=${encodeURIComponent(url)}`;
+        const data = await fetchJson(noembedUrl);
+        if (!data || data.error)
+            return null;
+        const urlObj = new URL(url);
+        const pathParts = urlObj.pathname.split('/').filter(Boolean);
+        const isClip = pathParts[1] === 'clip' || pathParts[0] === 'clip' || url.includes('clips.twitch.tv');
+        const channelName = !isClip ? pathParts[0] : '';
+        const contentType = isClip ? 'clip' : 'channel';
+        const structured = {
+            title: data.title || '',
+            author: data.author_name || channelName,
+            authorUrl: data.author_url || '',
+            thumbnailUrl: data.thumbnail_url || '',
+            provider: 'Twitch',
+            contentType,
+            channelName: channelName || data.author_name || '',
+        };
+        const typeEmoji = isClip ? '🎬' : '🎮';
+        const titleText = structured.title || structured.channelName;
+        const cleanContent = `## ${typeEmoji} Twitch ${contentType}: ${titleText}\n\n**Channel:** [${structured.author}](${structured.authorUrl})\n**URL:** ${url}`;
+        return { domain: 'twitch.tv', type: contentType, structured, cleanContent };
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'Twitch oEmbed failed:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}

package/dist/ee/extractors/twitter.d.ts

@@ -0,0 +1,2 @@
+import type { DomainExtractResult } from './types.js';
+export declare function twitterExtractor(html: string, url: string): Promise<DomainExtractResult | null>;