@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/ee/extractors/wikipedia.js

@@ -0,0 +1,235 @@
+import { simpleFetch } from '../../core/fetcher.js';
+import { stripHtml, fetchJson } from './shared.js';
+// ---------------------------------------------------------------------------
+// 5. Wikipedia extractor
+// ---------------------------------------------------------------------------
+/** Remove Wikipedia-specific noise from extracted content. */
+function cleanWikipediaContent(content) {
+    return content
+        // Remove [edit] links
+        .replace(/\[edit\]/gi, '')
+        // Remove citation brackets [1], [2], etc.
+        .replace(/\[\d+\]/g, '')
+        // Remove [citation needed], [verification], etc.
+        .replace(/\[(citation needed|verification|improve this article|adding citations[^\]]*|when\?|where\?|who\?|clarification needed|dubious[^\]]*|failed verification[^\]]*|unreliable source[^\]]*)\]/gi, '')
+        // Remove [Learn how and when to remove this message]
+        .replace(/\[Learn how and when to remove this message\]/gi, '')
+        // Clean up excess whitespace
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}
+/** Max rows to extract from a single HTML table to prevent token explosion. */
+const MAX_TABLE_ROWS = 50;
+/**
+ * Convert an HTML <table> string to a markdown pipe table.
+ * Returns null if the table can't be meaningfully converted (e.g. layout table).
+ * Handles colspan/rowspan by flattening, caps at MAX_TABLE_ROWS.
+ */
+function htmlTableToMarkdown(tableHtml) {
+    // Extract all rows
+    const rowMatches = tableHtml.match(/<tr[^>]*>([\s\S]*?)<\/tr>/gi);
+    if (!rowMatches || rowMatches.length < 2)
+        return null;
+    // Parse a row into cell texts
+    function parseRow(rowHtml) {
+        const cells = [];
+        const cellRegex = /<t[hd][^>]*>([\s\S]*?)<\/t[hd]>/gi;
+        let m;
+        while ((m = cellRegex.exec(rowHtml)) !== null) {
+            const colspanMatch = m[0].match(/colspan=["']?(\d+)/i);
+            const span = colspanMatch ? Math.min(parseInt(colspanMatch[1], 10), 6) : 1;
+            const text = stripHtml(m[1]).replace(/\|/g, '\\|').replace(/\n/g, ' ').trim();
+            cells.push(text);
+            // Fill colspan with empty cells
+            for (let s = 1; s < span; s++)
+                cells.push('');
+        }
+        return cells;
+    }
+    // Detect header row: first row with <th> elements
+    let headerRowIdx = -1;
+    for (let i = 0; i < Math.min(rowMatches.length, 3); i++) {
+        if (/<th[\s>]/i.test(rowMatches[i])) {
+            headerRowIdx = i;
+            break;
+        }
+    }
+    let headers;
+    let dataStartIdx;
+    if (headerRowIdx >= 0) {
+        headers = parseRow(rowMatches[headerRowIdx]);
+        dataStartIdx = headerRowIdx + 1;
+    }
+    else {
+        // No header row — use first row as header
+        headers = parseRow(rowMatches[0]);
+        dataStartIdx = 1;
+    }
+    if (headers.length < 2)
+        return null;
+    // Skip tables that look like layout (single column or no real content)
+    if (headers.every(h => !h))
+        return null;
+    const colCount = headers.length;
+    const mdLines = [];
+    // Header row
+    mdLines.push('| ' + headers.map(h => h || ' ').join(' | ') + ' |');
+    // Separator row
+    mdLines.push('| ' + headers.map(() => '---').join(' | ') + ' |');
+    // Data rows (capped at MAX_TABLE_ROWS)
+    let rowCount = 0;
+    for (let r = dataStartIdx; r < rowMatches.length && rowCount < MAX_TABLE_ROWS; r++) {
+        const cells = parseRow(rowMatches[r]);
+        if (cells.length === 0)
+            continue;
+        // Pad or trim to match column count
+        while (cells.length < colCount)
+            cells.push('');
+        const row = cells.slice(0, colCount);
+        // Skip completely empty rows
+        if (row.every(c => !c))
+            continue;
+        mdLines.push('| ' + row.map(c => c || ' ').join(' | ') + ' |');
+        rowCount++;
+    }
+    if (rowCount === 0)
+        return null;
+    const truncNote = (rowMatches.length - dataStartIdx > MAX_TABLE_ROWS)
+        ? `\n\n*Table truncated to ${MAX_TABLE_ROWS} rows.*`
+        : '';
+    return mdLines.join('\n') + truncNote;
+}
+/**
+ * Extract wikitables from raw Wikipedia HTML.
+ * Returns markdown for data tables (class="wikitable"), ignoring navboxes/infoboxes/layout tables.
+ */
+function extractWikitables(html) {
+    const tables = [];
+    // Match tables with class="wikitable" — these are always data tables on Wikipedia
+    const tableRegex = /<table[^>]*class="[^"]*wikitable[^"]*"[^>]*>([\s\S]*?)<\/table>/gi;
+    let match;
+    while ((match = tableRegex.exec(html)) !== null) {
+        const fullTable = match[0];
+        // Skip navboxes and sidebar tables
+        if (/navbox|sidebar|metadata/i.test(fullTable.slice(0, 200)))
+            continue;
+        // Try to extract a caption
+        const captionMatch = fullTable.match(/<caption[^>]*>([\s\S]*?)<\/caption>/i);
+        const caption = captionMatch ? stripHtml(captionMatch[1]).trim() : '';
+        const md = htmlTableToMarkdown(fullTable);
+        if (md) {
+            const prefix = caption ? `**${caption}**\n\n` : '';
+            tables.push(prefix + md);
+        }
+    }
+    return tables;
+}
+export async function wikipediaExtractor(_html, url, options) {
+    const urlObj = new URL(url);
+    const pathParts = urlObj.pathname.split('/').filter(Boolean);
+    // Only handle article pages: /wiki/Article_Title
+    if (pathParts[0] !== 'wiki' || pathParts.length < 2)
+        return null;
+    const articleTitle = decodeURIComponent(pathParts[1]);
+    // Skip special pages (contain a colon, e.g. Special:Random, Talk:Article)
+    if (articleTitle.includes(':'))
+        return null;
+    // For list/comparison/data-heavy articles, skip the summary API entirely.
+    // The summary API only returns paragraph text — no tables. Return null so the
+    // normal HTML→markdown pipeline fetches the full page and preserves tables.
+    const isListArticle = /^List_of|^Lists_of|^Comparison_of|^List_/i.test(articleTitle);
+    if (isListArticle)
+        return null;
+    const lang = urlObj.hostname.split('.')[0] || 'en';
+    const summaryUrl = `https://${lang}.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(articleTitle)}`;
+    // Wikipedia REST API requires a descriptive User-Agent (https://meta.wikimedia.org/wiki/User-Agent_policy)
+    const wikiHeaders = { 'User-Agent': 'WebPeel/0.17.1 (https://webpeel.dev; jake@jakeliu.me) Node.js', 'Api-User-Agent': 'WebPeel/0.17.1 (https://webpeel.dev; jake@jakeliu.me)' };
+    // Detect data-heavy pages: "List of ...", "Comparison of ...", tables in the raw HTML
+    const isListPage = /^List[_ ]of[_ ]/i.test(articleTitle) || /^Comparison[_ ]of[_ ]/i.test(articleTitle);
+    const rawHasWikitables = _html && /class="[^"]*wikitable/i.test(_html);
+    const hasTableData = isListPage || rawHasWikitables;
+    try {
+        const data = await fetchJson(summaryUrl, wikiHeaders);
+        if (!data || data.type === 'https://mediawiki.org/wiki/HyperSwitch/errors/not_found')
+            return null;
+        const structured = {
+            title: data.title || articleTitle.replace(/_/g, ' '),
+            description: data.description || '',
+            extract: data.extract || '',
+            extractHtml: data.extract_html || '',
+            thumbnail: data.thumbnail?.source || null,
+            url: data.content_urls?.desktop?.page || url,
+            lastModified: data.timestamp || null,
+            coordinates: data.coordinates || null,
+        };
+        // Default: use summary API (200-400 tokens). Only fetch full article if budget > 5000.
+        const budget = options?.budget ?? 0;
+        const useFull = budget > 5000;
+        let bodyContent = structured.extract;
+        let mobileHtmlSize;
+        let tableSections = [];
+        // Extract tables from the raw HTML we already have (no extra API call needed)
+        if (hasTableData && _html) {
+            tableSections = extractWikitables(_html);
+        }
+        if (useFull) {
+            try {
+                const fullUrl = `https://${lang}.wikipedia.org/api/rest_v1/page/mobile-html/${encodeURIComponent(articleTitle)}`;
+                const fullResult = await simpleFetch(fullUrl, undefined, 15000, {
+                    ...wikiHeaders,
+                    'Accept': 'text/html',
+                });
+                if (fullResult?.html) {
+                    mobileHtmlSize = fullResult.html.length;
+                    let fullContent = '';
+                    const sectionMatches = fullResult.html.match(/<section[^>]*>([\s\S]*?)<\/section>/gi) || [];
+                    for (const section of sectionMatches) {
+                        const headingMatch = section.match(/<h[2-6][^>]*id="([^"]*)"[^>]*class="[^"]*pcs-edit-section-title[^"]*"[^>]*>([\s\S]*?)<\/h[2-6]>/i);
+                        const heading = headingMatch ? stripHtml(headingMatch[2]).trim() : '';
+                        const paragraphs = section.match(/<p[^>]*>([\s\S]*?)<\/p>/gi) || [];
+                        const sectionText = paragraphs.map((p) => stripHtml(p).trim()).filter((t) => t.length > 0).join('\n\n');
+                        if (sectionText) {
+                            const prefix = heading ? `## ${heading}\n\n` : '';
+                            fullContent += `\n\n${prefix}${sectionText}`;
+                        }
+                    }
+                    bodyContent = cleanWikipediaContent(fullContent) || structured.extract;
+                    // Also extract tables from mobile-html if we didn't get them from raw HTML
+                    if (tableSections.length === 0) {
+                        tableSections = extractWikitables(fullResult.html);
+                    }
+                }
+            }
+            catch (e) {
+                if (process.env.DEBUG)
+                    console.debug('[webpeel]', 'Wikipedia mobile-html failed, using summary:', e instanceof Error ? e.message : e);
+            }
+        }
+        const articleUrl = structured.url;
+        const lines = [
+            `# ${structured.title}`,
+            '',
+        ];
+        if (structured.description)
+            lines.push(`*${structured.description}*`, '');
+        lines.push(bodyContent);
+        // Append extracted tables
+        if (tableSections.length > 0) {
+            lines.push('', '---', '');
+            for (const table of tableSections) {
+                lines.push(table, '');
+            }
+        }
+        if (structured.coordinates) {
+            lines.push('', `📍 Coordinates: ${structured.coordinates.lat}, ${structured.coordinates.lon}`);
+        }
+        lines.push('', `📖 [Read full article on Wikipedia](${articleUrl})`);
+        const cleanContent = lines.join('\n');
+        return { domain: 'wikipedia.org', type: 'article', structured, cleanContent, rawHtmlSize: mobileHtmlSize };
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'Wikipedia API failed:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}

package/dist/ee/extractors/yelp.d.ts

@@ -0,0 +1,2 @@
+import type { DomainExtractResult } from './types.js';
+export declare function yelpExtractor(_html: string, url: string): Promise<DomainExtractResult | null>;

package/dist/ee/extractors/yelp.js

@@ -0,0 +1,216 @@
+// ---------------------------------------------------------------------------
+// Yelp extractor — parse JSON-LD + meta from stealth-rendered HTML
+// ---------------------------------------------------------------------------
+export async function yelpExtractor(_html, url) {
+    const YELP_API_KEY = process.env.YELP_API_KEY;
+    // Helper to call Yelp Fusion API
+    async function yelpFetch(path, params) {
+        const base = 'https://api.yelp.com/v3';
+        const qs = params ? '?' + new URLSearchParams(params).toString() : '';
+        const res = await fetch(`${base}${path}${qs}`, {
+            headers: { 'Authorization': `Bearer ${YELP_API_KEY}` },
+        });
+        if (!res.ok) {
+            throw new Error(`Yelp API ${res.status}: ${res.statusText}`);
+        }
+        return res.json();
+    }
+    try {
+        const parsed = new URL(url);
+        const pathname = parsed.pathname;
+        const searchParams = parsed.searchParams;
+        // ----------------------------------------------------------------
+        // If no API key, fall back to the legacy HTML-scraping approach
+        // ----------------------------------------------------------------
+        if (!YELP_API_KEY) {
+            // Legacy fallback: minimal result pointing user to Yelp
+            const term = searchParams.get('find_desc') || searchParams.get('cflt') || 'businesses';
+            const loc = searchParams.get('find_loc') || '';
+            const isBiz = pathname.startsWith('/biz/');
+            const cleanContent = isBiz
+                ? `# Yelp Business\n\n*No YELP_API_KEY configured — visit [Yelp](${url}) for details.*`
+                : `# 🔍 Yelp Search: ${term}${loc ? ` in ${loc}` : ''}\n\n*No YELP_API_KEY configured — [View on Yelp](${url})*`;
+            return {
+                domain: 'yelp.com',
+                type: isBiz ? 'business' : 'search',
+                structured: { url },
+                cleanContent,
+            };
+        }
+        // ----------------------------------------------------------------
+        // Business page: /biz/<alias>
+        // ----------------------------------------------------------------
+        if (pathname.startsWith('/biz/')) {
+            const alias = pathname.replace('/biz/', '').split('?')[0].split('#')[0];
+            let biz;
+            try {
+                biz = await yelpFetch(`/businesses/${alias}`);
+            }
+            catch (e) {
+                if (process.env.DEBUG)
+                    console.debug('[webpeel] Yelp biz fetch failed:', e instanceof Error ? e.message : e);
+                return null;
+            }
+            // Fetch reviews (best-effort)
+            let reviews = [];
+            try {
+                const revData = await yelpFetch(`/businesses/${alias}/reviews`, { limit: '3' });
+                reviews = revData.reviews || [];
+            }
+            catch { /* reviews are optional */ }
+            const name = biz.name || alias;
+            const rating = biz.rating != null ? biz.rating.toFixed(1) : '?';
+            const reviewCount = biz.review_count ?? 0;
+            const addr = biz.location;
+            const address = addr
+                ? [addr.address1, addr.city, addr.state, addr.zip_code].filter(Boolean).join(', ')
+                : '';
+            const phone = biz.display_phone || biz.phone || '';
+            const price = biz.price || '';
+            const categories = (biz.categories || []).map((c) => c.title).join(' | ');
+            const yelpUrl = biz.url || url;
+            // Hours
+            let hoursStr = '';
+            if (biz.hours && biz.hours.length > 0) {
+                const dayNames = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'];
+                const dayMap = {};
+                for (const slot of biz.hours[0].open || []) {
+                    const fmt = (t) => {
+                        const h = parseInt(t.slice(0, 2), 10);
+                        const m = t.slice(2);
+                        const period = h >= 12 ? 'PM' : 'AM';
+                        const h12 = h % 12 || 12;
+                        return `${h12}:${m} ${period}`;
+                    };
+                    const day = slot.day;
+                    if (!dayMap[day])
+                        dayMap[day] = [];
+                    dayMap[day].push(`${fmt(slot.start)}–${fmt(slot.end)}`);
+                }
+                hoursStr = Object.entries(dayMap)
+                    .map(([d, times]) => `${dayNames[parseInt(d, 10)]}: ${times.join(', ')}`)
+                    .join(' | ');
+            }
+            const lines = [
+                `# ${name} ⭐ ${rating} (${reviewCount.toLocaleString()} reviews)`,
+                '',
+            ];
+            if (address)
+                lines.push(`📍 ${address}`);
+            if (categories)
+                lines.push(`🏷️ ${categories}${price ? ` | 💰 ${price}` : ''}`);
+            else if (price)
+                lines.push(`💰 ${price}`);
+            if (phone)
+                lines.push(`📞 ${phone}`);
+            if (hoursStr)
+                lines.push(`🕐 ${hoursStr}`);
+            if (biz.is_closed === true)
+                lines.push(`⚠️ *Permanently closed*`);
+            lines.push('');
+            if (reviews.length > 0) {
+                for (const rev of reviews) {
+                    const stars = '⭐'.repeat(Math.round(rev.rating || 0));
+                    const text = (rev.text || '').replace(/\n+/g, ' ').trim().slice(0, 200);
+                    lines.push(`> ${stars} — ${text}${(rev.text || '').length > 200 ? '…' : ''}`);
+                    lines.push('');
+                }
+            }
+            lines.push(`[View on Yelp](${yelpUrl})`);
+            return {
+                domain: 'yelp.com',
+                type: 'business',
+                structured: { name, rating: parseFloat(rating), reviewCount, address, phone, price, categories, url: yelpUrl },
+                cleanContent: lines.join('\n'),
+            };
+        }
+        // ----------------------------------------------------------------
+        // Search / Category URL: /search?find_desc=...&find_loc=...
+        //                        /search?cflt=restaurants&find_loc=...
+        // ----------------------------------------------------------------
+        const findDesc = searchParams.get('find_desc') || '';
+        const cflt = searchParams.get('cflt') || '';
+        const findLoc = searchParams.get('find_loc') || '';
+        if (!findLoc && !findDesc && !cflt) {
+            // Not a recognized pattern
+            return null;
+        }
+        const apiParams = { limit: '10' };
+        if (findLoc)
+            apiParams.location = findLoc;
+        if (findDesc)
+            apiParams.term = findDesc;
+        if (cflt && !findDesc)
+            apiParams.categories = cflt;
+        let data;
+        try {
+            data = await yelpFetch('/businesses/search', apiParams);
+        }
+        catch (e) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel] Yelp search failed:', e instanceof Error ? e.message : e);
+            return null;
+        }
+        const businesses = data.businesses || [];
+        const total = data.total ?? businesses.length;
+        // Build header
+        const searchLabel = findDesc || cflt || 'Businesses';
+        const locationLabel = findLoc || '';
+        const emoji = cflt === 'restaurants' || findDesc?.toLowerCase().includes('restaurant') ? '🍽️'
+            : findDesc?.toLowerCase().includes('pizza') ? '🍕'
+                : findDesc?.toLowerCase().includes('coffee') || findDesc?.toLowerCase().includes('cafe') ? '☕'
+                    : findDesc?.toLowerCase().includes('bar') ? '🍺'
+                        : '🔍';
+        const titleParts = [searchLabel.charAt(0).toUpperCase() + searchLabel.slice(1)];
+        if (locationLabel)
+            titleParts.push(`in ${locationLabel}`);
+        const lines = [
+            `# ${emoji} Yelp — ${titleParts.join(' ')}`,
+            '',
+            `*${businesses.length} of ${total.toLocaleString()} results via Yelp Fusion API*`,
+            '',
+        ];
+        for (let i = 0; i < businesses.length; i++) {
+            const b = businesses[i];
+            const bName = b.name || 'Unknown';
+            const bRating = b.rating != null ? b.rating.toFixed(1) : '?';
+            const bReviews = b.review_count ?? 0;
+            const bAddr = b.location;
+            const bAddress = bAddr
+                ? [bAddr.address1, bAddr.city, bAddr.state, bAddr.zip_code].filter(Boolean).join(', ')
+                : '';
+            const bPhone = b.display_phone || '';
+            const bPrice = b.price || '';
+            const bCategories = (b.categories || []).map((c) => c.title).join(' | ');
+            const bUrl = b.url || '';
+            const bSnippet = b.snippet_text || '';
+            lines.push(`## ${i + 1}. ${bName} ⭐ ${bRating} (${bReviews.toLocaleString()} reviews)`);
+            if (bAddress)
+                lines.push(`📍 ${bAddress}`);
+            const tagLine = [bCategories && `🏷️ ${bCategories}`, bPrice && `💰 ${bPrice}`].filter(Boolean).join(' | ');
+            if (tagLine)
+                lines.push(tagLine);
+            if (bPhone)
+                lines.push(`📞 ${bPhone}`);
+            if (bSnippet)
+                lines.push(`> ${bSnippet.replace(/\n+/g, ' ').trim().slice(0, 150)}`);
+            if (bUrl)
+                lines.push(`[View on Yelp](${bUrl})`);
+            lines.push('');
+        }
+        if (businesses.length === 0) {
+            lines.push(`*No results found for "${searchLabel}"${locationLabel ? ` in ${locationLabel}` : ''}.*`);
+        }
+        return {
+            domain: 'yelp.com',
+            type: 'search',
+            structured: { query: searchLabel, location: locationLabel, total, count: businesses.length, businesses },
+            cleanContent: lines.join('\n'),
+        };
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'Yelp extractor error:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}

package/dist/ee/extractors/youtube.d.ts

@@ -0,0 +1,2 @@
+import type { DomainExtractResult } from './types.js';
+export declare function youtubeExtractor(_html: string, url: string): Promise<DomainExtractResult | null>;

package/dist/ee/extractors/youtube.js

@@ -0,0 +1,189 @@
+import { getYouTubeTranscript } from '../../core/youtube.js';
+import { fetchJson } from './shared.js';
+// ---------------------------------------------------------------------------
+// 6. YouTube extractor (oEmbed API-first)
+// ---------------------------------------------------------------------------
+export async function youtubeExtractor(_html, url) {
+    // Helper: wrap a promise with a timeout
+    function withTimeout(promise, ms) {
+        return Promise.race([
+            promise,
+            new Promise((_, reject) => setTimeout(() => reject(new Error(`Timeout after ${ms}ms`)), ms)),
+        ]);
+    }
+    // Run transcript fetch and oEmbed fetch in parallel
+    // Proxy-based extraction takes 2-5s, but retry logic may need more time
+    const transcriptPromise = withTimeout(getYouTubeTranscript(url), 30000);
+    const oembedPromise = fetchJson(`https://www.youtube.com/oembed?url=${encodeURIComponent(url)}&format=json`);
+    const noembedPromise = fetchJson(`https://noembed.com/embed?url=${encodeURIComponent(url)}`).catch(() => null);
+    // Fetch subscriber count from channel page (lightweight, parallel)
+    const subscriberPromise = (async () => {
+        try {
+            // Wait for oEmbed to get channel URL, then fetch subscriber count from channel page
+            const oembed = await oembedPromise;
+            const channelUrl = oembed?.author_url;
+            if (!channelUrl)
+                return '';
+            const resp = await fetch(channelUrl, {
+                headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36' },
+                signal: AbortSignal.timeout(5000),
+            });
+            const html = await resp.text();
+            // Look for subscriber count in page metadata (e.g. "4.12M subscribers")
+            const subMatch = html.match(/(\d+(?:\.\d+)?[KMBkmb]?)\s*subscribers/i);
+            return subMatch ? subMatch[1] + ' subscribers' : '';
+        }
+        catch {
+            return '';
+        }
+    })();
+    const [transcriptResult, oembedResult, noembedResult, subscriberResult] = await Promise.allSettled([
+        transcriptPromise,
+        oembedPromise,
+        noembedPromise,
+        subscriberPromise,
+    ]);
+    const transcript = transcriptResult.status === 'fulfilled' ? transcriptResult.value : null;
+    const oembedData = oembedResult.status === 'fulfilled' ? oembedResult.value : null;
+    const noembedData = noembedResult.status === 'fulfilled' ? noembedResult.value : null;
+    const subscriberCount = subscriberResult.status === 'fulfilled' ? subscriberResult.value : '';
+    if (process.env.DEBUG) {
+        if (transcriptResult.status === 'rejected') {
+            console.debug('[webpeel]', 'YouTube transcript failed:', transcriptResult.reason instanceof Error ? transcriptResult.reason.message : transcriptResult.reason);
+        }
+        if (oembedResult.status === 'rejected') {
+            console.debug('[webpeel]', 'YouTube oEmbed failed:', oembedResult.reason instanceof Error ? oembedResult.reason.message : oembedResult.reason);
+        }
+    }
+    // If transcript succeeded, build rich content
+    if (transcript) {
+        const title = transcript.title || oembedData?.title || '';
+        const channel = transcript.channel || oembedData?.author_name || '';
+        const channelUrl = oembedData?.author_url || `https://www.youtube.com/@${channel}`;
+        const description = transcript.description || noembedData?.description || oembedData?.description || '';
+        const thumbnailUrl = oembedData?.thumbnail_url || '';
+        const publishDate = transcript.publishDate || '';
+        const hasTranscript = transcript.segments.length > 0;
+        const structured = {
+            title,
+            channel,
+            channelUrl,
+            subscriberCount: subscriberCount || undefined,
+            duration: transcript.duration,
+            publishDate,
+            language: transcript.language,
+            availableLanguages: transcript.availableLanguages,
+            transcriptSegments: transcript.segments.length,
+            wordCount: transcript.wordCount ?? 0,
+            viewCount: transcript.viewCount ?? '',
+            likeCount: transcript.likeCount ?? '',
+            description,
+            thumbnailUrl,
+            chapters: transcript.chapters ?? [],
+            keyPoints: transcript.keyPoints ?? [],
+            source: 'transcript',
+        };
+        // Format the publish date nicely if it's an ISO date
+        let publishStr = '';
+        if (publishDate) {
+            try {
+                const d = new Date(publishDate);
+                publishStr = d.toLocaleDateString('en-US', { month: 'short', year: 'numeric', day: 'numeric' });
+            }
+            catch {
+                publishStr = publishDate;
+            }
+        }
+        // Format view count (e.g. "1,234,567" → "1.2M views")
+        let viewStr = '';
+        if (transcript.viewCount) {
+            const v = parseInt(transcript.viewCount, 10);
+            if (!isNaN(v)) {
+                if (v >= 1_000_000)
+                    viewStr = `${(v / 1_000_000).toFixed(1).replace(/\.0$/, '')}M views`;
+                else if (v >= 1_000)
+                    viewStr = `${(v / 1_000).toFixed(1).replace(/\.0$/, '')}K views`;
+                else
+                    viewStr = `${v.toLocaleString()} views`;
+            }
+        }
+        // Build header line
+        const channelPart = subscriberCount ? `${channel} (${subscriberCount})` : channel;
+        const headerParts = [`**Channel:** ${channelPart}`];
+        if (transcript.duration && transcript.duration !== '0:00')
+            headerParts.push(`**Duration:** ${transcript.duration}`);
+        if (viewStr)
+            headerParts.push(`**${viewStr}**`);
+        if (publishStr)
+            headerParts.push(`**Published:** ${publishStr}`);
+        const headerLine = headerParts.join(' | ');
+        const parts = [];
+        parts.push(`# ${title}`);
+        parts.push(headerLine);
+        /**
+         * Strip music note symbols from transcript/caption text.
+         * YouTube auto-captions include ♪ and 🎵 as music cues.
+         * Patterns cleaned:
+         *   [♪♪♪]  →  (removed)
+         *   ♪ text ♪  →  text
+         *   standalone ♪ / 🎵  →  (removed)
+         */
+        const cleanMusicNotes = (text) => text
+            // Remove bracketed music cues: [♪], [♪♪♪], [🎵🎵🎵], etc.
+            .replace(/\[[♪🎵]+\]/g, '')
+            // Unwrap ♪ text ♪ → text (keep the words between notes)
+            .replace(/♪\s*([^♪]*?)\s*♪/g, (_, inner) => inner.trim())
+            // Remove any remaining standalone ♪ or 🎵
+            .replace(/[♪🎵]+/g, '')
+            // Collapse extra whitespace introduced by removals
+            .replace(/\s{2,}/g, ' ')
+            .trim();
+        // Summary section
+        if (transcript.summary && hasTranscript) {
+            let summaryText = cleanMusicNotes(transcript.summary);
+            summaryText = summaryText.replace(/([.!?])\s+(?=[A-Z])/g, '$1\n\n');
+            parts.push(`## Summary\n\n${summaryText}`);
+        }
+        else if (!hasTranscript && transcript.fullText) {
+            parts.push(`## Description\n\n${transcript.fullText}`);
+        }
+        // Key Points section
+        if (transcript.keyPoints && transcript.keyPoints.length > 0) {
+            const kpLines = transcript.keyPoints.map(kp => `- ${kp}`).join('\n');
+            parts.push(`## Key Points\n\n${kpLines}`);
+        }
+        // Chapters section
+        if (transcript.chapters && transcript.chapters.length > 0) {
+            const chLines = transcript.chapters.map(ch => `- ${ch.time} — ${ch.title}`).join('\n');
+            parts.push(`## Chapters\n\n${chLines}`);
+        }
+        // Full Transcript section (only if we have real transcript segments)
+        // Add intelligent paragraph breaks for readability
+        if (hasTranscript) {
+            let readableText = cleanMusicNotes(transcript.fullText);
+            // Break into paragraphs: after sentence-ending punctuation followed by a capital letter
+            readableText = readableText.replace(/([.!?])\s+(?=[A-Z])/g, '$1\n\n');
+            // Collapse any triple+ newlines
+            readableText = readableText.replace(/\n{3,}/g, '\n\n');
+            parts.push(`## Full Transcript\n\n${readableText}`);
+        }
+        const cleanContent = parts.join('\n\n');
+        return { domain: 'youtube.com', type: 'video', structured, cleanContent };
+    }
+    // Fall back to oEmbed if transcript failed
+    if (oembedData && oembedData.title) {
+        const structured = {
+            title: oembedData.title,
+            channel: oembedData.author_name || '',
+            channelUrl: oembedData.author_url || '',
+            thumbnailUrl: oembedData.thumbnail_url || '',
+            description: noembedData?.description || '',
+            type: oembedData.type || 'video',
+            source: 'oembed',
+        };
+        const descSection = structured.description ? `\n\n${structured.description}` : '\n\nYouTube video';
+        const cleanContent = `## 🎬 ${structured.title}\n\n**Channel:** [${structured.channel}](${structured.channelUrl})${descSection}`;
+        return { domain: 'youtube.com', type: 'video', structured, cleanContent };
+    }
+    return null;
+}