@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/budget.js

@@ -0,0 +1,324 @@
+/**
+ * Smart content distillation for WebPeel
+ *
+ * Intelligently compresses content to fit within a token budget using
+ * heuristic-based techniques — no LLM required.
+ *
+ * This is NOT simple truncation: it prioritises information-dense content
+ * and progressively removes lower-value sections while preserving structure.
+ *
+ * @module budget
+ */
+import { estimateTokens } from './markdown.js';
+/* ------------------------------------------------------------------ */
+/*  Constants                                                          */
+/* ------------------------------------------------------------------ */
+/**
+ * Heading patterns that indicate low-value boilerplate sections.
+ * When a heading matches, its entire section is removed.
+ */
+const BOILERPLATE_HEADING_PATTERNS = [
+    /^#{1,3}\s*(cookie(s| notice| policy| banner| consent)?|privacy( policy)?|terms( of (use|service))?|disclaimer|copyright)/i,
+    /^#{1,3}\s*(about us|contact( us)?|subscribe|newsletter|follow us|social media)/i,
+    /^#{1,3}\s*(related posts?|you may also|more from|popular posts?|trending|recent posts?)/i,
+    /^#{1,3}\s*(comments?|leave a (comment|reply)|tags?|categories?|share this)/i,
+    /^#{1,3}\s*(table of contents?|toc|index)/i,
+    /^#{1,3}\s*(advertisement|sponsored|promoted|ad(s| section)?)/i,
+    /^#{1,3}\s*(navigation|menu|sidebar|footer|header)/i,
+    /^#{1,3}\s*(sign[\s-]*up|log[\s-]*in|register|create( an)? account|get started)/i,
+];
+/** Maximum data rows to keep when compressing a markdown table */
+const MAX_TABLE_ROWS = 3;
+/** Tokens per listing item used for budget estimation in extract-all mode */
+export const TOKENS_PER_LISTING_ITEM = 50;
+/* ------------------------------------------------------------------ */
+/*  Public API                                                         */
+/* ------------------------------------------------------------------ */
+/**
+ * Distill content to fit within a token budget using smart compression.
+ *
+ * Strategy (applied progressively until within budget):
+ * 1. Remove image markdown — ![alt](url) → keep meaningful alt text
+ * 2. Remove boilerplate sections (cookie banners, nav headings, etc.)
+ * 3. Compress tables to MAX_TABLE_ROWS data rows
+ * 4. Collapse redundant whitespace
+ * 5. Remove low information-density paragraphs
+ * 6. Hard-truncate with notice as last resort
+ *
+ * @param content  The content string to distill
+ * @param budget   Maximum token budget (rough: 1 token ≈ 4 chars)
+ * @param format   Content format: 'markdown' | 'text' | 'json'
+ * @returns        Distilled content within the budget
+ */
+export function distillToBudget(content, budget, format) {
+    if (!content || budget <= 0)
+        return content;
+    if (estimateTokens(content) <= budget)
+        return content;
+    if (format === 'json') {
+        return distillJson(content, budget);
+    }
+    return distillMarkdown(content, budget);
+}
+/**
+ * Calculate how many listing items fit within a token budget.
+ *
+ * @param totalItems  Total available items
+ * @param budget      Token budget
+ * @returns           { maxItems, truncated, totalAvailable }
+ */
+export function budgetListings(totalItems, budget) {
+    const maxItems = Math.max(1, Math.floor(budget / TOKENS_PER_LISTING_ITEM));
+    const truncated = maxItems < totalItems;
+    return {
+        maxItems: truncated ? maxItems : totalItems,
+        truncated,
+        totalAvailable: totalItems,
+    };
+}
+/* ------------------------------------------------------------------ */
+/*  Markdown / text distillation                                       */
+/* ------------------------------------------------------------------ */
+function distillMarkdown(content, budget) {
+    let result = content;
+    // Step 1: Remove decorative images (minimal info loss)
+    if (estimateTokens(result) > budget) {
+        result = removeImages(result);
+    }
+    // Step 2: Remove boilerplate sections
+    if (estimateTokens(result) > budget) {
+        result = removeBoilerplateSections(result);
+    }
+    // Step 3: Compress wide tables
+    if (estimateTokens(result) > budget) {
+        result = compressTables(result);
+    }
+    // Step 4: Collapse redundant whitespace
+    if (estimateTokens(result) > budget) {
+        result = compressWhitespace(result);
+    }
+    // Step 5: Remove low-density paragraphs
+    if (estimateTokens(result) > budget) {
+        result = removeWeakParagraphs(result, budget);
+    }
+    // Step 6: Hard-truncate with notice as last resort
+    if (estimateTokens(result) > budget) {
+        result = hardTruncate(result, budget);
+    }
+    return result.trim();
+}
+/**
+ * Remove image markdown — replace informative alt text, drop decorative images.
+ */
+function removeImages(content) {
+    return content
+        .replace(/!\[([^\]]*)\]\([^)]+\)/g, (_, alt) => {
+        const a = alt.trim();
+        // Keep short, descriptive alt text as a text label
+        return a.length > 0 && a.length < 60 ? `[Image: ${a}]` : '';
+    })
+        // Clean up empty image labels that remain
+        .replace(/\[Image: \]\s*/g, '');
+}
+/**
+ * Remove boilerplate sections by matching heading patterns.
+ *
+ * When a boilerplate heading is found, everything up to (but not including)
+ * the next heading of equal or higher importance is removed.
+ */
+function removeBoilerplateSections(content) {
+    const lines = content.split('\n');
+    const result = [];
+    let skipping = false;
+    let skipDepth = 0;
+    for (const line of lines) {
+        const headingMatch = line.match(/^(#{1,6})\s/);
+        if (headingMatch) {
+            const depth = headingMatch[1].length;
+            // Stop skipping when we encounter a heading of equal or higher priority
+            if (skipping && depth <= skipDepth) {
+                skipping = false;
+            }
+            // Check if this heading starts a boilerplate section
+            if (!skipping && BOILERPLATE_HEADING_PATTERNS.some(p => p.test(line))) {
+                skipping = true;
+                skipDepth = depth;
+                continue;
+            }
+        }
+        if (!skipping) {
+            result.push(line);
+        }
+    }
+    return result.join('\n');
+}
+/**
+ * Compress markdown tables to MAX_TABLE_ROWS data rows + header + separator.
+ */
+function compressTables(content) {
+    const lines = content.split('\n');
+    const result = [];
+    let inTable = false;
+    let headerDone = false;
+    let separatorDone = false;
+    let dataRows = 0;
+    let truncatedNote = false;
+    for (const line of lines) {
+        const trimmed = line.trim();
+        const isTableRow = trimmed.startsWith('|') && trimmed.endsWith('|');
+        const isSeparator = isTableRow && /^\|[\s|:-]+\|$/.test(trimmed);
+        if (isTableRow) {
+            if (!inTable) {
+                // New table begins
+                inTable = true;
+                headerDone = false;
+                separatorDone = false;
+                dataRows = 0;
+                truncatedNote = false;
+            }
+            if (!headerDone) {
+                result.push(line);
+                headerDone = true;
+            }
+            else if (isSeparator && !separatorDone) {
+                result.push(line);
+                separatorDone = true;
+            }
+            else if (!isSeparator) {
+                if (dataRows < MAX_TABLE_ROWS) {
+                    result.push(line);
+                    dataRows++;
+                }
+                else if (!truncatedNote) {
+                    result.push(`| ... | *(${MAX_TABLE_ROWS}+ rows — additional rows omitted)* | ... |`);
+                    truncatedNote = true;
+                }
+                // Further rows silently dropped
+            }
+        }
+        else {
+            inTable = false;
+            result.push(line);
+        }
+    }
+    return result.join('\n');
+}
+/**
+ * Collapse runs of 3+ blank lines to a single blank line.
+ */
+function compressWhitespace(content) {
+    return content.replace(/\n{3,}/g, '\n\n');
+}
+/**
+ * Remove paragraphs scored as low information-density until within budget.
+ *
+ * Scoring heuristics:
+ * - Word count is the base score
+ * - Very short paragraphs (< 50 chars) are heavily penalised
+ * - Unusual avg word length penalised (nav menus, link lists)
+ * - Long bullet lists scored slightly lower
+ * - Headings and code blocks are never removed
+ */
+function removeWeakParagraphs(content, budget) {
+    const paragraphs = content.split('\n\n');
+    const scored = paragraphs.map((para, i) => {
+        const trimmed = para.trim();
+        const isHeading = /^#{1,6}\s/.test(trimmed);
+        const isCodeBlock = trimmed.startsWith('```');
+        const isHtmlComment = trimmed.startsWith('<!--');
+        // Never remove structural elements
+        if (isHeading || isCodeBlock || isHtmlComment) {
+            return { para, score: Number.MAX_SAFE_INTEGER, i };
+        }
+        // Strip markdown formatting for text analysis
+        const textOnly = trimmed.replace(/[#*_\[\]\(\)\-`|>~]/g, '');
+        const words = textOnly.split(/\s+/).filter(w => w.length > 0);
+        let score = words.length;
+        // Heavily penalise very short paragraphs (likely nav labels / single words)
+        if (textOnly.length < 50)
+            score *= 0.15;
+        // Penalise unusual avg word lengths (short = icon labels, long = data URIs)
+        const avgWordLen = words.length > 0 ? textOnly.length / words.length : 0;
+        if (avgWordLen < 3 || avgWordLen > 15)
+            score *= 0.4;
+        // Slightly penalise long bullet lists (repetitive structure)
+        const lines = trimmed.split('\n');
+        const bulletLines = lines.filter(l => /^[-*]\s/.test(l.trim()));
+        if (bulletLines.length > 3 && bulletLines.length === lines.length) {
+            score *= 0.7;
+        }
+        return { para, score, i };
+    });
+    // Sort ascending — weakest paragraphs first
+    const byScore = [...scored].sort((a, b) => a.score - b.score);
+    const removed = new Set();
+    let current = content;
+    for (const item of byScore) {
+        if (estimateTokens(current) <= budget)
+            break;
+        // Don't remove paragraphs with reasonable content
+        if (item.score >= 8)
+            break;
+        removed.add(item.i);
+        current = scored
+            .filter(s => !removed.has(s.i))
+            .map(s => s.para)
+            .join('\n\n');
+    }
+    return current;
+}
+/**
+ * Hard-truncate at a clean line boundary, appending a notice.
+ * Used only as the last resort after all other compression steps fail.
+ */
+function hardTruncate(content, budget) {
+    // Leave ~15 tokens for the truncation notice
+    const maxChars = Math.max((budget - 15) * 4, 0);
+    if (content.length <= maxChars)
+        return content;
+    // Find the last newline before the character limit
+    let cut = maxChars;
+    while (cut > 0 && content[cut] !== '\n')
+        cut--;
+    if (cut === 0)
+        cut = maxChars; // No newline found — hard cut
+    return content.slice(0, cut).trimEnd() + '\n\n[Content distilled to fit budget]';
+}
+/* ------------------------------------------------------------------ */
+/*  JSON distillation                                                  */
+/* ------------------------------------------------------------------ */
+/**
+ * Distill JSON content:
+ * - Arrays: binary-search for the maximum number of items that fit
+ * - Objects: fall back to text truncation
+ */
+function distillJson(content, budget) {
+    try {
+        const parsed = JSON.parse(content);
+        if (Array.isArray(parsed)) {
+            // Binary search for max items that fit within budget
+            let lo = 0;
+            let hi = parsed.length;
+            while (lo < hi) {
+                const mid = Math.ceil((lo + hi) / 2);
+                const slice = parsed.slice(0, mid);
+                if (estimateTokens(JSON.stringify(slice, null, 2)) <= budget) {
+                    lo = mid;
+                }
+                else {
+                    hi = mid - 1;
+                }
+            }
+            return JSON.stringify(parsed.slice(0, lo), null, 2);
+        }
+        // Non-array JSON — fall back to text truncation
+        const str = JSON.stringify(parsed, null, 2);
+        if (estimateTokens(str) <= budget)
+            return str;
+        return hardTruncate(str, budget);
+    }
+    catch {
+        // Invalid JSON — treat as plain text
+        return hardTruncate(content, budget);
+    }
+}

package/dist/core/business-intel.d.ts

@@ -0,0 +1,47 @@
+/**
+ * business-intel.ts — Extract structured business intelligence from a URL.
+ *
+ * Uses peel() to fetch the website, then extracts:
+ * - Name, description, industry from schema.org + OG tags
+ * - Products and pricing from /pricing and /plans pages
+ * - Tech stack from headers and script patterns
+ * - Social media links
+ * - Review aggregates
+ */
+export interface BusinessIntel {
+    name?: string;
+    description?: string;
+    industry?: string;
+    products?: string[];
+    pricing?: {
+        plan: string;
+        price: string;
+    }[];
+    reviews?: {
+        source: string;
+        rating: number;
+        count: number;
+    }[];
+    socialMedia?: {
+        platform: string;
+        url: string;
+    }[];
+    techStack?: string[];
+    employees?: string;
+    founded?: string;
+}
+/**
+ * Extract structured business intelligence from a website URL.
+ *
+ * Fetches the homepage and optionally the /pricing page, then extracts
+ * structured data including tech stack, social media, pricing, and more.
+ *
+ * @example
+ * ```typescript
+ * const intel = await getBusinessIntel('https://stripe.com');
+ * console.log(intel.name);       // "Stripe"
+ * console.log(intel.techStack);  // ["React", "Cloudflare", ...]
+ * console.log(intel.pricing);    // [{plan: "Starter", price: "$0"}, ...]
+ * ```
+ */
+export declare function getBusinessIntel(url: string): Promise<BusinessIntel>;

package/dist/core/business-intel.js

@@ -0,0 +1,279 @@
+/**
+ * business-intel.ts — Extract structured business intelligence from a URL.
+ *
+ * Uses peel() to fetch the website, then extracts:
+ * - Name, description, industry from schema.org + OG tags
+ * - Products and pricing from /pricing and /plans pages
+ * - Tech stack from headers and script patterns
+ * - Social media links
+ * - Review aggregates
+ */
+import { peel } from '../index.js';
+// ─── Tech stack detectors ─────────────────────────────────────────────────
+const TECH_PATTERNS = [
+    // Frontend frameworks
+    { name: 'React', pattern: /react(?:\.min)?\.js|__reactFiber|react-dom/i, type: 'script' },
+    { name: 'Vue.js', pattern: /vue(?:\.min)?\.js|Vue\.component|__vue_/i, type: 'script' },
+    { name: 'Angular', pattern: /angular(?:\.min)?\.js|ng-version|ng-app/i, type: 'script' },
+    { name: 'Next.js', pattern: /__NEXT_DATA__|next\/dist\/|_next\/static/i, type: 'script' },
+    { name: 'Nuxt.js', pattern: /__NUXT__|_nuxt\/|nuxtjs\.org/i, type: 'script' },
+    { name: 'Svelte', pattern: /svelte\/internal|SvelteComponent/i, type: 'script' },
+    // E-commerce & CMS
+    { name: 'Shopify', pattern: /shopify\.com|Shopify\.theme|cdn\.shopify/i, type: 'script' },
+    { name: 'WordPress', pattern: /wp-content\/|wp-includes\/|WordPress/i, type: 'script' },
+    { name: 'Webflow', pattern: /webflow\.com|Webflow\.require/i, type: 'script' },
+    { name: 'Squarespace', pattern: /squarespace\.com|SQUARESPACE_ROLLUPS/i, type: 'script' },
+    { name: 'Wix', pattern: /wix\.com|wixstatic\.com/i, type: 'script' },
+    // Analytics & marketing
+    { name: 'Google Analytics', pattern: /google-analytics\.com|gtag\(|ga\('send/i, type: 'script' },
+    { name: 'Segment', pattern: /segment\.com|analytics\.identify/i, type: 'script' },
+    { name: 'Mixpanel', pattern: /mixpanel\.com|mixpanel\.track/i, type: 'script' },
+    { name: 'Intercom', pattern: /intercom\.io|window\.Intercom/i, type: 'script' },
+    { name: 'Hubspot', pattern: /hubspot\.com|hs-scripts\.com/i, type: 'script' },
+    { name: 'Stripe', pattern: /js\.stripe\.com|Stripe\(/i, type: 'script' },
+    // Server / infrastructure (detected via headers)
+    { name: 'Vercel', pattern: /vercel/i, type: 'header' },
+    { name: 'Netlify', pattern: /netlify/i, type: 'header' },
+    { name: 'Cloudflare', pattern: /cloudflare/i, type: 'header' },
+    { name: 'AWS', pattern: /amazonaws\.com|x-amz-/i, type: 'header' },
+    { name: 'Nginx', pattern: /nginx/i, type: 'header' },
+    { name: 'Apache', pattern: /apache/i, type: 'header' },
+];
+const SOCIAL_PATTERNS = [
+    { platform: 'Twitter/X', pattern: /(?:twitter\.com|x\.com)\/([^/"?\s]+)/i },
+    { platform: 'LinkedIn', pattern: /linkedin\.com\/(?:company|in)\/([^/"?\s]+)/i },
+    { platform: 'Facebook', pattern: /facebook\.com\/([^/"?\s]+)/i },
+    { platform: 'Instagram', pattern: /instagram\.com\/([^/"?\s]+)/i },
+    { platform: 'YouTube', pattern: /youtube\.com\/(?:channel|c|@)\/([^/"?\s]+)/i },
+    { platform: 'GitHub', pattern: /github\.com\/([^/"?\s]+)/i },
+    { platform: 'Discord', pattern: /discord\.(?:gg|com\/invite)\/([^/"?\s]+)/i },
+    { platform: 'TikTok', pattern: /tiktok\.com\/@([^/"?\s]+)/i },
+];
+const INDUSTRY_KEYWORDS = {
+    'SaaS / Software': ['software', 'saas', 'platform', 'api', 'developer', 'cloud', 'app'],
+    'E-commerce': ['shop', 'store', 'buy', 'cart', 'checkout', 'product', 'shipping'],
+    'Finance / Fintech': ['payment', 'invoice', 'banking', 'crypto', 'invest', 'finance', 'loan'],
+    'Healthcare': ['health', 'medical', 'patient', 'clinic', 'doctor', 'hospital', 'pharma'],
+    'Education': ['course', 'learn', 'training', 'education', 'school', 'university', 'tutor'],
+    'Marketing': ['marketing', 'seo', 'email campaign', 'crm', 'lead', 'analytics'],
+    'AI / Machine Learning': ['ai', 'machine learning', 'nlp', 'model', 'inference', 'llm'],
+    'Food & Restaurant': ['restaurant', 'food', 'menu', 'delivery', 'catering', 'dining'],
+    'Travel': ['travel', 'hotel', 'flight', 'booking', 'tourism', 'vacation'],
+    'Real Estate': ['real estate', 'property', 'mortgage', 'rent', 'apartment', 'lease'],
+    'Media / Content': ['news', 'blog', 'podcast', 'video', 'streaming', 'media', 'content'],
+};
+function extractPricing(content) {
+    const plans = [];
+    const seen = new Set();
+    // Look for plan name + price in proximity
+    const lines = content.split('\n');
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const planMatch = line.match(/\b(free|starter|basic|pro|professional|business|enterprise|premium|growth|scale|team|individual|personal)\b/i);
+        if (!planMatch)
+            continue;
+        // Search nearby lines for a price
+        const context = lines.slice(Math.max(0, i - 1), Math.min(lines.length, i + 5)).join(' ');
+        const priceMatch = context.match(/\$([\d,]+(?:\.\d{2})?)/);
+        if (priceMatch) {
+            const key = `${planMatch[1].toLowerCase()}:${priceMatch[0]}`;
+            if (!seen.has(key)) {
+                seen.add(key);
+                plans.push({ plan: planMatch[1], price: priceMatch[0] });
+            }
+        }
+        else if (/\bfree\b/i.test(planMatch[1])) {
+            const key = `${planMatch[1].toLowerCase()}:$0`;
+            if (!seen.has(key)) {
+                seen.add(key);
+                plans.push({ plan: planMatch[1], price: '$0' });
+            }
+        }
+    }
+    return plans.slice(0, 8);
+}
+function detectTechStack(content, headers) {
+    const detected = new Set();
+    // Check headers
+    const headerString = Object.entries(headers)
+        .map(([k, v]) => `${k}: ${v}`)
+        .join('\n')
+        .toLowerCase();
+    // Check content (HTML/scripts)
+    for (const tech of TECH_PATTERNS) {
+        if (tech.type === 'header') {
+            if (tech.pattern.test(headerString))
+                detected.add(tech.name);
+        }
+        else {
+            if (tech.pattern.test(content))
+                detected.add(tech.name);
+        }
+    }
+    return [...detected].sort();
+}
+function extractSocialMedia(content) {
+    const found = [];
+    const seen = new Set();
+    for (const { platform, pattern } of SOCIAL_PATTERNS) {
+        const matches = content.matchAll(new RegExp(pattern.source, 'gi'));
+        for (const match of matches) {
+            const fullMatch = match[0];
+            if (!fullMatch.includes('share') && !fullMatch.includes('intent') && !seen.has(fullMatch)) {
+                seen.add(fullMatch);
+                // Build full URL
+                let url = fullMatch;
+                if (!url.startsWith('http'))
+                    url = 'https://' + url;
+                found.push({ platform, url });
+                break; // one per platform
+            }
+        }
+    }
+    return found;
+}
+function detectIndustry(text) {
+    const lower = text.toLowerCase();
+    const scores = {};
+    for (const [industry, keywords] of Object.entries(INDUSTRY_KEYWORDS)) {
+        scores[industry] = keywords.filter(kw => lower.includes(kw)).length;
+    }
+    const best = Object.entries(scores)
+        .filter(([, score]) => score > 0)
+        .sort(([, a], [, b]) => b - a)[0];
+    return best ? best[0] : undefined;
+}
+function extractSchemaOrgData(content) {
+    const result = {};
+    // JSON-LD: look for Organization or LocalBusiness schema
+    const jsonLdMatch = content.match(/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi);
+    if (jsonLdMatch) {
+        for (const block of jsonLdMatch) {
+            try {
+                const json = JSON.parse(block.replace(/<script[^>]*>|<\/script>/gi, ''));
+                const items = Array.isArray(json) ? json : [json];
+                for (const item of items) {
+                    if (!result.name && item.name)
+                        result.name = item.name;
+                    if (!result.description && item.description)
+                        result.description = item.description;
+                    if (!result.founded && item.foundingDate)
+                        result.founded = String(item.foundingDate);
+                    if (!result.employees && item.numberOfEmployees) {
+                        const emp = item.numberOfEmployees;
+                        result.employees = typeof emp === 'object' ? `${emp.minValue ?? ''}–${emp.maxValue ?? ''}` : String(emp);
+                    }
+                    // Review aggregate
+                    if (item.aggregateRating) {
+                        result.reviews = result.reviews || [];
+                        result.reviews.push({
+                            source: 'Schema.org',
+                            rating: parseFloat(item.aggregateRating.ratingValue) || 0,
+                            count: parseInt(item.aggregateRating.reviewCount) || 0,
+                        });
+                    }
+                }
+            }
+            catch {
+                // Invalid JSON-LD — skip
+            }
+        }
+    }
+    // Open Graph fallback
+    if (!result.name) {
+        const ogTitle = content.match(/property="og:title"\s+content="([^"]+)"/);
+        if (ogTitle)
+            result.name = ogTitle[1];
+    }
+    if (!result.description) {
+        const ogDesc = content.match(/(?:property="og:description"|name="description")\s+content="([^"]+)"/);
+        if (ogDesc)
+            result.description = ogDesc[1];
+    }
+    return result;
+}
+// ─── Main export ──────────────────────────────────────────────────────────
+/**
+ * Extract structured business intelligence from a website URL.
+ *
+ * Fetches the homepage and optionally the /pricing page, then extracts
+ * structured data including tech stack, social media, pricing, and more.
+ *
+ * @example
+ * ```typescript
+ * const intel = await getBusinessIntel('https://stripe.com');
+ * console.log(intel.name);       // "Stripe"
+ * console.log(intel.techStack);  // ["React", "Cloudflare", ...]
+ * console.log(intel.pricing);    // [{plan: "Starter", price: "$0"}, ...]
+ * ```
+ */
+export async function getBusinessIntel(url) {
+    // Normalize URL
+    if (!url.startsWith('http'))
+        url = 'https://' + url;
+    const parsed = new URL(url);
+    const origin = parsed.origin;
+    // 1. Fetch homepage
+    const homeResult = await peel(url, {
+        format: 'html',
+        timeout: 15000,
+    });
+    const homeContent = homeResult.content || '';
+    const homeHtml = homeResult.rawHtml || homeContent;
+    // 2. Extract schema.org / OG data
+    const schemaData = extractSchemaOrgData(homeHtml);
+    // 3. Detect tech stack from content + headers
+    const responseHeaders = homeResult.headers || {};
+    const techStack = detectTechStack(homeHtml + homeContent, responseHeaders);
+    // 4. Extract social media links
+    const socialMedia = extractSocialMedia(homeHtml + homeContent);
+    // 5. Detect industry from description + content
+    const textForIndustry = [schemaData.description, homeContent].filter(Boolean).join(' ');
+    const industry = detectIndustry(textForIndustry);
+    // 6. Try to fetch pricing page (best-effort)
+    let pricing = [];
+    const pricingPaths = ['/pricing', '/plans', '/pricing-plans', '/subscribe'];
+    for (const path of pricingPaths) {
+        try {
+            const pricingUrl = origin + path;
+            const pricingResult = await peel(pricingUrl, { timeout: 8000 });
+            if (pricingResult.content && pricingResult.content.length > 200) {
+                pricing = extractPricing(pricingResult.content);
+                if (pricing.length > 0)
+                    break;
+            }
+        }
+        catch {
+            // Pricing page not found — continue
+        }
+    }
+    // If no pricing found from pricing page, try extracting from homepage
+    if (pricing.length === 0) {
+        pricing = extractPricing(homeContent);
+    }
+    // 7. Extract products list from homepage (look for feature/product lists)
+    const products = [];
+    const productSection = homeContent.match(/(?:products?|features?|solutions?)[^\n]*\n((?:[^\n]+\n){1,10})/i);
+    if (productSection) {
+        const lines = productSection[1]
+            .split('\n')
+            .map(l => l.trim())
+            .filter(l => l.length > 3 && l.length < 80 && !l.startsWith('#') && !l.startsWith('http'));
+        products.push(...lines.slice(0, 8));
+    }
+    // Build final result
+    const intel = {
+        ...schemaData,
+    };
+    if (industry)
+        intel.industry = industry;
+    if (products.length > 0)
+        intel.products = products;
+    if (pricing.length > 0)
+        intel.pricing = pricing;
+    if (socialMedia.length > 0)
+        intel.socialMedia = socialMedia;
+    if (techStack.length > 0)
+        intel.techStack = techStack;
+    return intel;
+}

package/dist/core/cache.d.ts

@@ -0,0 +1,13 @@
+/**
+ * In-memory LRU response cache.
+ */
+export interface CacheResult<T = unknown> {
+    value: T;
+    stale: boolean;
+}
+export declare function getCached<T = unknown>(url: string): T | null;
+export declare function getCachedWithSWR<T = unknown>(url: string): CacheResult<T> | null;
+export declare function markRevalidating(url: string): boolean;
+export declare function setCached<T = unknown>(url: string, result: T): void;
+export declare function clearCache(): void;
+export declare function setCacheTTL(ms: number): void;