headhunt 1.0.0

package/src/scraper.js

@@ -0,0 +1,815 @@
+const axios = require('axios');
+const cheerio = require('cheerio');
+const { URL } = require('url');
+const { fmt } = require('./colors');
+const { httpsAgent, USER_AGENT, SCORE_WEIGHTS } = require('./constants');
+const log = require('./logger');
+
+class SEOScraper {
+    constructor(url, options = {}) {
+        this.url = url;
+        this.options = options;
+        this.html = '';
+        this.$ = null;
+        this.fetchTime = 0;
+        this.httpStatus = null;
+        this.responseHeaders = {};
+        this.finalUrl = url;
+
+        this.metadata = {
+            url,
+            finalUrl: url,
+            timestamp: new Date().toISOString(),
+            httpStatus: null,
+            responseHeaders: {},
+            fetchTimeMs: 0,
+            seo: {},
+            scores: {},
+            recommendations: [],
+            overallScore: 0,
+            grade: '',
+            summary: '',
+        };
+    }
+
+    async fetchPage() {
+        const start = Date.now();
+        log.info(`Fetching \u2192 ${fmt.cyan(this.url)}`);
+
+        const response = await axios.get(this.url, {
+            headers: {
+                'User-Agent': USER_AGENT,
+                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                'Accept-Language': 'en-US,en;q=0.5',
+                'Accept-Encoding': 'gzip, deflate, br',
+                'Connection': 'keep-alive',
+                'Upgrade-Insecure-Requests': '1',
+                'Cache-Control': 'no-cache',
+            },
+            httpsAgent,
+            timeout: 30000,
+            maxRedirects: 10,
+        });
+
+        this.fetchTime = Date.now() - start;
+        this.html = response.data;
+        this.$ = cheerio.load(this.html);
+        this.httpStatus = response.status;
+        this.responseHeaders = response.headers;
+        this.finalUrl = response.request?.res?.responseUrl || this.url;
+
+        this.metadata.httpStatus = this.httpStatus;
+        this.metadata.fetchTimeMs = this.fetchTime;
+        this.metadata.finalUrl = this.finalUrl;
+        this.metadata.responseHeaders = {
+            'content-type': response.headers['content-type'] || null,
+            'x-powered-by': response.headers['x-powered-by'] || null,
+            'server': response.headers['server'] || null,
+            'cache-control': response.headers['cache-control'] || null,
+            'strict-transport-security': response.headers['strict-transport-security'] || null,
+            'x-frame-options': response.headers['x-frame-options'] || null,
+            'x-content-type-options': response.headers['x-content-type-options'] || null,
+            'content-security-policy': response.headers['content-security-policy'] ? '(present)' : null,
+            'referrer-policy': response.headers['referrer-policy'] || null,
+            'permissions-policy': response.headers['permissions-policy'] ? '(present)' : null,
+        };
+
+        log.ok(`Fetched in ${fmt.yellow(this.fetchTime + 'ms')} \u2014 HTTP ${fmt.green(this.httpStatus)}`);
+        if (this.finalUrl !== this.url) {
+            log.info(`Redirected \u2192 ${fmt.dim(this.finalUrl)}`);
+        }
+    }
+
+    extractBasicMetadata() {
+        const title = this.$('title').text().trim() || null;
+        const desc = this.$('meta[name="description"]').attr('content') || null;
+        const keywords = this.$('meta[name="keywords"]').attr('content') || null;
+        const canonical = this.$('link[rel="canonical"]').attr('href') || null;
+        const robots = this.$('meta[name="robots"]').attr('content') || null;
+        const googlebot = this.$('meta[name="googlebot"]').attr('content') || null;
+        const viewport = this.$('meta[name="viewport"]').attr('content') || null;
+        const charset = this.$('meta[charset]').attr('charset') ||
+            (this.$('meta[http-equiv="Content-Type"]').attr('content') || '').match(/charset=([^\s;]+)/i)?.[1] || null;
+        const language = this.$('html').attr('lang') || null;
+        const themeColor = this.$('meta[name="theme-color"]').attr('content') || null;
+        const author = this.$('meta[name="author"]').attr('content') || null;
+        const generator = this.$('meta[name="generator"]').attr('content') || null;
+        const rating = this.$('meta[name="rating"]').attr('content') || null;
+        const revisit = this.$('meta[name="revisit-after"]').attr('content') || null;
+
+        const cms = this.detectCMS(generator);
+
+        const hreflang = [];
+        this.$('link[rel="alternate"][hreflang]').each((i, el) => {
+            hreflang.push({
+                lang: this.$(el).attr('hreflang'),
+                href: this.$(el).attr('href'),
+            });
+        });
+
+        const prevPage = this.$('link[rel="prev"]').attr('href') || null;
+        const nextPage = this.$('link[rel="next"]').attr('href') || null;
+
+        const basic = {
+            title, titleLength: title?.length ?? 0,
+            metaDescription: desc, descriptionLength: desc?.length ?? 0,
+            keywords, keywordCount: keywords ? keywords.split(',').length : 0,
+            canonical, canonicalMatchesFinal: canonical ? this.urlsMatch(canonical, this.finalUrl) : null,
+            robots, googlebot,
+            viewport, charset, language, themeColor,
+            author, generator, cms, rating, revisit,
+            hreflang, pagination: { prev: prevPage, next: nextPage },
+        };
+
+        this.metadata.seo.basic = basic;
+    }
+
+    detectCMS(generator) {
+        const html = this.html.toLowerCase();
+
+        if (html.includes('wp-content') || html.includes('wp-json')) return 'WordPress';
+        if (html.includes('shopify')) return 'Shopify';
+        if (html.includes('drupal')) return 'Drupal';
+        if (html.includes('joomla')) return 'Joomla';
+        if (html.includes('squarespace')) return 'Squarespace';
+        if (html.includes('webflow')) return 'Webflow';
+        if (html.includes('wix.com')) return 'Wix';
+        if (html.includes('ghost')) return 'Ghost';
+        if (html.includes('nextjs') || html.includes('__next')) return 'Next.js';
+        if (html.includes('nuxt') || html.includes('__nuxt')) return 'Nuxt.js';
+        if (generator) return generator.split(' ')[0];
+        return null;
+    }
+
+    urlsMatch(a, b) {
+        try {
+            const ua = new URL(a), ub = new URL(b);
+            return ua.hostname === ub.hostname && ua.pathname.replace(/\/$/, '') === ub.pathname.replace(/\/$/, '');
+        } catch { return null; }
+    }
+
+    extractOpenGraphMetadata() {
+        const og = {};
+        this.$('meta[property^="og:"]').each((i, el) => {
+            const prop = this.$(el).attr('property')?.replace('og:', '');
+            const content = this.$(el).attr('content');
+            if (prop && content) og[prop] = content;
+        });
+        this.metadata.seo.openGraph = og;
+    }
+
+    extractTwitterMetadata() {
+        const tw = {};
+        this.$('meta[name^="twitter:"]').each((i, el) => {
+            const name = this.$(el).attr('name')?.replace('twitter:', '');
+            const content = this.$(el).attr('content');
+            if (name && content) tw[name] = content;
+        });
+        this.metadata.seo.twitter = tw;
+    }
+
+    extractSchemaMetadata() {
+        const jsonLd = [];
+        const schemaTypes = [];
+
+        this.$('script[type="application/ld+json"]').each((i, el) => {
+            try {
+                const parsed = JSON.parse(this.$(el).html());
+                jsonLd.push(parsed);
+                const type = parsed['@type'];
+                if (type) schemaTypes.push(Array.isArray(type) ? type.join(', ') : type);
+            } catch (_) { }
+        });
+
+        const microdata = {};
+        this.$('[itemscope]').each((i, el) => {
+            const itemtype = this.$(el).attr('itemtype');
+            if (!itemtype) return;
+            const type = itemtype.split('/').pop();
+            if (!microdata[type]) microdata[type] = [];
+            const item = {};
+            this.$(el).find('[itemprop]').each((j, prop) => {
+                const name = this.$(prop).attr('itemprop');
+                item[name] = this.$(prop).attr('content') || this.$(prop).text().trim().substring(0, 200);
+            });
+            microdata[type].push(item);
+        });
+
+        this.metadata.seo.schema = { jsonLd, schemaTypes, microdata };
+    }
+
+    extractTechnicalMetadata() {
+        const $ = this.$;
+        const headers = this.metadata.responseHeaders;
+
+        const technical = {
+            doctype: this.html.match(/<!DOCTYPE[^>]*>/i)?.[0] || null,
+            htmlVersion: this.getHtmlVersion(),
+            hasViewport: $('meta[name="viewport"]').length > 0,
+            hasCharset: $('meta[charset]').length > 0,
+            hasCanonical: $('link[rel="canonical"]').length > 0,
+            hasRobots: $('meta[name="robots"]').length > 0,
+            hasSitemap: $('link[rel="sitemap"]').length > 0,
+            hasRSS: $('link[type="application/rss+xml"]').length > 0,
+            hasAMP: $('link[rel="amphtml"]').length > 0,
+            hasFavicon: $('link[rel="icon"], link[rel="shortcut icon"]').length > 0,
+            hasManifest: $('link[rel="manifest"]').length > 0,
+            hasServiceWorker: this.html.includes('serviceWorker') || this.html.includes('service-worker'),
+            sitemapUrl: $('link[rel="sitemap"]').attr('href') || null,
+            rssUrl: $('link[type="application/rss+xml"]').attr('href') || null,
+            ampUrl: $('link[rel="amphtml"]').attr('href') || null,
+            manifestUrl: $('link[rel="manifest"]').attr('href') || null,
+
+            https: this.url.startsWith('https://'),
+            hsts: !!headers['strict-transport-security'],
+            xFrameOptions: !!headers['x-frame-options'],
+            xContentTypeOptions: !!headers['x-content-type-options'],
+            csp: !!headers['content-security-policy'],
+            referrerPolicy: !!headers['referrer-policy'],
+            permissionsPolicy: !!headers['permissions-policy'],
+            serverInfo: headers['server'] || null,
+            poweredBy: headers['x-powered-by'] || null,
+
+            wasRedirected: this.finalUrl !== this.url,
+            redirectTarget: this.finalUrl !== this.url ? this.finalUrl : null,
+
+            fetchTimeMs: this.fetchTime,
+            httpStatus: this.httpStatus,
+        };
+
+        this.metadata.seo.technical = technical;
+    }
+
+    getHtmlVersion() {
+        const d = (this.html.match(/<!DOCTYPE[^>]*>/i)?.[0] || '').toLowerCase();
+        if (!d) return 'Unknown';
+        if (d === '<!doctype html>') return 'HTML5';
+        if (d.includes('html 4.01')) return 'HTML 4.01';
+        if (d.includes('xhtml 1.0')) return 'XHTML 1.0';
+        if (d.includes('xhtml 1.1')) return 'XHTML 1.1';
+        return 'Unknown';
+    }
+
+    extractHeadings() {
+        const headings = { h1: [], h2: [], h3: [], h4: [], h5: [], h6: [], counts: {} };
+        let order = [];
+
+        for (let i = 1; i <= 6; i++) {
+            const tag = `h${i}`;
+            this.$(tag).each((idx, el) => {
+                const text = this.$(el).text().trim();
+                const id = this.$(el).attr('id') || null;
+                if (text) {
+                    headings[tag].push({ text: text.substring(0, 200), length: text.length, id });
+                    order.push({ level: i, text: text.substring(0, 100) });
+                }
+            });
+            headings.counts[tag] = headings[tag].length;
+        }
+
+        const issues = [];
+        let lastLevel = 0;
+        for (const h of order) {
+            if (lastLevel && h.level > lastLevel + 1) {
+                issues.push(`Skipped from H${lastLevel} to H${h.level}: "${h.text.substring(0, 60)}"`);
+            }
+            lastLevel = h.level;
+        }
+
+        headings.hierarchyIssues = issues;
+        headings.order = order.slice(0, 30);
+
+        this.metadata.seo.headings = headings;
+    }
+
+    extractImages() {
+        if (this.options.noImages) { this.metadata.seo.images = { skipped: true }; return; }
+
+        const images = { withAlt: [], withoutAlt: [], lazyLoaded: [], oversized: [], counts: {} };
+
+        this.$('img').each((i, el) => {
+            const src = this.$(el).attr('src') || this.$(el).attr('data-src');
+            const alt = this.$(el).attr('alt');
+            const loading = this.$(el).attr('loading');
+            const width = parseInt(this.$(el).attr('width') || '0');
+            const height = parseInt(this.$(el).attr('height') || '0');
+            const srcset = this.$(el).attr('srcset');
+
+            if (!src) return;
+
+            const img = {
+                src: src.substring(0, 300),
+                alt: alt || null,
+                hasAlt: alt !== undefined && alt !== null,
+                altEmpty: alt === '',
+                isLazy: loading === 'lazy',
+                hasDimensions: width > 0 && height > 0,
+                hasSrcset: !!srcset,
+                title: this.$(el).attr('title') || null,
+            };
+
+            if (img.hasAlt && !img.altEmpty) images.withAlt.push(img);
+            else images.withoutAlt.push(img);
+            if (img.isLazy) images.lazyLoaded.push(img);
+        });
+
+        const total = images.withAlt.length + images.withoutAlt.length;
+        images.counts = {
+            total,
+            withAlt: images.withAlt.length,
+            withoutAlt: images.withoutAlt.length,
+            lazyLoaded: images.lazyLoaded.length,
+            altCoverage: total > 0 ? Math.round((images.withAlt.length / total) * 100) : 100,
+        };
+
+        this.metadata.seo.images = images;
+    }
+
+    extractLinks() {
+        if (this.options.noLinks) { this.metadata.seo.links = { skipped: true }; return; }
+
+        const links = { internal: [], external: [], nofollow: [], sponsored: [], ugc: [] };
+        const seen = new Set();
+
+        this.$('a[href]').each((i, el) => {
+            const href = this.$(el).attr('href') || '';
+            const rel = (this.$(el).attr('rel') || '').toLowerCase();
+            const text = this.$(el).text().trim().substring(0, 100);
+
+            if (!href || href.startsWith('#') || href.startsWith('javascript:') || href.startsWith('mailto:') || href.startsWith('tel:')) return;
+
+            const link = {
+                url: href.substring(0, 500),
+                text: text || null,
+                rel: this.$(el).attr('rel') || null,
+                isNofollow: rel.includes('nofollow'),
+                isSponsored: rel.includes('sponsored'),
+                isUGC: rel.includes('ugc'),
+                title: this.$(el).attr('title') || null,
+                hasText: !!text,
+            };
+
+            const key = href;
+            if (!seen.has(key)) {
+                seen.add(key);
+                if (href.startsWith('http')) {
+                    try {
+                        const isInternal = new URL(href).hostname === new URL(this.url).hostname;
+                        if (isInternal) links.internal.push(link);
+                        else links.external.push(link);
+                    } catch { links.external.push(link); }
+                } else {
+                    links.internal.push(link);
+                }
+            }
+
+            if (link.isNofollow) links.nofollow.push(link);
+            if (link.isSponsored) links.sponsored.push(link);
+            if (link.isUGC) links.ugc.push(link);
+        });
+
+        const emptyTextLinks = [...links.internal, ...links.external].filter(l => !l.hasText);
+
+        links.counts = {
+            total: links.internal.length + links.external.length,
+            internal: links.internal.length,
+            external: links.external.length,
+            nofollow: links.nofollow.length,
+            emptyText: emptyTextLinks.length,
+        };
+
+        this.metadata.seo.links = links;
+    }
+
+    extractContentMetadata() {
+        const $ = this.$;
+
+        $('script, style, noscript, nav, footer, header').remove();
+        const bodyText = $('body').text().replace(/\s+/g, ' ').trim();
+        const wordCount = bodyText ? bodyText.split(/\s+/).length : 0;
+
+        const title = this.metadata.seo.basic?.title || '';
+        const titleWords = title.toLowerCase().split(/\s+/).filter(w => w.length > 3);
+        const densities = {};
+        if (bodyText && titleWords.length) {
+            const lowerText = bodyText.toLowerCase();
+            for (const word of titleWords) {
+                const count = (lowerText.match(new RegExp(`\\b${word}\\b`, 'g')) || []).length;
+                densities[word] = {
+                    count,
+                    density: wordCount > 0 ? ((count / wordCount) * 100).toFixed(2) + '%' : '0%',
+                };
+            }
+        }
+
+        const paragraphs = $('p').filter((i, el) => $(el).text().trim().length > 50).length;
+        const iframes = $('iframe').length;
+        const htmlLang = this.metadata.seo.basic?.language || '';
+
+        this.metadata.seo.content = {
+            wordCount,
+            paragraphCount: paragraphs,
+            iframeCount: iframes,
+            estimatedReadingMinutes: Math.ceil(wordCount / 200),
+            keywordDensities: densities,
+            htmlLang,
+        };
+    }
+
+    extractPerformanceMetrics() {
+        const $ = this.$;
+        const html = this.html;
+
+        const scripts = $('script[src]').map((i, el) => $(el).attr('src')).get();
+        const styles = $('link[rel="stylesheet"]').map((i, el) => $(el).attr('href')).get();
+
+        const renderBlocking = {
+            scripts: scripts.filter(s => s).length,
+            styles: styles.filter(s => s).length,
+        };
+
+        const headScripts = $('head script[src]:not([defer]):not([async])').length;
+        const deferScripts = $('script[defer]').length;
+        const asyncScripts = $('script[async]').length;
+        const moduleScripts = $('script[type="module"]').length;
+
+        const htmlSize = Buffer.byteLength(html, 'utf8');
+        const estimatedResources = renderBlocking.styles * 15000 + renderBlocking.scripts * 50000 + $('img').length * 80000;
+
+        this.metadata.seo.performance = {
+            htmlSizeBytes: htmlSize,
+            htmlSizeKB: Math.round(htmlSize / 1024),
+            fetchTimeMs: this.fetchTime,
+            totalImages: $('img').length,
+            totalScripts: $('script').length,
+            totalExternalScripts: scripts.length,
+            totalStylesheets: styles.length,
+            totalInlineScripts: $('script:not([src])').length,
+            totalInlineStyles: $('style').length,
+            headScripts,
+            deferScripts,
+            asyncScripts,
+            moduleScripts,
+            renderBlockingRisk: headScripts > 0,
+            estimated: {
+                externalResourcesBytes: estimatedResources,
+                totalPageBytes: htmlSize + estimatedResources,
+                totalPageKB: Math.round((htmlSize + estimatedResources) / 1024),
+            },
+        };
+    }
+
+    computeScores() {
+        const s = this.metadata.seo;
+        const scores = {};
+
+        {
+            let score = 0; const max = SCORE_WEIGHTS.title.max;
+            const title = s.basic?.title;
+            if (title) {
+                score += 5;
+                if (title.length >= 30 && title.length <= 60) score += 7;
+                else if (title.length >= 20 && title.length <= 70) score += 4;
+                else score += 1;
+                if (title.length > 0) score += 3;
+            }
+            scores.title = { score: Math.min(score, max), max };
+        }
+
+        {
+            let score = 0; const max = SCORE_WEIGHTS.description.max;
+            const desc = s.basic?.metaDescription;
+            if (desc) {
+                score += 4;
+                if (desc.length >= 120 && desc.length <= 160) score += 6;
+                else if (desc.length >= 80 && desc.length <= 200) score += 3;
+                else score += 1;
+                if (desc.length > 0) score += 2;
+            }
+            scores.description = { score: Math.min(score, max), max };
+        }
+
+        {
+            let score = 0; const max = SCORE_WEIGHTS.headings.max;
+            const h = s.headings;
+            if (h) {
+                const h1count = h.counts.h1 || 0;
+                if (h1count === 1) score += 5;
+                else if (h1count > 1) score += 2;
+                if (h.counts.h2 > 0) score += 3;
+                if (h.hierarchyIssues?.length === 0) score += 2;
+            }
+            scores.headings = { score: Math.min(score, max), max };
+        }
+
+        {
+            let score = 0; const max = SCORE_WEIGHTS.canonical.max;
+            if (s.basic?.canonical) {
+                score += 3;
+                if (s.basic.canonicalMatchesFinal) score += 2;
+                else score += 1;
+            }
+            scores.canonical = { score: Math.min(score, max), max };
+        }
+
+        {
+            let score = 0; const max = SCORE_WEIGHTS.robots.max;
+            const robots = s.basic?.robots;
+            if (robots) {
+                if (robots.includes('noindex')) score += 1;
+                else score += 3;
+            } else {
+                score += 2;
+            }
+            scores.robots = { score: Math.min(score, max), max };
+        }
+
+        {
+            let score = 0; const max = SCORE_WEIGHTS.openGraph.max;
+            const og = s.openGraph || {};
+            if (og.title) score += 2;
+            if (og.description) score += 2;
+            if (og.image) score += 2;
+            if (og.type) score += 1;
+            if (og.url) score += 1;
+            scores.openGraph = { score: Math.min(score, max), max };
+        }
+
+        {
+            let score = 0; const max = SCORE_WEIGHTS.twitterCard.max;
+            const tw = s.twitter || {};
+            if (tw.card) score += 2;
+            if (tw.title) score += 1;
+            if (tw.description) score += 1;
+            if (tw.image) score += 1;
+            scores.twitterCard = { score: Math.min(score, max), max };
+        }
+
+        {
+            let score = 0; const max = SCORE_WEIGHTS.schema.max;
+            const schema = s.schema;
+            if (schema?.jsonLd?.length > 0) {
+                score += 5;
+                if (schema.jsonLd.length >= 2) score += 2;
+                if (schema.schemaTypes?.some(t => ['WebPage', 'WebSite', 'Article', 'Product', 'Organization', 'LocalBusiness', 'BreadcrumbList'].includes(t))) score += 3;
+            }
+            if (Object.keys(schema?.microdata || {}).length > 0) score += 2;
+            scores.schema = { score: Math.min(score, max), max };
+        }
+
+        {
+            let score = 0; const max = SCORE_WEIGHTS.images.max;
+            const img = s.images;
+            if (!img || img.skipped) { scores.images = { score: max, max }; }
+            else {
+                const coverage = img.counts.altCoverage || 100;
+                if (coverage === 100) score += 4;
+                else if (coverage >= 80) score += 3;
+                else if (coverage >= 50) score += 1;
+
+                const lazyRatio = img.counts.total > 0 ? img.counts.lazyLoaded / img.counts.total : 1;
+                if (lazyRatio >= 0.5) score += 2;
+                if (img.counts.total === 0 || coverage >= 80) score += 2;
+                scores.images = { score: Math.min(score, max), max };
+            }
+        }
+
+        {
+            let score = 0; const max = SCORE_WEIGHTS.mobile.max;
+            if (s.technical?.hasViewport) score += 3;
+            const vp = s.basic?.viewport || '';
+            if (vp.includes('width=device-width')) score += 2;
+            scores.mobile = { score: Math.min(score, max), max };
+        }
+
+        {
+            let score = 0; const max = SCORE_WEIGHTS.performance.max;
+            const p = s.performance;
+            if (p) {
+                if (p.htmlSizeKB < 100) score += 2;
+                else if (p.htmlSizeKB < 200) score += 1;
+                if (p.fetchTimeMs < 2000) score += 2;
+                else if (p.fetchTimeMs < 4000) score += 1;
+                if (!p.renderBlockingRisk) score += 2;
+                else if (p.deferScripts + p.asyncScripts > 0) score += 1;
+                if (p.totalImages === 0 || (s.images?.counts?.lazyLoaded || 0) / Math.max(p.totalImages, 1) > 0.3) score += 1;
+            }
+            scores.performance = { score: Math.min(score, max), max };
+        }
+
+        {
+            let score = 0; const max = SCORE_WEIGHTS.links.max;
+            const lk = s.links;
+            if (lk && !lk.skipped) {
+                if (lk.counts.internal > 0) score += 2;
+                if (lk.counts.external > 0) score += 1;
+                if (lk.counts.emptyText === 0) score += 2;
+                else if (lk.counts.emptyText < 5) score += 1;
+            } else {
+                score = max;
+            }
+            scores.links = { score: Math.min(score, max), max };
+        }
+
+        {
+            let score = 0; const max = SCORE_WEIGHTS.security.max;
+            const tech = s.technical;
+            if (tech) {
+                if (tech.https) score += 2;
+                if (tech.hsts) score += 1;
+                if (tech.xFrameOptions || tech.csp) score += 1;
+                if (tech.xContentTypeOptions) score += 1;
+            }
+            scores.security = { score: Math.min(score, max), max };
+        }
+
+        {
+            let score = 0; const max = SCORE_WEIGHTS.content.max;
+            const c = s.content;
+            if (c) {
+                if (c.wordCount >= 300) score += 2;
+                else if (c.wordCount >= 100) score += 1;
+                if (c.paragraphCount >= 3) score += 1;
+                if (c.htmlLang) score += 1;
+            }
+            scores.content = { score: Math.min(score, max), max };
+        }
+
+        const totalScore = Object.values(scores).reduce((sum, v) => sum + v.score, 0);
+        const totalMax = Object.values(scores).reduce((sum, v) => sum + v.max, 0);
+        const overallPercent = Math.round((totalScore / totalMax) * 100);
+
+        this.metadata.scores = scores;
+        this.metadata.overallScore = overallPercent;
+        this.metadata.grade = this.computeGrade(overallPercent);
+        this.metadata.summary = this.computeSummary(overallPercent, scores);
+    }
+
+    computeGrade(score) {
+        if (score >= 90) return 'A+';
+        if (score >= 85) return 'A';
+        if (score >= 80) return 'A-';
+        if (score >= 75) return 'B+';
+        if (score >= 70) return 'B';
+        if (score >= 65) return 'B-';
+        if (score >= 60) return 'C+';
+        if (score >= 55) return 'C';
+        if (score >= 50) return 'C-';
+        if (score >= 40) return 'D';
+        return 'F';
+    }
+
+    computeSummary(score, scores) {
+        const weakest = Object.entries(scores)
+            .map(([k, v]) => ({ key: k, pct: v.score / v.max }))
+            .sort((a, b) => a.pct - b.pct)
+            .slice(0, 2)
+            .map(v => SCORE_WEIGHTS[v.key]?.label || v.key);
+
+        if (score >= 85) return `Excellent SEO foundation detected. Minor refinements in ${weakest.join(' and ')} would push this to peak performance.`;
+        if (score >= 70) return `Solid SEO signals present. Primary opportunities in ${weakest.join(' and ')} \u2014 addressing these could meaningfully boost rankings.`;
+        if (score >= 55) return `Moderate SEO health. Significant gaps in ${weakest.join(' and ')} are likely suppressing organic visibility.`;
+        if (score >= 40) return `Below-average SEO configuration. Core issues in ${weakest.join(' and ')} suggest this page is leaving substantial traffic on the table.`;
+        return `Critical SEO deficiencies detected. Foundational work needed across multiple categories before this page can effectively compete in search.`;
+    }
+
+    generateRecommendations() {
+        const s = this.metadata.seo;
+        const recs = [];
+
+        const add = (priority, category, issue, recommendation, impact, effort) => {
+            recs.push({ priority, category, issue, recommendation, impact, effort });
+        };
+
+        const title = s.basic?.title;
+        if (!title) {
+            add('CRITICAL', 'Title Tag', 'Page is missing a <title> tag.', 'Add a descriptive title tag (50\u201360 characters) containing your primary keyword near the beginning. Search engines use the title as the primary ranking signal and click-through label in SERPs.', 'Very High', 'Low');
+        } else if (title.length < 30) {
+            add('HIGH', 'Title Tag', `Title is too short (${title.length} chars): "${title}"`, 'Expand your title to 50\u201360 characters. Short titles fail to capture keyword opportunities and may appear truncated or thin to search engines.', 'High', 'Low');
+        } else if (title.length > 60) {
+            add('MEDIUM', 'Title Tag', `Title may be truncated in SERPs (${title.length} chars).`, 'Trim your title to under 60 characters. Google typically truncates display titles beyond this threshold, reducing click-through rates.', 'Medium', 'Low');
+        }
+
+        const desc = s.basic?.metaDescription;
+        if (!desc) {
+            add('HIGH', 'Meta Description', 'No meta description found.', 'Write a compelling meta description of 120\u2013160 characters that summarizes the page and includes a call-to-action. While not a direct ranking factor, descriptions strongly influence click-through rates.', 'High', 'Low');
+        } else if (desc.length < 120) {
+            add('MEDIUM', 'Meta Description', `Description is too short (${desc.length} chars).`, 'Expand your meta description to at least 120 characters. Longer descriptions provide more context to searchers and reduce the chance Google auto-generates a less compelling snippet.', 'Medium', 'Low');
+        } else if (desc.length > 160) {
+            add('LOW', 'Meta Description', `Description may be truncated (${desc.length} chars).`, 'Consider trimming your description to 160 characters to ensure the full text displays in desktop SERPs.', 'Low', 'Low');
+        }
+
+        const h1count = s.headings?.counts?.h1 || 0;
+        if (h1count === 0) {
+            add('CRITICAL', 'Heading Structure', 'No H1 tag found on the page.', 'Add exactly one H1 tag that clearly describes the page topic and includes your primary keyword. The H1 is the strongest on-page heading signal search engines use to understand page context.', 'Very High', 'Low');
+        } else if (h1count > 1) {
+            add('MEDIUM', 'Heading Structure', `Multiple H1 tags detected (${h1count} found).`, 'Reduce to a single H1 per page. Multiple H1s dilute topical focus and can confuse search engine interpretation of the primary page topic.', 'Medium', 'Low');
+        }
+
+        if (s.headings?.hierarchyIssues?.length > 0) {
+            add('MEDIUM', 'Heading Structure', `${s.headings.hierarchyIssues.length} heading hierarchy jump(s) detected.`, `Fix heading order to follow a logical H1 \u2192 H2 \u2192 H3 sequence without skipping levels. Issues: ${s.headings.hierarchyIssues.slice(0, 2).join('; ')}`, 'Medium', 'Medium');
+        }
+
+        if (!s.basic?.canonical) {
+            add('HIGH', 'Canonical URL', 'No canonical tag found.', 'Add a <link rel="canonical"> tag pointing to the preferred version of this URL. This prevents duplicate content penalties from URL parameter variants, trailing slashes, and protocol differences.', 'High', 'Low');
+        } else if (s.basic.canonicalMatchesFinal === false) {
+            add('HIGH', 'Canonical URL', 'Canonical URL does not match the final URL after redirects.', 'Update your canonical tag to point to the resolved final URL. A mismatched canonical can signal conflicting authority signals to search engines.', 'High', 'Low');
+        }
+
+        const og = s.openGraph || {};
+        const missingOG = ['title', 'description', 'image', 'type'].filter(k => !og[k]);
+        if (missingOG.length === 4) {
+            add('HIGH', 'Open Graph', 'No Open Graph tags found.', 'Implement Open Graph meta tags (og:title, og:description, og:image, og:type). These directly control how your content appears when shared on Facebook, LinkedIn, Slack, and other platforms \u2014 significantly affecting social click-through rates.', 'High', 'Medium');
+        } else if (missingOG.length > 0) {
+            add('MEDIUM', 'Open Graph', `Missing OG properties: ${missingOG.join(', ')}.`, `Add the missing og:${missingOG[0]} tag${missingOG.length > 1 ? ' (and others)' : ''}. Incomplete Open Graph data often results in poor social sharing previews.`, 'Medium', 'Low');
+        }
+
+        const tw = s.twitter || {};
+        if (!tw.card) {
+            add('MEDIUM', 'Twitter Card', 'No Twitter Card meta tags found.', 'Add twitter:card, twitter:title, twitter:description, and twitter:image tags. These control your content\'s appearance in Twitter/X shares and can significantly increase engagement from social traffic.', 'Medium', 'Low');
+        }
+
+        if (!s.schema?.jsonLd?.length && !Object.keys(s.schema?.microdata || {}).length) {
+            add('HIGH', 'Structured Data', 'No structured data (schema.org) found.', 'Implement JSON-LD structured data appropriate to your content type (WebPage, Article, Product, Organization, LocalBusiness, BreadcrumbList). Structured data enables rich results in SERPs and is increasingly used by AI search summaries.', 'Very High', 'Medium');
+        }
+
+        const img = s.images;
+        if (img && !img.skipped) {
+            if (img.counts.withoutAlt > 0) {
+                const pct = 100 - img.counts.altCoverage;
+                const priority = pct > 50 ? 'HIGH' : 'MEDIUM';
+                add(priority, 'Image Optimization', `${img.counts.withoutAlt} image(s) missing alt text (${pct}% uncovered).`, 'Add descriptive alt attributes to all meaningful images. Alt text is critical for accessibility, image search visibility, and provides contextual signals to search engines about image content.', 'High', 'Medium');
+            }
+            if (img.counts.lazyLoaded === 0 && img.counts.total > 2) {
+                add('MEDIUM', 'Image Optimization', 'No lazy-loading detected on images.', 'Add loading="lazy" to all below-the-fold images. Lazy loading reduces initial page weight and improves Core Web Vitals scores (LCP, FID), which are Google ranking factors.', 'Medium', 'Low');
+            }
+        }
+
+        if (!s.technical?.hasViewport) {
+            add('CRITICAL', 'Mobile Readiness', 'No viewport meta tag found.', 'Add <meta name="viewport" content="width=device-width, initial-scale=1"> to the <head>. Without this, Google\'s mobile-first indexing will penalize this page and mobile users will see a broken zoomed-out layout.', 'Very High', 'Low');
+        }
+
+        const perf = s.performance;
+        if (perf?.renderBlockingRisk) {
+            add('MEDIUM', 'Performance', `${perf.headScripts} render-blocking script(s) in <head>.`, 'Add defer or async attributes to non-critical scripts in <head>. Render-blocking scripts delay First Contentful Paint (FCP) and increase Time to Interactive (TTI), both measured by Core Web Vitals.', 'Medium', 'Low');
+        }
+        if (perf?.htmlSizeKB > 200) {
+            add('MEDIUM', 'Performance', `HTML size is large (${perf.htmlSizeKB}KB).`, 'Investigate opportunities to reduce HTML payload: remove unused markup, minimize inline scripts/styles, and consider server-side rendering optimizations. Large HTML documents increase parse time and bandwidth consumption.', 'Medium', 'High');
+        }
+        if (perf?.fetchTimeMs > 3000) {
+            add('HIGH', 'Performance', `Server response time is slow (${perf.fetchTimeMs}ms TTFB).`, 'Investigate server-side performance: enable caching, use a CDN, optimize database queries, or upgrade hosting. Google\'s recommended TTFB threshold is under 800ms for a good user experience.', 'High', 'High');
+        }
+
+        if (!s.technical?.https) {
+            add('CRITICAL', 'Security', 'Page is served over HTTP (not HTTPS).', 'Migrate to HTTPS immediately. Google has used HTTPS as a ranking signal since 2014, and browsers now display "Not Secure" warnings on HTTP pages, drastically reducing user trust and conversions.', 'Very High', 'High');
+        }
+        if (!s.technical?.hsts && s.technical?.https) {
+            add('LOW', 'Security', 'HSTS header not detected.', 'Enable HTTP Strict Transport Security (HSTS) on your server. HSTS prevents protocol downgrade attacks and ensures all connections use HTTPS, which signals security maturity to search engines and users.', 'Low', 'Medium');
+        }
+
+        if (!s.technical?.hasFavicon) {
+            add('LOW', 'Technical', 'No favicon detected.', 'Add a favicon (<link rel="icon">). While not a ranking factor, favicons appear in browser tabs and bookmarks, reinforcing brand recognition and user trust during search result navigation.', 'Low', 'Low');
+        }
+
+        if (s.links?.counts?.emptyText > 3) {
+            add('MEDIUM', 'Link Structure', `${s.links.counts.emptyText} links have no anchor text.`, 'Add descriptive anchor text to all links. Search engines use anchor text to understand the context and relevance of linked pages \u2014 empty anchors waste this signal entirely.', 'Medium', 'Medium');
+        }
+
+        if (s.content?.wordCount < 300 && s.content?.wordCount > 0) {
+            add('MEDIUM', 'Content Quality', `Page has low word count (${s.content.wordCount} words).`, 'Expand page content to at least 300 words. Thin content is a known quality signal issue in Google\'s algorithms. More comprehensive content tends to rank for broader keyword sets.', 'High', 'High');
+        }
+
+        const priorityOrder = { CRITICAL: 0, HIGH: 1, MEDIUM: 2, LOW: 3, GOOD: 4 };
+        recs.sort((a, b) => (priorityOrder[a.priority] ?? 9) - (priorityOrder[b.priority] ?? 9));
+
+        this.metadata.recommendations = recs;
+    }
+
+    async analyze() {
+        const steps = [
+            ['Basic Metadata', () => this.extractBasicMetadata()],
+            ['Open Graph', () => this.extractOpenGraphMetadata()],
+            ['Twitter Card', () => this.extractTwitterMetadata()],
+            ['Schema Markup', () => this.extractSchemaMetadata()],
+            ['Technical SEO', () => this.extractTechnicalMetadata()],
+            ['Link Analysis', () => this.extractLinks()],
+            ['Heading Structure', () => this.extractHeadings()],
+            ['Image Analysis', () => this.extractImages()],
+            ['Content Analysis', () => this.extractContentMetadata()],
+            ['Performance Metrics', () => this.extractPerformanceMetrics()],
+            ['Scoring Engine', () => this.computeScores()],
+            ['Recommendations', () => this.generateRecommendations()],
+        ];
+
+        for (const [name, fn] of steps) {
+            try {
+                fn();
+                log.ok(name);
+            } catch (e) {
+                log.warn(`${name}: ${e.message}`);
+            }
+        }
+    }
+}
+
+module.exports = SEOScraper;