@aexol/spectral 0.9.42 → 0.9.44

package/dist/extensions/seo/utils/parser.js

@@ -0,0 +1,265 @@
+/**
+ * HTML parser using cheerio.
+ *
+ * Ported from the Python `parse_html.py` in claude-seo. Extracts all
+ * on-page SEO elements: title, meta, canonical, headings, links, images,
+ * JSON-LD schema, Open Graph, Twitter Card, hreflang, and word count.
+ */
+import * as cheerio from "cheerio";
+const SUSPICIOUS_HEADING_TEXTS = new Set([
+    "",
+    "click here",
+    "read more",
+    "learn more",
+    "more",
+    "here",
+]);
+function detectLazyMethod($, el) {
+    const loading = $(el).attr("loading");
+    if (loading === "lazy")
+        return "native";
+    const dataSrc = $(el).attr("data-src");
+    if (dataSrc)
+        return "dataSrc";
+    const dataLazy = $(el).attr("data-lazy") ?? $(el).attr("data-lazy-src");
+    if (dataLazy)
+        return "dataLazy";
+    return "none";
+}
+function isInternal(href, baseUrl) {
+    try {
+        const url = new URL(href, baseUrl);
+        const base = new URL(baseUrl);
+        return url.hostname === base.hostname;
+    }
+    catch {
+        return false;
+    }
+}
+function resolveUrl(href, baseUrl) {
+    try {
+        return new URL(href, baseUrl).href;
+    }
+    catch {
+        return href;
+    }
+}
+/** Parse JSON-LD blocks from the HTML. */
+function parseJsonLd($, baseUrl) {
+    const blocks = [];
+    $('script[type="application/ld+json"]').each((_, el) => {
+        const raw = $(el).text().trim();
+        if (!raw)
+            return;
+        try {
+            const parsed = JSON.parse(raw);
+            const items = Array.isArray(parsed) ? parsed : [parsed];
+            for (const item of items) {
+                const type = item?.["@type"] ?? "Unknown";
+                const issues = [];
+                if (!item?.["@context"])
+                    issues.push("Missing @context");
+                if (typeof type === "string" && type.includes("/")) {
+                    // Could be a full URL type — that's fine.
+                }
+                // Check for relative URLs.
+                const jsonStr = JSON.stringify(item);
+                if (jsonStr.includes('"url": "/') || jsonStr.includes('"url":"/')) {
+                    issues.push("Contains relative URL (should be absolute)");
+                }
+                blocks.push({
+                    type: typeof type === "string" ? type : "Composite",
+                    format: "json-ld",
+                    raw,
+                    parsed: item,
+                    issues,
+                });
+            }
+        }
+        catch {
+            blocks.push({
+                type: "InvalidJSON",
+                format: "json-ld",
+                raw,
+                parsed: null,
+                issues: ["Invalid JSON syntax"],
+            });
+        }
+    });
+    return blocks;
+}
+/** Detect microdata markup. */
+function detectMicrodata($) {
+    const blocks = [];
+    $("[itemscope]").each((_, el) => {
+        const type = $(el).attr("itemtype") ?? "Unknown";
+        blocks.push({
+            type: type.replace("https://schema.org/", ""),
+            format: "microdata",
+            raw: $.html(el),
+            issues: [],
+        });
+    });
+    return blocks;
+}
+/** Parse a full PageData object from HTML. */
+export function parsePage(html, url, baseUrl) {
+    const $ = cheerio.load(html);
+    const effectiveBaseUrl = baseUrl ?? url ?? "https://example.com";
+    // Title
+    const title = $("title").first().text().trim() || null;
+    // Meta description
+    const metaDesc = $('meta[name="description"]').attr("content")?.trim() || null;
+    // Meta robots
+    const metaRobots = $('meta[name="robots"]').attr("content")?.trim() || null;
+    // Canonical
+    const canonical = $('link[rel="canonical"]').attr("href") || null;
+    // Open Graph
+    const ogTags = [];
+    $('meta[property^="og:"]').each((_, el) => {
+        ogTags.push({
+            property: $(el).attr("property") ?? "",
+            content: $(el).attr("content") ?? "",
+        });
+    });
+    // Twitter Card
+    const twitterTags = [];
+    $('meta[name^="twitter:"]').each((_, el) => {
+        twitterTags.push({
+            name: $(el).attr("name") ?? "",
+            content: $(el).attr("content") ?? "",
+        });
+    });
+    // Headings
+    const headings = [];
+    let h1Count = 0, h2Count = 0, h3Count = 0;
+    $("h1, h2, h3, h4, h5, h6").each((_, el) => {
+        const tagName = el.tagName.toLowerCase();
+        const level = Number.parseInt(tagName.replace("h", ""), 10);
+        const text = $(el).text().trim();
+        if (level === 1)
+            h1Count++;
+        if (level === 2)
+            h2Count++;
+        if (level === 3)
+            h3Count++;
+        headings.push({
+            level,
+            text,
+            suspicious: SUSPICIOUS_HEADING_TEXTS.has(text.toLowerCase()),
+        });
+    });
+    // Images
+    const images = [];
+    $("img").each((_, el) => {
+        const src = $(el).attr("src") ?? $(el).attr("data-src") ?? "";
+        if (!src || src.startsWith("data:"))
+            return;
+        images.push({
+            src: resolveUrl(src, effectiveBaseUrl),
+            alt: $(el).attr("alt") ?? "",
+            width: $(el).attr("width"),
+            height: $(el).attr("height"),
+            loading: $(el).attr("loading"),
+            lazyMethod: detectLazyMethod($, el),
+            fetchpriority: $(el).attr("fetchpriority"),
+        });
+    });
+    // Links
+    const links = [];
+    let internalLinkCount = 0;
+    let externalLinkCount = 0;
+    $("a[href]").each((_, el) => {
+        const href = $(el).attr("href") ?? "";
+        if (!href || href.startsWith("#") || href.startsWith("javascript:"))
+            return;
+        const resolved = resolveUrl(href, effectiveBaseUrl);
+        const rel = $(el).attr("rel") ?? "";
+        const nofollow = rel.toLowerCase().includes("nofollow");
+        const internal = isInternal(resolved, effectiveBaseUrl);
+        if (internal)
+            internalLinkCount++;
+        else
+            externalLinkCount++;
+        links.push({
+            href: resolved,
+            text: $(el).text().trim(),
+            rel,
+            type: internal ? "internal" : "external",
+            nofollow,
+        });
+    });
+    // Schema
+    const schemaBlocks = [...parseJsonLd($, effectiveBaseUrl), ...detectMicrodata($)];
+    // Hreflang
+    const hreflang = [];
+    $('link[rel="alternate"][hreflang]').each((_, el) => {
+        hreflang.push({
+            href: resolveUrl($(el).attr("href") ?? "", effectiveBaseUrl),
+            hreflang: $(el).attr("hreflang") ?? "",
+        });
+    });
+    // Word count (from body text)
+    const bodyText = $("body").text().trim();
+    const wordCount = bodyText ? bodyText.split(/\s+/).filter(Boolean).length : 0;
+    // Language
+    const language = $("html").attr("lang") ?? null;
+    // Viewport
+    const viewport = $('meta[name="viewport"]').attr("content") ?? null;
+    // Charset
+    const charset = $("meta[charset]").attr("charset") ?? null;
+    return {
+        url: url ?? effectiveBaseUrl,
+        baseUrl: effectiveBaseUrl,
+        title,
+        titleLength: title?.length ?? 0,
+        metaDescription: metaDesc,
+        metaDescriptionLength: metaDesc?.length ?? 0,
+        metaRobots,
+        canonical,
+        ogTags,
+        twitterTags,
+        headings,
+        h1Count,
+        h2Count,
+        h3Count,
+        images,
+        links,
+        internalLinkCount,
+        externalLinkCount,
+        schemaBlocks,
+        hreflang,
+        wordCount,
+        language,
+        viewport,
+        charset,
+    };
+}
+/** Extract the visible text content from HTML body. */
+export function extractText(html) {
+    const $ = cheerio.load(html);
+    // Remove script and style elements.
+    $("script, style, noscript").remove();
+    return $("body").text().replace(/\s+/g, " ").trim();
+}
+/** Extract raw JSON-LD blocks as parsed objects. */
+export function extractJsonLd(html) {
+    const $ = cheerio.load(html);
+    const results = [];
+    $('script[type="application/ld+json"]').each((_, el) => {
+        const raw = $(el).text().trim();
+        if (!raw)
+            return;
+        try {
+            const parsed = JSON.parse(raw);
+            if (Array.isArray(parsed))
+                results.push(...parsed);
+            else
+                results.push(parsed);
+        }
+        catch {
+            // skip invalid JSON
+        }
+    });
+    return results;
+}

package/dist/extensions/seo/utils/renderer.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Lightweight Playwright renderer for SPA-aware page fetching.
+ *
+ * Ported from the Python `render_page.py` in claude-seo. Uses a standalone
+ * Playwright browser context (not shared with the browser extension) to keep
+ * the SEO extension self-contained.
+ */
+import type { SeoConfig } from "../config.js";
+import type { FetchResult } from "../types.js";
+export interface RenderOptions {
+    viewport?: "desktop" | "mobile";
+    timeout?: number;
+    blockResources?: string[];
+    /** Whether to capture the accessibility tree (for agent UX audits). */
+    captureA11y?: boolean;
+    /** Whether to capture console errors. */
+    captureConsole?: boolean;
+    /** User agent override. */
+    userAgent?: string;
+}
+export interface RenderResult extends FetchResult {
+    isSpa: boolean;
+    rawContent: string;
+    consoleErrors: string[];
+    renderMs: number;
+    accessibilityTree: unknown | null;
+}
+/** Close the shared browser instance (called on session shutdown). */
+export declare function closeRenderer(): Promise<void>;
+/** Check whether Playwright Chromium is available. */
+export declare function isRendererAvailable(): Promise<boolean>;
+/**
+ * Render a page with Playwright, returning the full DOM after JS execution.
+ *
+ * This is slower than a plain HTTP fetch but necessary for single-page
+ * applications where critical content is injected via JavaScript.
+ */
+export declare function renderPage(url: string, config: SeoConfig, opts?: RenderOptions): Promise<RenderResult>;
+//# sourceMappingURL=renderer.d.ts.map

package/dist/extensions/seo/utils/renderer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"renderer.d.ts","sourceRoot":"","sources":["../../../../src/extensions/seo/utils/renderer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,KAAK,EAAE,WAAW,EAAe,MAAM,aAAa,CAAC;AAE5D,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,SAAS,GAAG,QAAQ,CAAC;IAChC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,uEAAuE;IACvE,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,yCAAyC;IACzC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,2BAA2B;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAa,SAAQ,WAAW;IAC/C,KAAK,EAAE,OAAO,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,OAAO,GAAG,IAAI,CAAC;CACnC;AAeD,sEAAsE;AACtE,wBAAsB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CAUnD;AAED,sDAAsD;AACtD,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,OAAO,CAAC,CAO5D;AAED;;;;;GAKG;AACH,wBAAsB,UAAU,CAC9B,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,SAAS,EACjB,IAAI,GAAE,aAAkB,GACvB,OAAO,CAAC,YAAY,CAAC,CA8JvB"}

package/dist/extensions/seo/utils/renderer.js

@@ -0,0 +1,190 @@
+const DESKTOP_VIEWPORT = { width: 1280, height: 800 };
+const MOBILE_VIEWPORT = { width: 390, height: 844 };
+let browserPromise = null;
+async function getBrowser() {
+    if (!browserPromise) {
+        const { chromium } = await import("playwright");
+        browserPromise = chromium.launch({ headless: true });
+    }
+    return browserPromise;
+}
+/** Close the shared browser instance (called on session shutdown). */
+export async function closeRenderer() {
+    if (browserPromise) {
+        try {
+            const browser = await browserPromise;
+            await browser.close();
+        }
+        catch {
+            // ignore
+        }
+        browserPromise = null;
+    }
+}
+/** Check whether Playwright Chromium is available. */
+export async function isRendererAvailable() {
+    try {
+        await getBrowser();
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Render a page with Playwright, returning the full DOM after JS execution.
+ *
+ * This is slower than a plain HTTP fetch but necessary for single-page
+ * applications where critical content is injected via JavaScript.
+ */
+export async function renderPage(url, config, opts = {}) {
+    const viewport = opts.viewport ?? config.defaultViewport;
+    const timeout = opts.timeout ?? config.timeout;
+    const blockResources = opts.blockResources ?? config.blockRenderResources;
+    const captureA11y = opts.captureA11y ?? false;
+    const captureConsole = opts.captureConsole ?? true;
+    const startTime = Date.now();
+    const browser = await getBrowser();
+    const context = await browser.newContext({
+        viewport: viewport === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT,
+        userAgent: opts.userAgent ?? config.userAgent,
+    });
+    // Block resource types to speed up rendering.
+    if (blockResources.length > 0) {
+        await context.route("**/*", (route) => {
+            const resourceType = route.request().resourceType();
+            if (blockResources.includes(resourceType)) {
+                route.abort().catch(() => { });
+            }
+            else {
+                route.continue().catch(() => { });
+            }
+        });
+    }
+    const page = await context.newPage();
+    const consoleErrors = [];
+    const redirectChain = [];
+    if (captureConsole) {
+        page.on("console", (msg) => {
+            if (msg.type() === "error") {
+                consoleErrors.push(msg.text());
+            }
+        });
+        page.on("pageerror", (err) => {
+            consoleErrors.push(err.message);
+        });
+    }
+    let response = null;
+    let finalUrl = url;
+    try {
+        response = await page.goto(url, {
+            waitUntil: "networkidle",
+            timeout,
+        });
+        if (response) {
+            // Track redirects by walking the redirectedFrom() chain.
+            let current = response.request().redirectedFrom();
+            while (current) {
+                const redirectResponse = await current.response().catch(() => null);
+                redirectChain.unshift({
+                    url: current.url(),
+                    status: redirectResponse?.status() ?? 0,
+                });
+                current = current.redirectedFrom();
+            }
+            finalUrl = page.url();
+        }
+    }
+    catch (err) {
+        await context.close();
+        return {
+            url,
+            finalUrl,
+            statusCode: 0,
+            statusText: "Error",
+            headers: {},
+            content: "",
+            redirectChain,
+            contentLengthBytes: 0,
+            fetchedAt: new Date().toISOString(),
+            error: err instanceof Error ? err.message : String(err),
+            isSpa: false,
+            rawContent: "",
+            consoleErrors,
+            renderMs: Date.now() - startTime,
+            accessibilityTree: null,
+        };
+    }
+    const rawContent = (await response?.text()) ?? "";
+    const content = await page.content();
+    // SPA heuristic: if the rendered DOM is significantly larger than the raw
+    // initial HTML, content was likely injected by JavaScript.
+    const isSpa = rawContent.length > 0 && content.length > rawContent.length * 1.5;
+    let accessibilityTree = null;
+    if (captureA11y) {
+        try {
+            // Extract accessibility-relevant data from the DOM since Playwright 1.60
+            // removed the page.accessibility.snapshot() API.
+            accessibilityTree = await page.evaluate(() => {
+                const INTERACTIVE_ROLES = new Set([
+                    "button", "link", "textbox", "checkbox", "radio", "combobox",
+                    "menuitem", "tab", "slider", "spinbutton", "switch", "searchbox",
+                    "treeitem", "option",
+                ]);
+                const all = document.querySelectorAll("*");
+                let totalNodes = 0;
+                let interactiveNodes = 0;
+                let unnamedInteractive = 0;
+                let genericNodes = 0;
+                const result = [];
+                for (const el of all) {
+                    totalNodes++;
+                    const role = el.getAttribute("role");
+                    const name = (el.getAttribute("aria-label") ?? el.textContent?.trim().slice(0, 50) ?? "");
+                    const tag = el.tagName.toLowerCase();
+                    const effectiveRole = role ?? (tag === "button" ? "button" : tag === "a" ? "link" : tag === "input" || tag === "textarea" || tag === "select" ? "textbox" : null);
+                    if (effectiveRole && INTERACTIVE_ROLES.has(effectiveRole)) {
+                        interactiveNodes++;
+                        if (!name || name.trim() === "")
+                            unnamedInteractive++;
+                    }
+                    if (effectiveRole === "generic" || (!role && !["button", "a", "input", "textarea", "select", "nav", "main", "header", "footer", "article", "section", "aside", "form"].includes(tag))) {
+                        genericNodes++;
+                    }
+                    if (interactiveNodes < 200) {
+                        result.push({ role: effectiveRole, name, tag });
+                    }
+                }
+                return { totalNodes, interactiveNodes, unnamedInteractive, genericNodes, sample: result.slice(0, 100) };
+            });
+        }
+        catch {
+            accessibilityTree = null;
+        }
+    }
+    const renderMs = Date.now() - startTime;
+    await context.close();
+    const headers = {};
+    if (response) {
+        const respHeaders = response.headers();
+        for (const [k, v] of Object.entries(respHeaders)) {
+            headers[k] = v;
+        }
+    }
+    return {
+        url,
+        finalUrl,
+        statusCode: response?.status() ?? 0,
+        statusText: response?.statusText() ?? "",
+        headers,
+        content,
+        redirectChain,
+        contentLengthBytes: Buffer.byteLength(content, "utf-8"),
+        fetchedAt: new Date().toISOString(),
+        isSpa,
+        rawContent,
+        consoleErrors,
+        renderMs,
+        accessibilityTree,
+    };
+}

package/dist/extensions/seo/utils/url-safety.d.ts

@@ -0,0 +1,31 @@
+export declare class UrlSafetyError extends Error {
+    constructor(message: string);
+}
+export interface ValidatedUrl {
+    href: string;
+    protocol: string;
+    hostname: string;
+    port: string;
+    pathname: string;
+    search: string;
+    /** Resolved IP addresses (only populated when SSRF protection is on). */
+    ips: string[];
+}
+/**
+ * Validate a URL string with optional SSRF protection.
+ *
+ * Throws UrlSafetyError if the URL is invalid, uses a non-http(s) protocol,
+ * points to a blocked hostname, or (when ssrfProtection=true) resolves to a
+ * private IP address.
+ */
+export declare function validateUrl(urlStr: string, opts?: {
+    ssrfProtection?: boolean;
+    resolveDns?: boolean;
+}): Promise<ValidatedUrl>;
+/** Quick synchronous URL validation without DNS resolution. */
+export declare function validateUrlBasic(urlStr: string): ValidatedUrl;
+/** Normalize a URL for baseline matching (strip UTM, sort params, lowercase host). */
+export declare function normalizeUrl(url: string): string;
+/** Hash a normalized URL for baseline deduplication. */
+export declare function urlHash(url: string): string;
+//# sourceMappingURL=url-safety.d.ts.map

package/dist/extensions/seo/utils/url-safety.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"url-safety.d.ts","sourceRoot":"","sources":["../../../../src/extensions/seo/utils/url-safety.ts"],"names":[],"mappings":"AAUA,qBAAa,cAAe,SAAQ,KAAK;gBAC3B,OAAO,EAAE,MAAM;CAI5B;AAiDD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,yEAAyE;IACzE,GAAG,EAAE,MAAM,EAAE,CAAC;CACf;AAED;;;;;;GAMG;AACH,wBAAsB,WAAW,CAC/B,MAAM,EAAE,MAAM,EACd,IAAI,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,UAAU,CAAC,EAAE,OAAO,CAAA;CAAO,GAC5D,OAAO,CAAC,YAAY,CAAC,CA8EvB;AAED,+DAA+D;AAC/D,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CA8B7D;AAED,sFAAsF;AACtF,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CA2BhD;AAED,wDAAwD;AACxD,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAE3C"}