@fulldotdev/scan 0.1.0

package/dist/engine/browser.js

@@ -0,0 +1,629 @@
+import { observeRendering, inspectRendering, inspectOverflow, } from "./browser-inspection.js";
+import { linkCandidates } from "./crawl-scope.js";
+import { compareRendered, extractPage, loadMoreText, resolveLink, } from "./html.js";
+import { srcsetUrls } from "./srcset.js";
+import puppeteer from "puppeteer";
+import lighthouse from "lighthouse";
+import axe from "axe-core";
+import { createHash } from "node:crypto";
+import { existsSync } from "node:fs";
+import { createEgressProxy } from "./proxy.js";
+import { resolvePublic, userAgent } from "./network.js";
+import { env } from "./env.js";
+import { errorMessage, } from "./types.js";
+// One proxy and Chromium per worker; Lighthouse resets storage per run, so
+// pages stay independent while the cold launch is paid once.
+export async function openBrowserSession() {
+    const proxy = await createEgressProxy();
+    try {
+        // CHROME_PATH, then a system Chrome, then the Chrome puppeteer installs.
+        const localPaths = [
+            "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+            "/usr/bin/google-chrome",
+            "/usr/bin/chromium",
+        ];
+        const executablePath = env("CHROME_PATH") || localPaths.find(existsSync) || undefined;
+        const args = [
+            "--no-sandbox",
+            "--disable-dev-shm-usage",
+            `--proxy-server=http://127.0.0.1:${proxy.port}`,
+            "--proxy-bypass-list=<-loopback>",
+            "--disable-quic",
+            "--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
+            "--disable-background-networking",
+        ];
+        const browser = await puppeteer.launch({
+            executablePath,
+            args,
+            headless: true,
+            protocolTimeout: 90000,
+        });
+        const chromeVersion = await browser.version();
+        return {
+            proxy,
+            lighthouse,
+            browser,
+            args,
+            chromeVersion,
+            get connected() {
+                return browser.connected;
+            },
+            close: async () => {
+                await browser.close().catch(() => { });
+                await proxy.close();
+            },
+        };
+    }
+    catch (error) {
+        await proxy.close();
+        throw error;
+    }
+}
+// Screenshots for the homepage and a rotating fifth of the pages: enough for
+// visual comparison without storing every page every day.
+export function screenshotSampled(url, rotation, home) {
+    if (url === home || new URL(url).pathname === "/")
+        return true;
+    return (createHash("sha256").update(url).digest()[0] % 5 === Math.abs(rotation) % 5);
+}
+const axeTags = [
+    "wcag2a",
+    "wcag2aa",
+    "wcag21a",
+    "wcag21aa",
+    "wcag22aa",
+    "best-practice",
+];
+// The per-page browser pass: one mobile navigation for own metrics and the
+// rendered DOM, then accessibility, overflow, keyboard and two safe
+// interactions (scroll, one load-more click). No forms are submitted.
+export async function collectBrowser(scan, url, http, session) {
+    await resolvePublic(new URL(url).hostname);
+    const robots = await http.getRobots(url);
+    if (!robots.allowed ||
+        robots.parser?.isAllowed(url, "FulldevScan") === false)
+        return {
+            observations: [
+                {
+                    kind: "browser",
+                    key: url,
+                    data: { status: "skipped", reason: "robots" },
+                },
+            ],
+        };
+    const { proxy, browser, chromeVersion } = session;
+    const blockedBefore = proxy.blocked.length;
+    const result = { observations: [], artifacts: [], candidates: [] };
+    const errors = [];
+    let page;
+    const budgetSeconds = scan.options.deepScan ? 180 : 120;
+    const watchdog = setTimeout(() => {
+        errors.push({
+            stage: "budget",
+            error: `Browser job exceeded ${budgetSeconds} seconds`,
+        });
+        // Closing the browser aborts the page; the worker reopens a session.
+        void browser.close();
+    }, budgetSeconds * 1000);
+    watchdog.unref();
+    try {
+        page = await browser.newPage();
+        await page.setUserAgent(userAgent);
+        await page.setCacheEnabled(false);
+        // Loaded as a phone: real numbers without throttling, comparable
+        // between days. Axe and keyboard checks run on the desktop size after.
+        await page.setViewport({
+            width: 412,
+            height: 915,
+            deviceScaleFactor: 1.75,
+            isMobile: true,
+            hasTouch: true,
+        });
+        // tsx preserves nested function names with this helper in serialized callbacks.
+        await page.evaluateOnNewDocument("globalThis.__name = (fn) => fn;");
+        if (scan.options.deepScan)
+            await observeRendering(page);
+        const renderingResponses = [];
+        const transfer = new Map();
+        const requestMeta = new Map();
+        // Static files the page loaded (also the ones that failed), so CSS
+        // backgrounds, fonts and script-inserted assets get checked. Documents,
+        // XHR, fetch and beacons are not files and are left out.
+        const staticTypes = /^(Image|Font|Stylesheet|Script|Media)$/;
+        const staticResources = new Set();
+        const cdp = await page.createCDPSession();
+        await cdp.send("Network.enable");
+        cdp.on("Network.requestWillBeSent", (e) => {
+            if (staticTypes.test(e.type) && /^https?:/.test(e.request?.url ?? ""))
+                staticResources.add(e.request.url);
+        });
+        cdp.on("Network.responseReceived", (e) => {
+            requestMeta.set(e.requestId, { url: e.response.url, type: e.type });
+            if (scan.options.deepScan)
+                renderingResponses.push({
+                    requestId: e.requestId,
+                    url: e.response.url,
+                    status: e.response.status,
+                    mimeType: e.response.mimeType,
+                    encodedBodySize: Number(e.response.headers["content-length"] ??
+                        e.response.headers["Content-Length"]) || null,
+                    contentEncoding: e.response.headers["content-encoding"] ?? null,
+                });
+        });
+        cdp.on("Network.loadingFinished", (e) => {
+            transfer.set(e.requestId, e.encodedDataLength ?? 0);
+            const response = renderingResponses.find((r) => r.requestId === e.requestId);
+            if (response)
+                response.transferBytes = e.encodedDataLength ?? null;
+        });
+        const consoleMessages = [], failures = [], network = [];
+        page.on("console", (message) => {
+            if (consoleMessages.length < 200)
+                consoleMessages.push({
+                    type: message.type(),
+                    text: message.text().slice(0, 2000),
+                    location: message.location(),
+                });
+        });
+        page.on("pageerror", (error) => {
+            if (consoleMessages.length < 200)
+                consoleMessages.push({
+                    type: "pageerror",
+                    text: String(error).slice(0, 2000),
+                });
+        });
+        page.on("requestfailed", (request) => {
+            if (failures.length < 500)
+                failures.push({
+                    url: request.url(),
+                    type: request.resourceType(),
+                    error: request.failure(),
+                });
+        });
+        page.on("response", (response) => {
+            if (network.length < 1000)
+                network.push({
+                    url: response.url(),
+                    status: response.status(),
+                    type: response.request().resourceType(),
+                    fromCache: response.fromCache(),
+                    timing: response.timing(),
+                });
+        });
+        await http.pace(url);
+        const started = Date.now();
+        const navigation = await page.goto(url, {
+            waitUntil: "domcontentloaded",
+            timeout: 30000,
+        });
+        const initialHtml = (await navigation?.text().catch(() => "")) ?? "";
+        const responseHeaders = navigation?.headers() ?? {};
+        await page
+            .waitForNetworkIdle({ idleTime: 500, timeout: 5000 })
+            .catch(() => { });
+        // Own metrics from the browser's performance APIs; no throttling, so
+        // they are lab numbers on a fast connection, not Lighthouse scores.
+        const timings = await page.evaluate(() => new Promise((resolve) => {
+            const nav = performance.getEntriesByType("navigation")[0];
+            let lcp = null;
+            let cls = 0;
+            try {
+                new PerformanceObserver((list) => {
+                    for (const entry of list.getEntries())
+                        lcp = entry.renderTime || entry.startTime;
+                }).observe({ type: "largest-contentful-paint", buffered: true });
+            }
+            catch { }
+            try {
+                new PerformanceObserver((list) => {
+                    for (const entry of list.getEntries())
+                        if (!entry.hadRecentInput)
+                            cls += entry.value;
+                }).observe({ type: "layout-shift", buffered: true });
+            }
+            catch { }
+            setTimeout(() => resolve({
+                ttfb: nav
+                    ? Math.round(nav.responseStart - nav.startTime)
+                    : null,
+                fcp: Math.round(performance
+                    .getEntriesByType("paint")
+                    .find((p) => p.name === "first-contentful-paint")
+                    ?.startTime ?? -1),
+                lcp: lcp === null ? null : Math.round(lcp),
+                cls: Math.round(cls * 1000) / 1000,
+                load: nav ? Math.round(nav.loadEventEnd) : null,
+            }), 1000);
+        }));
+        if (scan.options.deepScan)
+            result.observations.push({
+                kind: "rendering",
+                key: `${url}|mobile`,
+                data: {
+                    url,
+                    device: "mobile",
+                    ...(await inspectRendering(page, cdp, initialHtml, renderingResponses)),
+                },
+            });
+        // Rendered DOM against the HTML the browser received for this same
+        // navigation: what scripts added, removed or changed.
+        const finalUrl = page.url();
+        const httpShape = (body, headers) => ({
+            url,
+            finalUrl,
+            status: navigation?.status() ?? null,
+            headers,
+            redirects: [],
+            body,
+            bytes: body.length,
+            durationMs: 0,
+            fetchedAt: new Date(started).toISOString(),
+            outcome: "ok",
+            attempts: 1,
+        });
+        const original = extractPage(httpShape(initialHtml, responseHeaders));
+        const renderedHtml = await page.content();
+        const rendered = extractPage(httpShape(renderedHtml, responseHeaders), "rendered-dom");
+        const comparison = compareRendered(original, rendered);
+        const compactLinks = (links) => links.map((l) => ({
+            href: l.href,
+            name: l.name,
+            location: l.location,
+            rel: l.rel,
+            pagination: l.pagination,
+            hreflang: l.hreflang,
+        }));
+        const origin = new URL(url).origin;
+        const byType = {};
+        let thirdPartyBytes = 0, totalBytes = 0;
+        for (const [id, bytes] of transfer) {
+            const meta = requestMeta.get(id);
+            const type = (meta?.type ?? "Other").toLowerCase();
+            byType[type] = (byType[type] ?? 0) + bytes;
+            totalBytes += bytes;
+            if (meta && !meta.url.startsWith(origin))
+                thirdPartyBytes += bytes;
+        }
+        const cookies = (await page.cookies()).map((c) => ({
+            name: c.name,
+            domain: c.domain,
+            thirdParty: !new URL(url).hostname.endsWith(c.domain.replace(/^\./, "")),
+        }));
+        result.observations.push({
+            kind: "browser-metrics",
+            key: url,
+            data: {
+                device: "mobile",
+                viewport: { width: 412, height: 915, deviceScaleFactor: 1.75 },
+                measurementVersion: 2,
+                throttling: "none",
+                ttfb: timings.ttfb,
+                fcp: timings.fcp >= 0 ? timings.fcp : null,
+                lcp: timings.lcp,
+                cls: timings.cls,
+                load: timings.load,
+                wallMs: Date.now() - started,
+                requests: transfer.size,
+                bytes: totalBytes,
+                byType,
+                thirdPartyBytes,
+                cookies: {
+                    total: cookies.length,
+                    thirdParty: cookies.filter((c) => c.thirdParty).length,
+                    names: cookies.slice(0, 50),
+                },
+                note: "Own headless Chrome at 412px, DPR 1.75 on a fast connection. Compare matching viewports/settings; older DPR 1 observations are not directly comparable.",
+            },
+        });
+        if (screenshotSampled(url, scan.rotation, scan.url))
+            result.artifacts.push({
+                kind: "screenshot",
+                key: url,
+                body: (await page.screenshot({ type: "jpeg", quality: 60 })),
+                contentType: "image/jpeg",
+            });
+        // Puppeteer's setViewport reloads when mobile/touch flags change. CDP lets
+        // us resize without a hidden navigation; deep mode navigates explicitly below.
+        const resize = (width, height) => cdp.send("Emulation.setDeviceMetricsOverride", {
+            width,
+            height,
+            deviceScaleFactor: scan.options.deepScan ? 1 : 1.75,
+            mobile: !scan.options.deepScan,
+        });
+        await resize(1350, 940);
+        if (scan.options.deepScan) {
+            await cdp.send("Emulation.setTouchEmulationEnabled", { enabled: false });
+            renderingResponses.length = 0;
+            const desktopNavigation = await page.goto(url, {
+                waitUntil: "domcontentloaded",
+                timeout: 30000,
+            });
+            await page
+                .waitForNetworkIdle({ idleTime: 500, timeout: 5000 })
+                .catch(() => { });
+            result.observations.push({
+                kind: "rendering",
+                key: `${url}|desktop`,
+                data: {
+                    url,
+                    device: "desktop",
+                    ...(await inspectRendering(page, cdp, (await desktopNavigation?.text().catch(() => "")) ?? "", renderingResponses)),
+                },
+            });
+        }
+        await page.evaluate(axe.source);
+        const accessibility = await page.evaluate(async (tags) => await window.axe.run(document, {
+            runOnly: { type: "tag", values: tags },
+        }), axeTags);
+        const compactNodes = (nodes, limit) => nodes.slice(0, limit).map((n) => ({
+            target: [n.target].flat().map(String).join(" ").slice(0, 300),
+            html: String(n.html ?? "").slice(0, 300),
+        }));
+        const compactRules = (rules, limit) => rules.map((v) => ({
+            id: v.id,
+            impact: v.impact ?? null,
+            help: v.help,
+            helpUrl: v.helpUrl,
+            tags: (v.tags ?? []).filter((t) => /^(wcag|best-practice)/.test(t)),
+            nodeCount: v.nodes?.length ?? 0,
+            nodes: compactNodes(v.nodes ?? [], limit),
+        }));
+        const overflow = [];
+        for (const width of [320, 375, 768]) {
+            await resize(width, 900);
+            overflow.push(await inspectOverflow(page));
+        }
+        await resize(1280, 900);
+        const controls = await page.evaluate(() => [
+            ...document.querySelectorAll('a[href],button,input,select,textarea,[tabindex],[role="button"]'),
+        ].map((el, index) => {
+            const html = el;
+            const r = html.getBoundingClientRect();
+            return {
+                index,
+                tag: el.tagName,
+                id: el.id,
+                role: el.getAttribute("role"),
+                tabIndex: html.tabIndex,
+                disabled: el.disabled ?? false,
+                visible: r.width > 0 &&
+                    r.height > 0 &&
+                    getComputedStyle(el).visibility !== "hidden",
+                name: el.getAttribute("aria-label") ||
+                    el.textContent?.trim().slice(0, 200) ||
+                    null,
+            };
+        }));
+        await page.evaluate(() => {
+            document.activeElement?.blur();
+            window.scrollTo(0, 0);
+        });
+        const tabSequence = [];
+        for (let i = 0; i < Math.min(controls.length + 3, 80); i++) {
+            await page.keyboard.press("Tab");
+            tabSequence.push(await page.evaluate(() => {
+                const el = document.activeElement;
+                if (!el || el === document.body)
+                    return null;
+                const r = el.getBoundingClientRect(), style = getComputedStyle(el);
+                // Focus indicator heuristic: an outline with width, a box shadow,
+                // or a focus ring class cannot be told apart from decoration here,
+                // so only "nothing at all" is recorded as absent.
+                const outline = style.outlineStyle !== "none" && parseFloat(style.outlineWidth) > 0;
+                return {
+                    tag: el.tagName,
+                    id: el.id,
+                    role: el.getAttribute("role"),
+                    name: el.getAttribute("aria-label") ||
+                        el.textContent?.trim().slice(0, 200),
+                    tabIndex: el.tabIndex,
+                    outline: style.outline,
+                    boxShadow: style.boxShadow,
+                    visibleIndicator: outline || style.boxShadow !== "none",
+                    rect: { x: r.x, y: r.y, width: r.width, height: r.height },
+                };
+            }));
+        }
+        const stops = tabSequence.filter(Boolean);
+        // Safe interactions: scroll to the bottom once and click one load-more
+        // control outside any form. Everything else (filters, forms, logins,
+        // consent choices, infinite scroll beyond this) is explicitly not done.
+        const linkSet = () => page.evaluate(() => [...document.querySelectorAll("a[href]")].map((a) => a.href));
+        const before = new Set(await linkSet());
+        await page.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight));
+        await page
+            .waitForNetworkIdle({ idleTime: 500, timeout: 3000 })
+            .catch(() => { });
+        const afterScroll = await linkSet();
+        const scrollAdded = afterScroll.filter((l) => !before.has(l));
+        const loadMore = await page.evaluate((pattern) => {
+            const regex = new RegExp(pattern, "i");
+            const candidates = [
+                ...document.querySelectorAll('button,a[href],[role="button"]'),
+            ].filter((el) => {
+                const text = (el.textContent ?? "").replace(/\s+/g, " ").trim();
+                const r = el.getBoundingClientRect();
+                // Links with a destination are followed by the crawl instead.
+                return (regex.test(text) &&
+                    !el.hasAttribute("href") &&
+                    !el.closest("form") &&
+                    !(el instanceof HTMLButtonElement && el.form) &&
+                    !el.hasAttribute("disabled") &&
+                    el.getAttribute("aria-disabled") !== "true" &&
+                    r.width > 0 &&
+                    r.height > 0);
+            });
+            const el = candidates[0];
+            if (!el)
+                return { found: false };
+            el.scrollIntoView({ block: "center" });
+            const text = (el.textContent ?? "").replace(/\s+/g, " ").trim();
+            const href = el.href || null;
+            el.click();
+            return { found: true, text, tag: el.tagName, href };
+        }, loadMoreText.source);
+        let loadMoreAdded = [];
+        if (loadMore.found) {
+            await page
+                .waitForNetworkIdle({ idleTime: 500, timeout: 4000 })
+                .catch(() => { });
+            const known = new Set(afterScroll);
+            loadMoreAdded = (await linkSet()).filter((l) => !known.has(l));
+        }
+        const interactions = {
+            scroll: {
+                performed: true,
+                linksBefore: before.size,
+                linksAdded: scrollAdded.length,
+            },
+            loadMore: loadMore.found
+                ? {
+                    found: true,
+                    text: loadMore.text,
+                    tag: loadMore.tag,
+                    href: loadMore.href,
+                    clicked: true,
+                    navigated: page.url() !== finalUrl,
+                    linksAdded: loadMoreAdded.length,
+                }
+                : { found: false, clicked: false, linksAdded: 0 },
+            notPerformed: [
+                "form submissions",
+                "filter, sort and dropdown controls",
+                "sign-in or account actions",
+                "cookie consent choices",
+                "repeated infinite-scroll loads beyond one pass",
+            ],
+        };
+        const discovered = [
+            ...new Set([
+                ...rendered.links.flatMap((l) => (l.href ? [l.href] : [])),
+                ...scrollAdded,
+                ...loadMoreAdded,
+            ]),
+        ];
+        for (const link of discovered) {
+            if (!/^https?:/.test(link))
+                continue;
+            result.candidates.push(...linkCandidates(link, scan, `browser:${url}`));
+        }
+        for (const declaration of rendered.declarations)
+            if (declaration.href) {
+                if (/canonical|alternate|next|prev/.test(declaration.rel) &&
+                    !/markdown/i.test(declaration.type ?? ""))
+                    result.candidates.push(...linkCandidates(declaration.href, scan, `${declaration.rel}:${url}`));
+                else
+                    result.candidates.push({
+                        url: declaration.href,
+                        source: `${declaration.rel}:${url}`,
+                        kind: "resource",
+                    });
+            }
+        // Files the rendered document declares plus the static files the page
+        // requested; XHR, fetch and beacon traffic is not a file and is left out.
+        const renderedResources = [
+            ...rendered.images.flatMap((image) => [
+                image.src,
+                ...srcsetUrls(image.srcset ?? "").map((entry) => resolveLink(entry, rendered.base)),
+            ]),
+            ...rendered.scripts.map((script) => script.src),
+            ...staticResources,
+        ];
+        for (const resource of new Set(renderedResources))
+            if (resource && /^https?:/.test(resource))
+                result.candidates.push({
+                    url: resource,
+                    source: `browser-resource:${url}`,
+                    kind: "resource",
+                });
+        result.observations.push({
+            kind: "rendered",
+            key: url,
+            data: {
+                url,
+                finalUrl,
+                status: navigation?.status() ?? null,
+                title: rendered.title,
+                descriptions: rendered.descriptions,
+                language: rendered.language,
+                canonicals: rendered.canonicals.map((c) => c.href),
+                robots: rendered.robots,
+                xRobotsTag: responseHeaders["x-robots-tag"] ?? null,
+                headings: rendered.headings.slice(0, 200),
+                links: compactLinks(rendered.links),
+                structuredData: rendered.structuredData.slice(0, 20),
+                hreflangs: rendered.hreflangs,
+                wordCount: rendered.wordCount,
+                textHash: rendered.textHash,
+                prices: rendered.prices,
+                availability: rendered.availability,
+                pagination: rendered.pagination,
+                loadMore: rendered.loadMore,
+                contacts: rendered.contacts,
+                comparison,
+                interactions,
+                interactionLinks: [...scrollAdded, ...loadMoreAdded],
+                representation: "rendered-dom",
+            },
+        }, {
+            kind: "browser-detail",
+            key: url,
+            data: {
+                chromeVersion,
+                navigationDevices: scan.options.deepScan
+                    ? ["mobile", "desktop"]
+                    : ["mobile"],
+                finalUrl,
+                console: consoleMessages,
+                failedRequests: failures,
+                network,
+                accessibility: {
+                    engine: accessibility.testEngine,
+                    tags: axeTags,
+                    violations: compactRules(accessibility.violations, 10),
+                    incomplete: compactRules(accessibility.incomplete, 5),
+                    passes: accessibility.passes.length,
+                    // Rule ids that passed: the proof a later scan needs before a
+                    // formerly violated rule may count as resolved.
+                    passedRules: accessibility.passes.map((p) => p.id),
+                    inapplicable: accessibility.inapplicable.length,
+                    scope: "axe-core rules for WCAG 2.0/2.1 A and AA, the WCAG 2.2 AA rules axe supports, and best practices; automated coverage only, no conformance claim",
+                },
+                overflow,
+                keyboard: {
+                    controls,
+                    tabSequence,
+                    stepLimit: 80,
+                    stops: stops.length,
+                    withoutVisibleIndicator: stops.filter((s) => !s.visibleIndicator)
+                        .length,
+                    scope: "observed Tab sequence and computed styles; no clicks or submissions, no claim of complete keyboard accessibility",
+                },
+                limits: {
+                    console: 200,
+                    requests: 1000,
+                    failures: 500,
+                    overflowElements: 100,
+                    axeNodesPerRule: 10,
+                },
+            },
+        });
+    }
+    catch (error) {
+        errors.push({ stage: "browser", error: errorMessage(error) });
+    }
+    finally {
+        clearTimeout(watchdog);
+        await page?.close().catch(() => { });
+    }
+    result.observations.push({
+        kind: "browser",
+        key: url,
+        data: {
+            status: errors.length ? "partial" : "completed",
+            errors,
+            blockedNetwork: proxy.blocked.slice(blockedBefore),
+        },
+    });
+    return result;
+}

package/dist/engine/collect.d.ts

@@ -0,0 +1,6 @@
+import { HttpClient } from "./network.js";
+import { type Scan, type JobResult, type HttpResult } from "./types.js";
+export declare function sampledForMarkdown(url: string): boolean;
+export declare function resourceEvidence(response: HttpResult): any;
+export declare function collectPage(scan: Scan, url: string, http: HttpClient, probeMarkdown?: boolean): Promise<JobResult>;
+export declare function collectResource(url: string, http: HttpClient, scan?: Pick<Scan, "url" | "options">): Promise<JobResult>;