@fulldotdev/scan 0.1.0

package/dist/engine/language.js

@@ -0,0 +1,75 @@
+// Written-language detection from stop words and language-switch link
+// recognition. Both are heuristics; a result is only reported when it is
+// clear, and never turned into an expectation about currency or shipping.
+const stopWords = {
+    nl: "de het een en van is dat op te zijn voor met niet ook aan bij wij je om uw".split(" "),
+    en: "the and of to a in is that for with you on are as this our be by or it".split(" "),
+    de: "der die und das ist nicht mit sie ein eine für auf den dem wir ich zu auch sich im".split(" "),
+    fr: "le la les et des est une un pour dans que qui vous nous sur pas avec ce au du".split(" "),
+    es: "el la los las y de que en un una es para con por se del al como más".split(" "),
+    it: "il la di che e un una per con non sono del della gli le è come anche".split(" "),
+    pt: "o a os as de que e um uma para com não do da em por se mais como".split(" "),
+};
+export function detectLanguage(text) {
+    const tokens = text
+        .toLowerCase()
+        .split(/[^\p{L}]+/u)
+        .filter(Boolean);
+    if (tokens.length < 50)
+        return null;
+    const hits = Object.entries(stopWords).map(([lang, words]) => {
+        const set = new Set(words);
+        return [lang, tokens.filter((t) => set.has(t)).length];
+    });
+    hits.sort((a, b) => b[1] - a[1]);
+    const [best, second] = hits;
+    return best[1] >= 10 && best[1] >= second[1] * 2 ? best[0] : null;
+}
+const languageNames = {
+    english: "en",
+    nederlands: "nl",
+    dutch: "nl",
+    deutsch: "de",
+    german: "de",
+    français: "fr",
+    francais: "fr",
+    french: "fr",
+    español: "es",
+    espanol: "es",
+    spanish: "es",
+    italiano: "it",
+    italian: "it",
+    português: "pt",
+    portugues: "pt",
+    portuguese: "pt",
+    svenska: "sv",
+    dansk: "da",
+    norsk: "no",
+    suomi: "fi",
+    polski: "pl",
+    čeština: "cs",
+    日本語: "ja",
+    中文: "zh",
+};
+export function languageOfTag(tag) {
+    return tag ? tag.toLowerCase().split(/[-_]/)[0] : null;
+}
+// Links that switch language: an hreflang attribute, a language name or a
+// two-letter code as the visible text, or a language-prefixed path from a
+// menu that names the language.
+export function languageLinks(links) {
+    return links.flatMap((link) => {
+        if (!link.href)
+            return [];
+        const text = (link.text ?? "").trim().toLowerCase();
+        const expected = languageOfTag(link.hreflang) ??
+            languageNames[text] ??
+            (/^[a-z]{2}(-[a-z]{2})?$/.test(text) &&
+                Object.values(languageNames).includes(text.slice(0, 2))
+                ? text.slice(0, 2)
+                : null);
+        return expected
+            ? [{ url: link.href, expected, text: link.text, location: link.location }]
+            : [];
+    });
+}

package/dist/engine/lighthouse-evidence.d.ts

@@ -0,0 +1,36 @@
+export declare function lighthouseEvidence(lhr: any, detailed?: boolean): {
+    failedAudits: any[];
+    diagnostics: {
+        [k: string]: any;
+    } | undefined;
+    lcp: {
+        element: any;
+        checks: any;
+        phases: {
+            [k: string]: any;
+        };
+        timingBasis: string;
+        source: string;
+    };
+};
+export declare function measurementSpread(runs: any[], requested: number, reason: string | null): {
+    requested: number;
+    completed: number;
+    valid: number;
+    reason: string | null;
+    incomplete: boolean;
+    performance: {
+        count: number;
+        min: number;
+        max: number | undefined;
+        median: number;
+        range: number;
+    } | null;
+    lcpMs: {
+        count: number;
+        min: number;
+        max: number | undefined;
+        median: number;
+        range: number;
+    } | null;
+};

package/dist/engine/lighthouse-evidence.js

@@ -0,0 +1,69 @@
+export function lighthouseEvidence(lhr, detailed = true) {
+    const audits = lhr.audits ?? {};
+    const retained = Object.values(audits).filter((a) => (typeof a.score === "number" && a.score < 1) ||
+        a.scoreDisplayMode === "error");
+    const diagnostics = Object.fromEntries([
+        "lcp-breakdown-insight",
+        "lcp-discovery-insight",
+        "largest-contentful-paint-element",
+        "render-blocking-insight",
+        "render-blocking-resources",
+        "image-delivery-insight",
+        "font-display-insight",
+        "layout-shifts-insight",
+        "network-dependency-tree-insight",
+    ]
+        .filter((id) => audits[id])
+        .map((id) => [id, audits[id]]));
+    const discoveryItems = audits["lcp-discovery-insight"]?.details?.items ?? [];
+    const breakdownItems = audits["lcp-breakdown-insight"]?.details?.items ?? [];
+    const checks = discoveryItems.find((i) => i.type === "checklist")?.items ?? null;
+    const phases = Object.fromEntries((breakdownItems.find((i) => i.type === "table")?.items ?? []).map((i) => [i.subpart, i.duration]));
+    return {
+        failedAudits: detailed
+            ? retained
+            : retained.map((a) => ({
+                id: a.id,
+                title: a.title,
+                score: a.score,
+                displayValue: a.displayValue,
+                errorMessage: a.errorMessage,
+            })),
+        diagnostics: detailed ? diagnostics : undefined,
+        lcp: {
+            element: [...discoveryItems, ...breakdownItems].find((i) => i.type === "node") ?? null,
+            checks,
+            phases,
+            timingBasis: "Observed trace timings, not the simulated LCP used in the performance score",
+            source: "Lighthouse trace; applies to image and background-image LCP",
+        },
+    };
+}
+export function measurementSpread(runs, requested, reason) {
+    const spread = (values) => {
+        const sorted = values.filter(Number.isFinite).sort((a, b) => a - b);
+        return sorted.length
+            ? {
+                count: sorted.length,
+                min: sorted[0],
+                max: sorted.at(-1),
+                median: sorted.length % 2
+                    ? sorted[Math.floor(sorted.length / 2)]
+                    : (sorted[sorted.length / 2 - 1] + sorted[sorted.length / 2]) / 2,
+                range: sorted.at(-1) - sorted[0],
+            }
+            : null;
+    };
+    return {
+        requested,
+        completed: runs.length,
+        valid: runs.filter((r) => !r.runtimeError && !r.auditErrors?.length && !r.missingScores?.length).length,
+        reason,
+        incomplete: runs.length < requested ||
+            runs.some((r) => r.runtimeError || r.auditErrors?.length || r.missingScores?.length),
+        performance: spread(runs.map((r) => typeof r.scores?.performance === "number"
+            ? r.scores.performance * 100
+            : NaN)),
+        lcpMs: spread(runs.map((r) => r.metrics?.["largest-contentful-paint"]?.numericValue ?? NaN)),
+    };
+}

package/dist/engine/lighthouse.d.ts

@@ -0,0 +1,4 @@
+import type { BrowserSession } from "./browser.js";
+import { HttpClient } from "./network.js";
+import { type JobResult, type Scan } from "./types.js";
+export declare function collectLighthouse(scan: Scan, url: string, http: HttpClient, session: BrowserSession): Promise<JobResult>;

package/dist/engine/lighthouse.js

@@ -0,0 +1,284 @@
+import { lighthouseEvidence, measurementSpread, } from "./lighthouse-evidence.js";
+import { resolvePublic, userAgent } from "./network.js";
+import { errorMessage } from "./types.js";
+// Lighthouse on the rotating sample: mobile and desktop, repeats in deep
+// scans, the mobile final screenshot as artifact. Separate from the per-page
+// browser pass because two navigations per device make it the most
+// expensive measurement.
+export async function collectLighthouse(scan, url, http, session) {
+    await resolvePublic(new URL(url).hostname);
+    const robots = await http.getRobots(url);
+    if (!robots.allowed ||
+        robots.parser?.isAllowed(url, "FulldevScan") === false)
+        return {
+            observations: [
+                {
+                    kind: "lighthouse-run",
+                    key: url,
+                    data: { status: "skipped", reason: "robots" },
+                },
+            ],
+        };
+    const { proxy, lighthouse, browser, args, chromeVersion } = session;
+    const blockedBefore = proxy.blocked.length;
+    const result = { observations: [], artifacts: [] };
+    const errors = [];
+    const jobStarted = Date.now();
+    const budgetSeconds = scan.options.deepScan ? 270 : 210;
+    const watchdog = setTimeout(() => {
+        errors.push({
+            stage: "budget",
+            error: `Lighthouse job exceeded ${budgetSeconds} seconds`,
+        });
+        void browser.close();
+    }, budgetSeconds * 1000);
+    watchdog.unref();
+    try {
+        const port = Number(new URL(browser.wsEndpoint()).port);
+        for (const device of ["mobile", "desktop"]) {
+            const measurements = [];
+            const screenshots = new Map();
+            let requestedRuns = 1;
+            let repeatReason = null;
+            for (let measurement = 0; measurement < requestedRuns; measurement++) {
+                if (measurement > 0 && Date.now() - jobStarted > 150000)
+                    break;
+                try {
+                    await http.pace(url);
+                    const run = await lighthouse(url, {
+                        port,
+                        output: "json",
+                        logLevel: "error",
+                        onlyCategories: [
+                            "performance",
+                            "accessibility",
+                            "best-practices",
+                            "seo",
+                        ],
+                        maxWaitForLoad: 30000,
+                        ...(device === "desktop"
+                            ? {
+                                formFactor: "desktop",
+                                screenEmulation: {
+                                    mobile: false,
+                                    width: 1350,
+                                    height: 940,
+                                    deviceScaleFactor: 1,
+                                    disabled: false,
+                                },
+                                throttling: {
+                                    rttMs: 40,
+                                    throughputKbps: 10240,
+                                    cpuSlowdownMultiplier: 1,
+                                    requestLatencyMs: 0,
+                                    downloadThroughputKbps: 0,
+                                    uploadThroughputKbps: 0,
+                                },
+                            }
+                            : {}),
+                        extraHeaders: { "User-Agent": userAgent },
+                    });
+                    if (!run)
+                        throw new Error("Lighthouse did not return a report");
+                    const lhr = run.lhr;
+                    const auditErrors = Object.entries(lhr.audits)
+                        .filter(([, audit]) => audit.scoreDisplayMode === "error")
+                        .map(([id, audit]) => ({ id, error: audit.errorMessage ?? null }));
+                    const missingScores = Object.entries(lhr.categories)
+                        .filter(([, category]) => category.score === null)
+                        .map(([id]) => id);
+                    const evidence = lighthouseEvidence(lhr, scan.options.deepScan);
+                    const audit = (id) => lhr.audits[id] ?? null;
+                    // Audits with measurable savings, largest first.
+                    const opportunities = Object.values(lhr.audits)
+                        .filter((a) => typeof a.score === "number" &&
+                        a.score < 0.9 &&
+                        (a.metricSavings ||
+                            a.details?.overallSavingsMs ||
+                            a.details?.overallSavingsBytes))
+                        .map((a) => {
+                        const savings = [
+                            a.details?.overallSavingsMs ?? 0,
+                            ...Object.values(a.metricSavings ?? {}).filter((n) => typeof n === "number"),
+                        ];
+                        return {
+                            id: a.id,
+                            title: a.title,
+                            displayValue: a.displayValue ?? null,
+                            savingsMs: Math.round(Math.max(...savings)) || null,
+                            savingsBytes: a.details?.overallSavingsBytes
+                                ? Math.round(a.details.overallSavingsBytes)
+                                : null,
+                            // Up to five affected files or elements as evidence.
+                            items: (a.details?.items ?? [])
+                                .slice(0, 5)
+                                .map((item) => ({
+                                url: item.url ?? item.entity ?? null,
+                                node: item.node?.selector ?? item.node?.snippet ?? null,
+                                wastedMs: item.wastedMs ?? null,
+                                wastedBytes: item.wastedBytes ?? null,
+                                totalBytes: item.totalBytes ?? null,
+                            })),
+                        };
+                    })
+                        .filter((o) => (o.savingsMs ?? 0) >= 100 || (o.savingsBytes ?? 0) >= 50000)
+                        .sort((a, b) => (b.savingsMs ?? 0) - (a.savingsMs ?? 0))
+                        .slice(0, 8);
+                    const weight = Object.fromEntries((audit("resource-summary")?.details?.items ?? []).map((item) => [
+                        item.resourceType,
+                        {
+                            bytes: item.transferSize ?? 0,
+                            requests: item.requestCount ?? 0,
+                        },
+                    ]));
+                    const unsizedImages = audit("unsized-images")?.details?.items?.length ?? 0;
+                    const screenshot = audit("final-screenshot")?.details?.data;
+                    if (device === "mobile" && typeof screenshot === "string") {
+                        const match = /^data:(image\/\w+);base64,(.+)$/.exec(screenshot);
+                        if (match)
+                            screenshots.set(lhr.fetchTime, {
+                                kind: "screenshot",
+                                key: url,
+                                body: Buffer.from(match[2], "base64"),
+                                contentType: match[1],
+                            });
+                    }
+                    measurements.push({
+                        ...evidence,
+                        url,
+                        device,
+                        lighthouseVersion: lhr.lighthouseVersion,
+                        chromeVersion,
+                        fetchTime: lhr.fetchTime,
+                        requestedUrl: lhr.requestedUrl,
+                        finalDisplayedUrl: lhr.finalDisplayedUrl,
+                        runtimeError: lhr.runtimeError ?? null,
+                        auditErrors,
+                        missingScores,
+                        launchArgs: args,
+                        runWarnings: lhr.runWarnings,
+                        scores: Object.fromEntries(Object.entries(lhr.categories).map(([k, v]) => [k, v.score])),
+                        metrics: Object.fromEntries([
+                            "first-contentful-paint",
+                            "largest-contentful-paint",
+                            "cumulative-layout-shift",
+                            "total-blocking-time",
+                            "speed-index",
+                            "interactive",
+                        ].map((id) => [id, audit(id)])),
+                        configSettings: lhr.configSettings,
+                        environment: lhr.environment,
+                        timing: lhr.timing,
+                        networkRequests: scan.options.deepScan
+                            ? audit("network-requests")
+                            : undefined,
+                        resourceSummary: scan.options.deepScan
+                            ? audit("resource-summary")
+                            : undefined,
+                        thirdParties: scan.options.deepScan
+                            ? (audit("third-parties-insight") ?? audit("third-party-summary"))
+                            : undefined,
+                        opportunities,
+                        weight,
+                        unsizedImages,
+                        note: scan.options.deepScan
+                            ? "Deep lab scan through public-egress proxy. Full failed audit details retained per run."
+                            : "Routine lab scan through public-egress proxy. Compact failed audit summaries; detailed diagnostics require a deep scan.",
+                    });
+                    if (measurement === 0 &&
+                        scan.options.deepScan &&
+                        (lhr.runtimeError ||
+                            auditErrors.length ||
+                            missingScores.length ||
+                            scan.options.lighthouseRepeatBelow === undefined ||
+                            (typeof lhr.categories.performance?.score === "number" &&
+                                lhr.categories.performance.score * 100 <
+                                    scan.options.lighthouseRepeatBelow))) {
+                        requestedRuns = scan.options.lighthouseMaxRuns;
+                        repeatReason =
+                            lhr.runtimeError || auditErrors.length || missingScores.length
+                                ? "incomplete-audit"
+                                : scan.options.lighthouseRepeatBelow === undefined
+                                    ? "deep-scan"
+                                    : `performance-below-${scan.options.lighthouseRepeatBelow}`;
+                    }
+                    if (lhr.runtimeError)
+                        errors.push({
+                            stage: `lighthouse-${device}`,
+                            error: lhr.runtimeError,
+                        });
+                    else if (auditErrors.length || missingScores.length)
+                        errors.push({
+                            stage: `lighthouse-${device}`,
+                            error: "Lighthouse returned incomplete audits or category scores",
+                            auditErrors,
+                            missingScores,
+                        });
+                }
+                catch (error) {
+                    if (measurement === 0 && scan.options.deepScan) {
+                        requestedRuns = scan.options.lighthouseMaxRuns;
+                        repeatReason = "lighthouse-error";
+                    }
+                    measurements.push({
+                        url,
+                        device,
+                        scores: {},
+                        metrics: {},
+                        runtimeError: { message: errorMessage(error) },
+                    });
+                    errors.push({
+                        stage: `lighthouse-${device}`,
+                        error: errorMessage(error),
+                    });
+                }
+            }
+            if (measurements.length) {
+                const spread = measurementSpread(measurements, requestedRuns, repeatReason);
+                const sorted = [...measurements]
+                    .filter((m) => !m.runtimeError && typeof m.scores?.performance === "number")
+                    .sort((a, b) => a.scores.performance - b.scores.performance);
+                const representative = sorted[Math.floor(sorted.length / 2)] ?? measurements[0];
+                const screenshot = screenshots.get(representative.fetchTime);
+                if (screenshot)
+                    result.artifacts.push(screenshot);
+                result.observations.push({
+                    kind: "lighthouse",
+                    key: `${url}|${device}`,
+                    data: {
+                        ...representative,
+                        ...(scan.options.deepScan
+                            ? {
+                                failedAudits: undefined,
+                                diagnostics: undefined,
+                                networkRequests: undefined,
+                                resourceSummary: undefined,
+                                thirdParties: undefined,
+                                measurements,
+                                representativeIndex: measurements.indexOf(representative),
+                            }
+                            : {}),
+                        spread,
+                        representative: "median performance run; all runs retained",
+                    },
+                });
+            }
+        }
+    }
+    catch (error) {
+        errors.push({ stage: "lighthouse", error: errorMessage(error) });
+    }
+    finally {
+        clearTimeout(watchdog);
+    }
+    result.observations.push({
+        kind: "lighthouse-run",
+        key: url,
+        data: {
+            status: errors.length ? "partial" : "completed",
+            errors,
+            blockedNetwork: proxy.blocked.slice(blockedBefore),
+        },
+    });
+    return result;
+}

package/dist/engine/log.d.ts

@@ -0,0 +1 @@
+export declare function log(event: string, fields?: Record<string, unknown>): void;

package/dist/engine/log.js

@@ -0,0 +1,4 @@
+// One JSON line per event; Netlify function logs are the only debugging surface.
+export function log(event, fields = {}) {
+    console.log(JSON.stringify({ event, time: new Date().toISOString(), ...fields }));
+}

package/dist/engine/network.d.ts

@@ -0,0 +1,53 @@
+import { Agent } from "undici";
+import { type HttpResult, type Options } from "./types.js";
+export declare const userAgent: string;
+declare const robotsParser: (url: string, body: string) => {
+    isAllowed: (url: string, agent?: string) => boolean | undefined;
+    getCrawlDelay: (agent?: string) => number | undefined;
+};
+export declare function sameSite(a: string, b: string): boolean;
+export declare function normalizeUrl(input: string, base?: string): string;
+export declare function slashTwin(url: string): string | null;
+export declare function setLocalTargetsAllowed(allowed: boolean): void;
+export declare function localTargets(): boolean;
+export declare function publicAddress(address: string): boolean;
+export declare function resolvePublic(hostname: string): Promise<{
+    address: string;
+    family: number;
+}[]>;
+type Robots = {
+    result: HttpResult;
+    parser: ReturnType<typeof robotsParser> | null;
+    allowed: boolean;
+};
+export declare class HttpSession {
+    robots: Map<string, Promise<Robots>>;
+    lastRequest: Map<string, number>;
+    slowdown: Map<string, number>;
+    private agents;
+    agent(origin: string, pinned: {
+        address: string;
+        family: number;
+    }, timeout: number): Agent;
+    close(): Promise<void>;
+}
+export declare class HttpClient {
+    readonly options: Options;
+    private resolver;
+    readonly deadline: number;
+    readonly session: HttpSession;
+    constructor(options: Options, resolver?: typeof resolvePublic, deadline?: number, session?: HttpSession);
+    get remainingMs(): number;
+    getRobots(url: string): Promise<Robots>;
+    get(input: string, settings?: {
+        accept?: string;
+        respectRobots?: boolean;
+        retry?: boolean;
+        maxBytes?: number;
+    }): Promise<HttpResult>;
+    private slowDown;
+    delayFor(origin: string): number;
+    pace(input: string): Promise<void>;
+    private request;
+}
+export {};