@fulldotdev/scan 0.1.0

package/dist/engine/network.js

@@ -0,0 +1,296 @@
+import dns from "node:dns/promises";
+import { isIP } from "node:net";
+import ipaddr from "ipaddr.js";
+import { Agent, fetch } from "undici";
+import robotsParserModule from "robots-parser";
+import { setTimeout as delay } from "node:timers/promises";
+import { errorMessage, version, } from "./types.js";
+export const userAgent = `FulldevScan/${version} (+https://scan.full.dev)`;
+const robotsParser = robotsParserModule;
+export function sameSite(a, b) {
+    return (new URL(a).hostname.replace(/^www\./, "") ===
+        new URL(b).hostname.replace(/^www\./, ""));
+}
+export function normalizeUrl(input, base) {
+    const url = new URL(input, base);
+    if (!["https:", "http:"].includes(url.protocol) ||
+        url.username ||
+        url.password)
+        throw new Error("Only public HTTP(S) URLs without credentials are supported");
+    if (url.port && !["80", "443"].includes(url.port) && !localTargetsAllowed)
+        throw new Error("Only ports 80 and 443 are supported");
+    url.hash = "";
+    return url.href;
+}
+// The same path with or without a trailing slash; null for the root or
+// URLs with a query string.
+export function slashTwin(url) {
+    const parsed = new URL(url);
+    if (parsed.pathname.length <= 1 || parsed.search)
+        return null;
+    parsed.pathname = parsed.pathname.endsWith("/")
+        ? parsed.pathname.slice(0, -1)
+        : `${parsed.pathname}/`;
+    return parsed.href;
+}
+// A scan of a local development server needs loopback and private
+// addresses and any port; every other scan refuses them.
+let localTargetsAllowed = false;
+export function setLocalTargetsAllowed(allowed) {
+    localTargetsAllowed = allowed;
+}
+export function localTargets() {
+    return localTargetsAllowed;
+}
+export function publicAddress(address) {
+    try {
+        const parsed = ipaddr.process(address);
+        if (localTargetsAllowed)
+            return ["unicast", "loopback", "private"].includes(parsed.range());
+        return parsed.range() === "unicast";
+    }
+    catch {
+        return false;
+    }
+}
+export async function resolvePublic(hostname) {
+    const host = hostname.replace(/^\[|\]$/g, "");
+    if (!localTargetsAllowed &&
+        (host === "localhost" ||
+            host.endsWith(".localhost") ||
+            host.endsWith(".local")))
+        throw new Error("Private host blocked");
+    const addresses = isIP(host)
+        ? [{ address: host, family: isIP(host) }]
+        : await Promise.race([
+            dns.lookup(host, { all: true }),
+            new Promise((_, reject) => {
+                const timer = setTimeout(() => reject(new Error("DNS lookup timeout")), 5000);
+                timer.unref();
+            }),
+        ]);
+    if (!addresses.length || addresses.some((a) => !publicAddress(a.address)))
+        throw new Error("Non-public address blocked");
+    return addresses;
+}
+// Shared across every job of one scan inside a worker: robots.txt is fetched
+// once per origin, request pacing spans jobs and connections are kept alive.
+export class HttpSession {
+    robots = new Map();
+    lastRequest = new Map();
+    // Origins that answered 429 during this scan get a longer gap between
+    // requests for the rest of the session (Shopify limits per minute).
+    slowdown = new Map();
+    agents = new Map();
+    agent(origin, pinned, timeout) {
+        const key = `${origin}|${pinned.address}`;
+        let agent = this.agents.get(key);
+        if (!agent) {
+            agent = new Agent({
+                connect: {
+                    timeout,
+                    lookup: (_host, options, callback) => {
+                        if (options.all)
+                            callback(null, [pinned]);
+                        else
+                            callback(null, pinned.address, pinned.family);
+                    },
+                },
+            });
+            this.agents.set(key, agent);
+        }
+        return agent;
+    }
+    async close() {
+        const agents = [...this.agents.values()];
+        this.agents.clear();
+        await Promise.all(agents.map((agent) => agent.close()));
+    }
+}
+export class HttpClient {
+    options;
+    resolver;
+    deadline;
+    session;
+    constructor(options, resolver = resolvePublic, deadline = Date.now() + 150000, session = new HttpSession()) {
+        this.options = options;
+        this.resolver = resolver;
+        this.deadline = deadline;
+        this.session = session;
+    }
+    get remainingMs() {
+        return Math.max(0, this.deadline - Date.now());
+    }
+    async getRobots(url) {
+        const origin = new URL(url).origin;
+        const robots = this.session.robots;
+        if (!robots.has(origin))
+            robots.set(origin, (async () => {
+                const result = await this.get(`${origin}/robots.txt`, {
+                    respectRobots: false,
+                    retry: false,
+                });
+                const usable = result.status === 200 &&
+                    !result.truncated &&
+                    !/^\s*</.test(result.body);
+                return {
+                    result,
+                    parser: usable ? robotsParser(result.finalUrl, result.body) : null,
+                    allowed: usable || result.status === 404 || result.status === 410,
+                };
+            })());
+        return robots.get(origin);
+    }
+    async get(input, settings = {}) {
+        let result = await this.request(input, settings);
+        if (settings.retry === false)
+            return result;
+        if (result.status === 429) {
+            // Rate limited (Shopify does this on product pages): slow this origin
+            // down for the rest of the scan, wait what the server asks (capped)
+            // and try once more.
+            this.slowDown(input);
+            const asked = Number(result.headers["retry-after"]) || 0;
+            await delay(Math.min(Math.max(asked * 1000, 2000), 8000));
+            result = { ...(await this.request(input, settings)), attempts: 2 };
+            if (result.status === 429)
+                this.slowDown(input);
+        }
+        else if ((result.status !== null && result.status >= 500) ||
+            result.outcome === "error") {
+            await delay(400);
+            result = { ...(await this.request(input, settings)), attempts: 2 };
+        }
+        return result;
+    }
+    // Gap between requests to one origin: the configured delay, or more once
+    // the origin has rate limited us. Doubles per 429, capped at 5 seconds.
+    slowDown(input) {
+        const origin = new URL(input).origin;
+        const current = this.session.slowdown.get(origin) ?? 0;
+        this.session.slowdown.set(origin, Math.min(current ? current * 2 : 1500, 5000));
+    }
+    delayFor(origin) {
+        return Math.max(this.options.requestDelayMs, this.session.slowdown.get(origin) ?? 0);
+    }
+    // Waits until this origin may be requested again and records the request.
+    // Also used before browser page loads, which count against the same limit.
+    async pace(input) {
+        const origin = new URL(input).origin;
+        const wait = Math.max(0, this.delayFor(origin) -
+            (Date.now() - (this.session.lastRequest.get(origin) ?? 0)));
+        if (wait)
+            await delay(wait);
+        this.session.lastRequest.set(origin, Date.now());
+    }
+    async request(input, settings) {
+        const start = performance.now();
+        const result = {
+            url: input,
+            finalUrl: input,
+            status: null,
+            headers: {},
+            redirects: [],
+            body: "",
+            bytes: 0,
+            durationMs: 0,
+            fetchedAt: new Date().toISOString(),
+            outcome: "error",
+            attempts: 1,
+        };
+        try {
+            if (this.remainingMs < 1000) {
+                result.outcome = "inconclusive";
+                result.error = "collector-budget-exhausted";
+                return result;
+            }
+            let current = normalizeUrl(input);
+            const visited = new Set();
+            for (let redirects = 0; redirects <= 5; redirects++) {
+                if (this.remainingMs < 1000) {
+                    result.outcome = "inconclusive";
+                    result.error = "collector-budget-exhausted";
+                    break;
+                }
+                if (visited.has(current))
+                    throw new Error("Redirect loop");
+                visited.add(current);
+                const url = new URL(current);
+                if (settings.respectRobots !== false &&
+                    url.pathname !== "/robots.txt") {
+                    const policy = await this.getRobots(current);
+                    if (!policy.allowed ||
+                        policy.parser?.isAllowed(current, "FulldevScan") === false) {
+                        result.outcome = "blocked";
+                        result.error =
+                            "Robots policy disallows this request or is unavailable";
+                        break;
+                    }
+                }
+                const addresses = await this.resolver(url.hostname);
+                const pinned = addresses[0];
+                await this.pace(current);
+                const dispatcher = this.session.agent(url.origin, pinned, this.options.requestTimeoutMs);
+                const response = await fetch(current, {
+                    dispatcher,
+                    redirect: "manual",
+                    signal: AbortSignal.timeout(Math.max(1, Math.min(this.options.requestTimeoutMs, this.remainingMs))),
+                    headers: {
+                        "user-agent": userAgent,
+                        accept: settings.accept ?? "*/*",
+                    },
+                });
+                result.finalUrl = current;
+                result.status = response.status;
+                result.headers = Object.fromEntries([...response.headers].filter(([k]) => !["set-cookie", "authorization", "proxy-authorization"].includes(k)));
+                if ([301, 302, 303, 307, 308].includes(response.status) &&
+                    response.headers.has("location")) {
+                    const location = normalizeUrl(response.headers.get("location"), current);
+                    result.redirects.push({
+                        url: current,
+                        status: response.status,
+                        location,
+                    });
+                    await response.body?.cancel();
+                    if (redirects === 5)
+                        throw new Error("Redirect limit reached");
+                    current = location;
+                    continue;
+                }
+                const chunks = [];
+                const reader = response.body?.getReader();
+                const limit = settings.maxBytes ?? this.options.maxResponseBytes;
+                if (reader)
+                    while (true) {
+                        const { done, value } = await reader.read();
+                        if (done)
+                            break;
+                        const remaining = limit - result.bytes;
+                        chunks.push(Buffer.from(value.slice(0, remaining)));
+                        result.bytes += Math.min(value.length, remaining);
+                        if (value.length > remaining || result.bytes >= limit) {
+                            result.truncated = true;
+                            await reader.cancel();
+                            break;
+                        }
+                    }
+                result.raw = Buffer.concat(chunks);
+                result.body = result.raw.toString("utf8");
+                result.outcome = [401, 403, 429].includes(response.status)
+                    ? "inconclusive"
+                    : response.ok
+                        ? "ok"
+                        : "http-error";
+                break;
+            }
+        }
+        catch (error) {
+            result.error = errorMessage(error);
+            result.outcome = /blocked|Only public|Only ports/.test(result.error)
+                ? "blocked"
+                : "error";
+        }
+        result.durationMs = Math.round(performance.now() - start);
+        return result;
+    }
+}

package/dist/engine/proxy.d.ts

@@ -0,0 +1,8 @@
+export declare function createEgressProxy(): Promise<{
+    port: number;
+    blocked: {
+        target: string;
+        error: string;
+    }[];
+    close: () => Promise<void>;
+}>;

package/dist/engine/proxy.js

@@ -0,0 +1,95 @@
+import http from "node:http";
+import net from "node:net";
+import { localTargets, normalizeUrl, resolvePublic } from "./network.js";
+// Every browser connection, including redirects/subresources, goes through DNS-pinned public egress.
+export async function createEgressProxy() {
+    const blocked = [];
+    const sockets = new Set();
+    const remember = (socket) => {
+        sockets.add(socket);
+        socket.on("close", () => sockets.delete(socket));
+        socket.setTimeout(30000, () => socket.destroy());
+    };
+    const report = (target, error) => {
+        if (blocked.length < 500)
+            blocked.push({ target, error: String(error) });
+    };
+    const server = http.createServer(async (req, res) => {
+        try {
+            const url = new URL(normalizeUrl(req.url ?? ""));
+            if (url.protocol !== "http:")
+                throw new Error("Use CONNECT for HTTPS");
+            const [address] = await resolvePublic(url.hostname);
+            const headers = {
+                ...req.headers,
+                host: url.host,
+            };
+            delete headers["proxy-authorization"];
+            delete headers["proxy-connection"];
+            const upstream = http.request({
+                host: address.address,
+                port: 80,
+                path: url.pathname + url.search,
+                method: req.method,
+                headers,
+                timeout: 15000,
+            }, (response) => {
+                res.writeHead(response.statusCode ?? 502, response.headers);
+                response.pipe(res);
+            });
+            upstream.on("socket", remember);
+            upstream.on("error", (error) => {
+                report(url.href, error);
+                res.destroy();
+            });
+            upstream.on("timeout", () => upstream.destroy());
+            req.on("aborted", () => upstream.destroy());
+            req.pipe(upstream);
+        }
+        catch (error) {
+            report(req.url ?? "", error);
+            res.writeHead(403);
+            res.end("Destination blocked");
+        }
+    });
+    server.on("connection", remember);
+    server.on("connect", async (req, client, head) => {
+        try {
+            const url = new URL(`https://${req.url}`);
+            normalizeUrl(url.href);
+            if (url.port && url.port !== "443" && !localTargets())
+                throw new Error("CONNECT only supports port 443");
+            const [address] = await resolvePublic(url.hostname);
+            const upstream = net.connect({
+                host: address.address,
+                port: Number(url.port || 443),
+            });
+            remember(upstream);
+            upstream.on("connect", () => {
+                client.write("HTTP/1.1 200 Connection Established\r\n\r\n");
+                if (head.length)
+                    upstream.write(head);
+                client.pipe(upstream);
+                upstream.pipe(client);
+            });
+            upstream.on("error", () => client.destroy());
+            client.on("error", () => upstream.destroy());
+            client.on("close", () => upstream.destroy());
+        }
+        catch (error) {
+            report(req.url ?? "", error);
+            client.end("HTTP/1.1 403 Forbidden\r\n\r\n");
+        }
+    });
+    await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+    const port = server.address().port;
+    return {
+        port,
+        blocked,
+        close: async () => {
+            for (const socket of sockets)
+                socket.destroy();
+            await new Promise((resolve) => server.close(() => resolve()));
+        },
+    };
+}

package/dist/engine/run.d.ts

@@ -0,0 +1,38 @@
+import { ScanState } from "./state.js";
+import { type JobKind, type Options, type Scan } from "./types.js";
+import { type PreviousScan, type Report, type StoredFinding } from "../report/build.js";
+export type ScanEvent = {
+    type: "start";
+    scan: Scan;
+} | {
+    type: "phase";
+    phase: "crawl" | "lighthouse" | "analysis" | "report";
+} | {
+    type: "job";
+    kind: JobKind;
+    key: string;
+    outcome: "completed" | "failed" | "retry";
+    attempt: number;
+    durationMs: number;
+    error?: string;
+    pending: number;
+} | {
+    type: "done";
+    scan: Scan;
+};
+export interface RunInput {
+    url: string;
+    options?: Partial<Options>;
+    rotation?: number;
+    previous?: PreviousScan;
+    budgetMs?: number;
+    signal?: AbortSignal;
+    onEvent?: (event: ScanEvent) => void;
+}
+export interface RunOutput {
+    scan: Scan;
+    report: Report;
+    findings: StoredFinding[];
+    state: ScanState;
+}
+export declare function runScan(input: RunInput): Promise<RunOutput>;

package/dist/engine/run.js

@@ -0,0 +1,202 @@
+import { randomUUID } from "node:crypto";
+import { analyze } from "./analysis.js";
+import { collectPage, collectResource, sampledForMarkdown } from "./collect.js";
+import { HttpClient, HttpSession, normalizeUrl, resolvePublic, setLocalTargetsAllowed, } from "./network.js";
+import { selectLighthousePages } from "./select.js";
+import { collectSitemap, discover } from "./site.js";
+import { ScanState } from "./state.js";
+import { errorMessage, optionsSchema, version, } from "./types.js";
+import { buildReport, } from "../report/build.js";
+const crawlKinds = [
+    "discover",
+    "sitemap",
+    "page",
+    "resource",
+    "browser",
+];
+const maxJobMs = 300000;
+export async function runScan(input) {
+    const options = optionsSchema.parse(input.options ?? {});
+    setLocalTargetsAllowed(options.allowLocal);
+    const url = normalizeUrl(input.url);
+    await resolvePublic(new URL(url).hostname);
+    const scan = {
+        id: randomUUID(),
+        url,
+        status: "running",
+        options,
+        createdAt: new Date().toISOString(),
+        rotation: input.rotation ?? 0,
+        coverage: { omittedCandidates: 0 },
+    };
+    const state = new ScanState(scan);
+    const emit = (event) => input.onEvent?.(event);
+    emit({ type: "start", scan });
+    const session = new HttpSession();
+    const http = new HttpClient(options, undefined, undefined, session);
+    const deadline = input.budgetMs ? Date.now() + input.budgetMs : Infinity;
+    let browser;
+    const browserSession = async () => {
+        const { openBrowserSession } = await import("./browser.js");
+        if (browser && !browser.connected) {
+            await browser.close();
+            browser = undefined;
+        }
+        return (browser ??= await openBrowserSession());
+    };
+    const run = async (job) => {
+        switch (job.kind) {
+            case "discover":
+                return discover(scan, http);
+            case "sitemap":
+                return collectSitemap(scan, job.key, http);
+            case "page":
+                return collectPage(scan, job.key, http, job.key === scan.url ||
+                    sampledForMarkdown(job.key) ||
+                    state.hasUsableMarkdown());
+            case "resource":
+                return collectResource(job.key, http, scan);
+            case "browser":
+                return (await import("./browser.js")).collectBrowser(scan, job.key, http, await browserSession());
+            case "lighthouse":
+                return (await import("./lighthouse.js")).collectLighthouse(scan, job.key, http, await browserSession());
+            case "analysis":
+                return analyze(scan, state.records(["page", "resource", "markdown", "rendered"]), state.urlRows());
+        }
+    };
+    const execute = async (job, kinds) => {
+        job.status = "running";
+        job.attempts++;
+        const started = performance.now();
+        try {
+            const result = await Promise.race([
+                run(job),
+                new Promise((_, reject) => {
+                    const timer = setTimeout(() => reject(new Error(`Job exceeded ${maxJobMs / 1000} seconds`)), maxJobMs);
+                    timer.unref();
+                }),
+            ]);
+            result.observations.push({
+                kind: "collector",
+                key: `${job.kind}:${job.key}`,
+                data: {
+                    collector: job.kind,
+                    version,
+                    attempt: job.attempts,
+                    durationMs: Math.round(performance.now() - started),
+                    observations: result.observations.length,
+                },
+            });
+            state.record(result);
+            job.status = "completed";
+            job.durationMs = Math.round(performance.now() - started);
+            emit({
+                type: "job",
+                kind: job.kind,
+                key: job.key,
+                outcome: "completed",
+                attempt: job.attempts,
+                durationMs: job.durationMs,
+                pending: state.pending(kinds),
+            });
+        }
+        catch (error) {
+            const message = errorMessage(error);
+            // Browser and Lighthouse jobs get one retry; the rest three attempts.
+            const limit = ["browser", "lighthouse"].includes(job.kind) ? 2 : 3;
+            job.error = message;
+            job.status = job.attempts < limit ? "queued" : "failed";
+            job.durationMs = Math.round(performance.now() - started);
+            if (job.status === "failed" && ["browser", "lighthouse"].includes(job.kind))
+                state.record({
+                    observations: [
+                        {
+                            kind: job.kind === "lighthouse" ? "lighthouse-run" : "browser",
+                            key: job.key,
+                            data: {
+                                status: "skipped",
+                                reason: "collector-failed",
+                                error: message,
+                                attempts: job.attempts,
+                            },
+                        },
+                    ],
+                });
+            emit({
+                type: "job",
+                kind: job.kind,
+                key: job.key,
+                outcome: job.status === "failed" ? "failed" : "retry",
+                attempt: job.attempts,
+                durationMs: job.durationMs,
+                error: message,
+                pending: state.pending(kinds),
+            });
+        }
+        // Chromium and Lighthouse leave memory behind; a fresh browser for the
+        // next page is cheaper than the process growing without bound.
+        if (["browser", "lighthouse"].includes(job.kind) &&
+            browser &&
+            process.memoryUsage().rss > 1500 * 1024 * 1024) {
+            await browser.close();
+            browser = undefined;
+        }
+    };
+    const drain = async (kinds) => {
+        let job;
+        while ((job = state.next(kinds))) {
+            if (Date.now() > deadline || input.signal?.aborted) {
+                state.cancelQueued();
+                scan.coverage.expired = true;
+                break;
+            }
+            await execute(job, kinds);
+        }
+    };
+    try {
+        emit({ type: "phase", phase: "crawl" });
+        state.enqueue("discover", "site");
+        await drain(crawlKinds);
+        emit({ type: "phase", phase: "lighthouse" });
+        const pages = state
+            .records(["page"])
+            .filter((p) => p.data?.http?.outcome === "ok" && p.data?.html !== false)
+            .sort((a, b) => a.key.localeCompare(b.key));
+        const { selected, templates } = selectLighthousePages(pages, options.browser && !scan.coverage.expired
+            ? options
+            : { lighthouse: "off", lighthousePerTemplate: 1 }, scan.rotation, scan.url);
+        scan.coverage.templates = templates;
+        scan.coverage.lighthouseEligible = pages.length;
+        scan.coverage.lighthouseSelected = selected;
+        scan.coverage.lighthouseSelection =
+            options.lighthouse === "all"
+                ? "Every eligible HTML page"
+                : options.lighthouse === "templates"
+                    ? `Homepage plus ${options.lighthousePerTemplate} rotating page per template (${templates} templates)`
+                    : "Lighthouse disabled";
+        for (const url of selected)
+            state.enqueue("lighthouse", url);
+        await drain(["lighthouse"]);
+    }
+    finally {
+        await browser?.close();
+        browser = undefined;
+    }
+    emit({ type: "phase", phase: "analysis" });
+    state.enqueue("analysis", "site");
+    await drain(["analysis"]);
+    await session.close();
+    emit({ type: "phase", phase: "report" });
+    scan.coverage.jobs = state.jobCounts();
+    const unfinished = state.jobs.some((j) => ["failed", "cancelled", "queued"].includes(j.status));
+    scan.status = unfinished || scan.coverage.expired ? "partial" : "completed";
+    scan.finishedAt = new Date().toISOString();
+    const { report, findings } = await buildReport(scan, {
+        records: state.records(),
+        urls: state.urlRows(),
+        artifacts: [...state.artifacts.values()],
+        previous: input.previous,
+    });
+    emit({ type: "done", scan });
+    return { scan, report, findings, state };
+}

package/dist/engine/select.d.ts

@@ -0,0 +1,6 @@
+import type { Observation, Options } from "./types.js";
+export declare function templateKey(page: Observation): string;
+export declare function selectLighthousePages(pages: Observation[], options: Pick<Options, "lighthouse" | "lighthousePerTemplate">, rotation: number, home: string): {
+    selected: string[];
+    templates: number;
+};

package/dist/engine/select.js

@@ -0,0 +1,38 @@
+import { pageType, templateGroup } from "./crawl-scope.js";
+import { structuredTypes } from "./html.js";
+// Pages that share a page type and a URL pattern almost always share a
+// template, and Lighthouse on one of them says what it would say for all.
+export function templateKey(page) {
+    const types = (page.data.structuredData ?? []).flatMap((s) => s.parseValid ? structuredTypes(s.data) : []);
+    const type = pageType(page.key, {
+        structuredTypes: types,
+        pagination: page.data.pagination ?? null,
+    });
+    return `${type}:${templateGroup(page.key)}`;
+}
+// The homepage always, then `perTemplate` pages of every template. Each
+// template walks through its pages from run to run so every page gets its
+// turn over time.
+export function selectLighthousePages(pages, options, rotation, home) {
+    const groups = new Map();
+    for (const page of pages) {
+        const id = templateKey(page);
+        groups.set(id, [...(groups.get(id) ?? []), page.key]);
+    }
+    if (options.lighthouse === "off")
+        return { selected: [], templates: groups.size };
+    const urls = pages.map((p) => p.key);
+    if (options.lighthouse === "all")
+        return { selected: [...new Set(urls)], templates: groups.size };
+    const homepage = urls.find((u) => u === home) ??
+        urls.find((u) => new URL(u).pathname === "/");
+    const picks = homepage ? [homepage] : [];
+    const take = options.lighthousePerTemplate;
+    for (const members of groups.values()) {
+        const sorted = [...members].sort();
+        const count = Math.min(take, sorted.length);
+        for (let j = 0; j < count; j++)
+            picks.push(sorted[(rotation * count + j) % sorted.length]);
+    }
+    return { selected: [...new Set(picks)], templates: groups.size };
+}