@fulldotdev/scan 0.1.0

package/dist/engine/site.js

@@ -0,0 +1,758 @@
+import { candidateKind, crawlUrl } from "./crawl-scope.js";
+import dns from "node:dns/promises";
+import tls from "node:tls";
+import { getDomain } from "tldts";
+import { XMLParser, XMLValidator } from "fast-xml-parser";
+import { gunzipSync } from "node:zlib";
+import { resolvePublic, sameSite, userAgent } from "./network.js";
+import { env } from "./env.js";
+import { resolveLink, markdownEvidence, markdownLinks } from "./html.js";
+import { errorMessage, httpEvidence, } from "./types.js";
+export const xmlParser = new XMLParser({
+    ignoreAttributes: false,
+    processEntities: true,
+    parseTagValue: false,
+    removeNSPrefix: true,
+});
+const array = (value) => value === undefined ? [] : Array.isArray(value) ? value : [value];
+export function parseSitemap(body) {
+    if (/<!DOCTYPE|<!ENTITY/i.test(body))
+        return {
+            valid: false,
+            error: "DTD/entity declarations are not supported",
+            urls: [],
+            sitemaps: [],
+        };
+    const valid = XMLValidator.validate(body);
+    if (valid !== true)
+        return { valid: false, error: valid.err.msg, urls: [], sitemaps: [] };
+    const data = xmlParser.parse(body);
+    const urls = array(data.urlset?.url).map((node) => ({
+        url: String(node.loc ?? ""),
+        lastmod: node.lastmod ?? null,
+        priority: node.priority ?? null,
+        changefreq: node.changefreq ?? null,
+    }));
+    const sitemaps = array(data.sitemapindex?.sitemap).map((node) => String(node.loc ?? ""));
+    return {
+        valid: !!(data.urlset !== undefined || data.sitemapindex !== undefined),
+        urls,
+        sitemaps,
+    };
+}
+// Crawlers grouped by what they do with a page. Products change; the bot
+// name stays the evidence.
+export const botPurposes = {
+    search: [
+        "Googlebot",
+        "bingbot",
+        "DuckDuckBot",
+        "Applebot",
+        "OAI-SearchBot",
+        "Claude-SearchBot",
+        "PerplexityBot",
+    ],
+    training: [
+        "GPTBot",
+        "ClaudeBot",
+        "Google-Extended",
+        "CCBot",
+        "Applebot-Extended",
+        "Bytespider",
+        "meta-externalagent",
+    ],
+    "user-request": ["ChatGPT-User", "Claude-User", "Perplexity-User"],
+};
+export function crawlerPolicy(robots, url) {
+    return {
+        robotsUrl: robots.result.finalUrl,
+        robotsOutcome: robots.result.outcome,
+        bots: Object.entries(botPurposes).flatMap(([purpose, bots]) => bots.map((bot) => ({
+            bot,
+            purpose,
+            allowed: robots.allowed
+                ? (robots.parser?.isAllowed(url, bot) ?? true)
+                : null,
+        }))),
+        interpretation: "declared robots policy; does not establish real bot access, usage, indexing or citations",
+    };
+}
+// One sitemap document per job, so a large index resumes where the previous
+// worker stopped. Child sitemaps and page URLs become candidates.
+export async function collectSitemap(scan, url, http) {
+    const response = await http.get(url, {
+        respectRobots: false,
+        retry: false,
+        maxBytes: scan.options.maxSitemapBytes,
+    });
+    let parsed = {
+        valid: false,
+        error: "Not a successful response",
+        urls: [],
+        sitemaps: [],
+    };
+    const artifacts = [];
+    if (response.status === 200 && !response.truncated) {
+        try {
+            const body = response.raw?.[0] === 0x1f && response.raw?.[1] === 0x8b
+                ? gunzipSync(response.raw, {
+                    maxOutputLength: scan.options.maxSitemapBytes,
+                }).toString("utf8")
+                : response.body;
+            parsed = parseSitemap(body);
+            artifacts.push({
+                kind: "sitemap",
+                key: url,
+                body,
+                contentType: "application/xml",
+            });
+        }
+        catch (error) {
+            parsed = {
+                valid: false,
+                error: errorMessage(error),
+                urls: [],
+                sitemaps: [],
+            };
+        }
+    }
+    const candidates = [];
+    let external = 0;
+    const entries = parsed.urls.map((entry) => {
+        const resolved = resolveLink(entry.url, url);
+        const scanTarget = resolved ? crawlUrl(resolved, scan) : null;
+        if (scanTarget && sameSite(scanTarget, scan.url))
+            candidates.push({
+                url: scanTarget,
+                source: `sitemap:${url}`,
+                kind: candidateKind(scanTarget, scan),
+            });
+        else
+            external++;
+        return {
+            url: entry.url,
+            scanUrl: scanTarget,
+            lastmod: entry.lastmod,
+            ...(entry.priority ? { priority: entry.priority } : {}),
+            ...(entry.changefreq ? { changefreq: entry.changefreq } : {}),
+        };
+    });
+    const children = parsed.sitemaps.flatMap((child) => {
+        const resolved = resolveLink(child, response.finalUrl || url);
+        return resolved ? [crawlUrl(resolved, scan)] : [];
+    });
+    for (const child of children)
+        if (sameSite(child, scan.url))
+            candidates.push({
+                url: child,
+                source: `sitemap:${url}`,
+                kind: "sitemap",
+            });
+    return {
+        observations: [
+            {
+                kind: "sitemap",
+                key: url,
+                data: {
+                    http: httpEvidence(response),
+                    valid: parsed.valid,
+                    error: "error" in parsed ? parsed.error : undefined,
+                    gzip: response.raw?.[0] === 0x1f && response.raw?.[1] === 0x8b,
+                    sitemaps: children,
+                    urls: entries,
+                    entryCount: entries.length,
+                    externalEntries: external,
+                    truncated: !!response.truncated,
+                },
+            },
+        ],
+        artifacts,
+        candidates,
+    };
+}
+export async function dnsSnapshot(host) {
+    const domain = getDomain(host) ?? host;
+    const tasks = [
+        ["a", () => dns.resolve4(host)],
+        ["aaaa", () => dns.resolve6(host)],
+        ["cname", () => dns.resolveCname(host)],
+        ["ns", () => dns.resolveNs(domain)],
+        ["mx", () => dns.resolveMx(domain)],
+        ["txt", () => dns.resolveTxt(domain)],
+        ["dmarc", () => dns.resolveTxt(`_dmarc.${domain}`)],
+        ["mtasts", () => dns.resolveTxt(`_mta-sts.${domain}`)],
+        ["bimi", () => dns.resolveTxt(`default._bimi.${domain}`)],
+        ["caa", () => dns.resolveCaa(domain)],
+    ];
+    const records = Object.fromEntries(await Promise.all(tasks.map(async ([key, fn]) => {
+        try {
+            return [
+                key,
+                {
+                    status: "observed",
+                    values: await Promise.race([
+                        fn(),
+                        new Promise((_, reject) => {
+                            const t = setTimeout(() => reject(new Error("DNS timeout")), 5000);
+                            t.unref();
+                        }),
+                    ]),
+                },
+            ];
+        }
+        catch (error) {
+            return [key, { status: "unavailable", error: errorMessage(error) }];
+        }
+    })));
+    const txt = records.txt.values?.map((parts) => parts.join("")) ?? [];
+    const dmarc = records.dmarc.values?.map((parts) => parts.join("")) ?? [];
+    return {
+        host,
+        domain,
+        records,
+        email: {
+            spf: txt.filter((t) => t.startsWith("v=spf1")),
+            dmarc: dmarc.filter((t) => t.startsWith("v=DMARC1")),
+            mtaSts: records.mtasts.values?.map((p) => p.join("")) ?? [],
+            bimi: records.bimi.values?.map((p) => p.join("")) ?? [],
+            scope: "public-record inspection only; no mailbox delivery or DKIM-selector discovery",
+        },
+    };
+}
+export async function tlsSnapshot(host) {
+    try {
+        const [address] = await resolvePublic(host);
+        return await new Promise((resolve) => {
+            const socket = tls.connect({
+                host: address.address,
+                port: 443,
+                servername: host,
+                rejectUnauthorized: false,
+            }, () => {
+                const cert = socket.getPeerCertificate();
+                const hostnameError = tls.checkServerIdentity(host, cert);
+                resolve({
+                    status: "observed",
+                    authorized: socket.authorized,
+                    authorizationError: socket.authorizationError ?? null,
+                    hostnameValid: !hostnameError,
+                    hostnameError: hostnameError?.message ?? null,
+                    protocol: socket.getProtocol(),
+                    cipher: socket.getCipher(),
+                    certificate: {
+                        subject: cert.subject,
+                        issuer: cert.issuer,
+                        subjectaltname: cert.subjectaltname,
+                        validFrom: cert.valid_from,
+                        validTo: cert.valid_to,
+                        fingerprint256: cert.fingerprint256,
+                        serialNumber: cert.serialNumber,
+                    },
+                    daysRemaining: cert.valid_to
+                        ? Math.floor((Date.parse(cert.valid_to) - Date.now()) / 86400000)
+                        : null,
+                });
+                socket.destroy();
+            });
+            socket.setTimeout(10000, () => socket.destroy(new Error("TLS timeout")));
+            socket.on("error", (error) => {
+                resolve({ status: "unavailable", error: errorMessage(error) });
+                socket.destroy();
+            });
+        });
+    }
+    catch (error) {
+        return { status: "unavailable", error: errorMessage(error) };
+    }
+}
+// Passive discovery of the files agents read: llms.txt, its full variant,
+// auth.md and the OAuth protected-resource metadata it should point at.
+// Everything is fetched with the normal public-network guard; nothing
+// registers, authenticates or follows an authorization flow.
+async function aiDiscovery(root, http, result) {
+    const text = async (path) => {
+        const url = `${root.origin}${path}`;
+        const response = await http.get(url, { retry: false });
+        return { url, response };
+    };
+    const llms = await text("/llms.txt");
+    const llmsFull = await text("/llms-full.txt");
+    const markdownFile = (file) => {
+        const content = file.response.body;
+        const links = file.response.outcome === "ok" ? markdownLinks(content, file.url) : [];
+        if (file.response.outcome === "ok")
+            result.artifacts.push({
+                kind: "ai-file",
+                key: file.url,
+                body: content,
+                contentType: "text/plain",
+            });
+        return {
+            url: file.url,
+            http: httpEvidence(file.response),
+            present: file.response.status === 200,
+            ...markdownEvidence(content, file.response.headers["content-type"]),
+            title: content.match(/^#\s+(.+)$/m)?.[1] ?? null,
+            headings: [...content.matchAll(/^#{1,3}\s+(.+)$/gm)]
+                .map((m) => m[1].trim())
+                .slice(0, 50),
+            links: links.slice(0, 500),
+            characters: content.length,
+        };
+    };
+    result.observations.push({
+        kind: "ai-discovery",
+        key: root.origin,
+        data: {
+            llms: markdownFile(llms),
+            llmsFull: markdownFile(llmsFull),
+            scope: "Passive GET of optional discovery files; absence is informational and nothing was registered or authenticated",
+        },
+    });
+    for (const file of [llms, llmsFull])
+        if (file.response.outcome === "ok")
+            for (const link of markdownLinks(file.response.body, file.url))
+                if (link.url)
+                    result.candidates.push({
+                        url: link.url,
+                        source: `llms:${file.url}`,
+                        kind: "resource",
+                    });
+}
+export async function discover(scan, http) {
+    const result = {
+        observations: [],
+        artifacts: [],
+        candidates: [{ url: scan.url, source: "submitted", kind: "page" }],
+    };
+    const root = new URL(scan.url);
+    await resolvePublic(root.hostname);
+    const [dnsData, tlsData, robots] = await Promise.all([
+        dnsSnapshot(root.hostname),
+        tlsSnapshot(root.hostname),
+        http.getRobots(scan.url),
+    ]);
+    result.observations.push({ kind: "dns", key: root.hostname, data: dnsData }, { kind: "tls", key: root.hostname, data: tlsData }, {
+        kind: "robots",
+        key: root.origin,
+        data: {
+            http: httpEvidence(robots.result),
+            content: robots.result.body,
+            crawlerPolicyEvaluatedPerPage: true,
+            policy: crawlerPolicy(robots, scan.url),
+        },
+    });
+    result.artifacts.push({
+        kind: "robots",
+        key: root.origin,
+        body: robots.result.body,
+        contentType: "text/plain",
+    });
+    const declared = [...robots.result.body.matchAll(/^sitemap:\s*(\S+)/gim)]
+        .map((m) => resolveLink(m[1], root.origin))
+        .filter((u) => !!u)
+        .map((u) => crawlUrl(u, scan));
+    for (const url of declared)
+        if (sameSite(url, scan.url))
+            result.candidates.push({ url, source: "robots", kind: "sitemap" });
+    for (const path of [
+        "/sitemap.xml",
+        "/sitemap_index.xml",
+        "/sitemap-index.xml",
+    ])
+        result.candidates.push({
+            url: `${root.origin}${path}`,
+            source: "default-location",
+            kind: "sitemap",
+        });
+    await aiDiscovery(root, http, result);
+    for (const path of ["/.well-known/security.txt", "/security.txt"]) {
+        const url = `${root.origin}${path}`;
+        const response = await http.get(url, { retry: false });
+        const content = response.body;
+        const fields = content
+            .split(/\r?\n/)
+            .filter((line) => !line.startsWith("#"))
+            .flatMap((line) => {
+            const m = /^([A-Za-z-]+):\s*(.+)$/.exec(line);
+            return m ? [{ name: m[1].toLowerCase(), value: m[2] }] : [];
+        });
+        result.observations.push({
+            kind: "security-txt",
+            key: url,
+            data: { http: httpEvidence(response), fields, content },
+        });
+        if (response.outcome === "ok")
+            result.artifacts.push({
+                kind: "security-txt",
+                key: url,
+                body: content,
+                contentType: "text/plain",
+            });
+    }
+    const variantHosts = new Set([root.hostname]);
+    const domain = getDomain(root.hostname);
+    if (domain &&
+        (root.hostname === domain || root.hostname === `www.${domain}`)) {
+        variantHosts.add(domain);
+        variantHosts.add(`www.${domain}`);
+    }
+    for (const host of variantHosts)
+        for (const protocol of ["http:", "https:"]) {
+            const url = `${protocol}//${host}/`;
+            const response = await http.get(url, { retry: false, maxBytes: 1000 });
+            result.observations.push({
+                kind: "url-variant",
+                key: url,
+                data: httpEvidence(response),
+            });
+        }
+    // Free external sources, all independent; none may fail the job. A local
+    // target has nothing registered anywhere, so they are skipped.
+    const registered = domain ?? root.hostname;
+    const siteAddress = dnsData.records.a?.values?.[0] ?? null;
+    if (!scan.options.external || scan.options.allowLocal)
+        return result;
+    const [domainData, field, history, safe, labs, preload, certificates, lists] = await Promise.all([
+        domainSnapshot(registered),
+        fieldData({ origin: root.origin }),
+        fieldHistory(root.origin),
+        safeBrowsing(scan.url),
+        sslLabs(root.hostname),
+        hstsPreload(registered),
+        certificateLogs(registered),
+        blocklists(registered, siteAddress),
+    ]);
+    result.observations.push({
+        kind: "domain",
+        key: registered,
+        data: domainData,
+    });
+    if (field)
+        result.observations.push({
+            kind: "field-data",
+            key: root.origin,
+            data: field,
+        });
+    if (history)
+        result.observations.push({
+            kind: "field-history",
+            key: root.origin,
+            data: history,
+        });
+    if (safe)
+        result.observations.push({
+            kind: "safe-browsing",
+            key: scan.url,
+            data: safe,
+        });
+    result.observations.push({ kind: "ssl-labs", key: root.hostname, data: labs }, { kind: "hsts-preload", key: registered, data: preload }, { kind: "certificates", key: registered, data: certificates }, { kind: "blocklists", key: registered, data: lists });
+    return result;
+}
+// Registration data through the RDAP bootstrap service: expiry date and
+// registrar. Public registry data only.
+export async function domainSnapshot(domain) {
+    try {
+        const response = await fetch(`https://rdap.org/domain/${encodeURIComponent(domain)}`, {
+            headers: {
+                accept: "application/rdap+json, application/json",
+                "user-agent": userAgent,
+            },
+            signal: AbortSignal.timeout(10000),
+        });
+        if (!response.ok)
+            return {
+                status: "unavailable",
+                domain,
+                error: `RDAP ${response.status}`,
+            };
+        const data = await response.json();
+        const expiration = (data.events ?? []).find((e) => e.eventAction === "expiration")?.eventDate;
+        const registrar = (data.entities ?? [])
+            .find((e) => (e.roles ?? []).includes("registrar"))
+            ?.vcardArray?.[1]?.find((v) => v[0] === "fn")?.[3];
+        return {
+            status: "observed",
+            domain,
+            expiresAt: expiration ?? null,
+            daysRemaining: expiration
+                ? Math.floor((Date.parse(expiration) - Date.now()) / 86400000)
+                : null,
+            registrar: registrar ?? null,
+            statuses: data.status ?? [],
+            source: "rdap.org",
+        };
+    }
+    catch (error) {
+        return { status: "unavailable", domain, error: errorMessage(error) };
+    }
+}
+// Real-user Core Web Vitals from the Chrome UX Report when a product-level
+// API key is configured. 28-day window, 75th percentile, for one origin or
+// one page URL; the level is part of the evidence.
+// The Chrome UX Report API allows about 150 queries per minute; page jobs
+// can run faster than that on small pages, so requests are paced.
+let lastFieldRequest = 0;
+export async function fieldData(target) {
+    const key = googleKey();
+    if (!key)
+        return null;
+    const level = "url" in target ? "url" : "origin";
+    const wait = lastFieldRequest + 450 - Date.now();
+    if (wait > 0)
+        await new Promise((r) => setTimeout(r, wait));
+    lastFieldRequest = Date.now();
+    try {
+        const response = await fetch(`https://chromeuxreport.googleapis.com/v1/records:queryRecord?key=${encodeURIComponent(key)}`, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify(target),
+            signal: AbortSignal.timeout(10000),
+        });
+        if (response.status === 404)
+            return {
+                available: false,
+                level,
+                note: level === "url"
+                    ? "Not enough Chrome traffic for this page"
+                    : "Not enough Chrome traffic for field data",
+            };
+        if (!response.ok)
+            return {
+                available: false,
+                level,
+                note: `Chrome UX Report ${response.status}`,
+            };
+        const { record } = await response.json();
+        const p75 = (metric) => {
+            const value = record?.metrics?.[metric]?.percentiles?.p75;
+            return value === undefined || value === null ? null : Number(value);
+        };
+        const date = (d) => d
+            ? `${d.year}-${String(d.month).padStart(2, "0")}-${String(d.day).padStart(2, "0")}`
+            : null;
+        return {
+            available: true,
+            level,
+            lcp: p75("largest_contentful_paint"),
+            inp: p75("interaction_to_next_paint"),
+            cls: p75("cumulative_layout_shift"),
+            period: {
+                from: date(record?.collectionPeriod?.firstDate),
+                to: date(record?.collectionPeriod?.lastDate),
+            },
+            source: "Chrome UX Report API",
+        };
+    }
+    catch (error) {
+        return { available: false, level, note: errorMessage(error) };
+    }
+}
+const googleKey = () => env("GOOGLE_API_KEY") || env("CRUX_API_KEY");
+async function getJson(url, init = {}, timeoutMs = 15000) {
+    const response = await fetch(url, {
+        ...init,
+        headers: {
+            "user-agent": userAgent,
+            accept: "application/json",
+            ...(init.headers ?? {}),
+        },
+        signal: AbortSignal.timeout(timeoutMs),
+    });
+    return {
+        response,
+        body: response.ok ? await response.json().catch(() => null) : null,
+    };
+}
+// Google Safe Browsing: is the site flagged as malware or phishing?
+export async function safeBrowsing(url) {
+    const key = googleKey();
+    if (!key)
+        return null;
+    try {
+        const { response, body } = await getJson(`https://safebrowsing.googleapis.com/v4/threatMatches:find?key=${encodeURIComponent(key)}`, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify({
+                client: { clientId: "fulldev-monitor", clientVersion: "1" },
+                threatInfo: {
+                    threatTypes: [
+                        "MALWARE",
+                        "SOCIAL_ENGINEERING",
+                        "UNWANTED_SOFTWARE",
+                        "POTENTIALLY_HARMFUL_APPLICATION",
+                    ],
+                    platformTypes: ["ANY_PLATFORM"],
+                    threatEntryTypes: ["URL"],
+                    threatEntries: [{ url }],
+                },
+            }),
+        });
+        if (!response.ok)
+            return {
+                status: "unavailable",
+                error: `Safe Browsing ${response.status}`,
+            };
+        const threats = (body?.matches ?? []).map((m) => String(m.threatType));
+        return {
+            status: "observed",
+            flagged: threats.length > 0,
+            threats: [...new Set(threats)],
+        };
+    }
+    catch (error) {
+        return { status: "unavailable", error: errorMessage(error) };
+    }
+}
+// Chrome UX Report history: weekly 75th percentiles for the last 25 weeks.
+export async function fieldHistory(origin) {
+    const key = googleKey();
+    if (!key)
+        return null;
+    try {
+        const { response, body } = await getJson(`https://chromeuxreport.googleapis.com/v1/records:queryHistoryRecord?key=${encodeURIComponent(key)}`, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify({ origin }),
+        });
+        if (response.status === 404)
+            return { available: false };
+        if (!response.ok)
+            return { available: false, note: `Chrome UX Report ${response.status}` };
+        const record = body?.record;
+        const series = (metric) => (record?.metrics?.[metric]?.percentilesTimeseries?.p75s ?? []).map((v) => (v === null || v === undefined ? null : Number(v)));
+        const periods = (record?.collectionPeriods ?? []).map((p) => {
+            const d = p.lastDate;
+            return d
+                ? `${d.year}-${String(d.month).padStart(2, "0")}-${String(d.day).padStart(2, "0")}`
+                : null;
+        });
+        const lcp = series("largest_contentful_paint"), inp = series("interaction_to_next_paint"), cls = series("cumulative_layout_shift");
+        return {
+            available: true,
+            points: periods.map((to, i) => ({
+                to,
+                lcp: lcp[i] ?? null,
+                inp: inp[i] ?? null,
+                cls: cls[i] ?? null,
+            })),
+        };
+    }
+    catch (error) {
+        return { available: false, note: errorMessage(error) };
+    }
+}
+// SSL Labs grade. Cached results up to a week old are accepted; a fresh
+// assessment started here is picked up by the next scan.
+export async function sslLabs(host) {
+    try {
+        const { response, body } = await getJson(`https://api.ssllabs.com/api/v3/analyze?host=${encodeURIComponent(host)}&publish=off&fromCache=on&maxAge=168&all=done`);
+        if (!response.ok)
+            return { status: "unavailable", error: `SSL Labs ${response.status}` };
+        if (body?.status !== "READY")
+            return {
+                status: "pending",
+                note: body?.status === "ERROR"
+                    ? body?.statusMessage
+                    : "assessment running, result on the next scan",
+            };
+        const endpoints = (body.endpoints ?? []);
+        const grades = endpoints
+            .map((e) => e.grade)
+            .filter(Boolean)
+            .sort();
+        return {
+            status: "observed",
+            grade: grades[0] ?? null,
+            hasWarnings: endpoints.some((e) => e.hasWarnings),
+            endpoints: endpoints.length,
+            testedAt: body.testTime ? new Date(body.testTime).toISOString() : null,
+            source: "SSL Labs",
+        };
+    }
+    catch (error) {
+        return { status: "unavailable", error: errorMessage(error) };
+    }
+}
+export async function hstsPreload(domain) {
+    try {
+        const { response, body } = await getJson(`https://hstspreload.org/api/v2/status?domain=${encodeURIComponent(domain)}`, {}, 8000);
+        if (!response.ok)
+            return {
+                status: "unavailable",
+                error: `hstspreload.org ${response.status}`,
+            };
+        return { status: "observed", preload: body?.status ?? "unknown" };
+    }
+    catch (error) {
+        return { status: "unavailable", error: errorMessage(error) };
+    }
+}
+// Certificate transparency logs: every hostname that ever got a certificate.
+export async function certificateLogs(domain) {
+    try {
+        const { response, body } = await getJson(`https://crt.sh/?q=${encodeURIComponent(`%.${domain}`)}&exclude=expired&output=json`, {}, 30000);
+        if (!response.ok || !Array.isArray(body))
+            return { status: "unavailable", error: `crt.sh ${response.status}` };
+        const monthAgo = Date.now() - 30 * 86400000;
+        const hostnames = new Set();
+        const issuers = new Set();
+        let recent = 0;
+        for (const entry of body) {
+            for (const name of String(entry.name_value ?? "").split("\n"))
+                if (name && !name.startsWith("*"))
+                    hostnames.add(name.toLowerCase());
+            if (entry.issuer_name)
+                issuers.add(String(entry.issuer_name).replace(/^.*O=([^,]+).*$/, "$1"));
+            if (Date.parse(entry.not_before) > monthAgo)
+                recent++;
+        }
+        return {
+            status: "observed",
+            hostnames: [...hostnames].sort().slice(0, 200),
+            issuers: [...issuers].slice(0, 20),
+            issuedLast30Days: recent,
+            source: "crt.sh",
+        };
+    }
+    catch (error) {
+        return { status: "unavailable", error: errorMessage(error) };
+    }
+}
+// SURBL answers plain DNS queries for free; Spamhaus refuses shared
+// resolvers such as Netlify's, so it is left out.
+export async function blocklists(domain, _address) {
+    const lookups = [
+        {
+            list: "SURBL",
+            query: `${domain}.multi.surbl.org`,
+            refused: /^127\.0\.0\.1$/,
+            listed: /^127\.0\.0\./,
+        },
+    ];
+    const results = await Promise.all(lookups.map(async ({ list, query, refused, listed }) => {
+        try {
+            const answers = await Promise.race([
+                dns.resolve4(query),
+                new Promise((_, reject) => {
+                    const t = setTimeout(() => reject(new Error("DNS timeout")), 5000);
+                    t.unref();
+                }),
+            ]);
+            if (answers.some((a) => refused.test(a)))
+                return { list, status: "unavailable", note: "query refused" };
+            return {
+                list,
+                status: answers.some((a) => listed.test(a)) ? "listed" : "clean",
+                answers,
+            };
+        }
+        catch (error) {
+            const code = error?.code;
+            return code === "ENOTFOUND" || code === "ENODATA"
+                ? { list, status: "clean" }
+                : { list, status: "unavailable", note: errorMessage(error) };
+        }
+    }));
+    return { status: "observed", results };
+}