@fulldotdev/scan 0.1.0

package/dist/engine/collect.js

@@ -0,0 +1,359 @@
+import { srcsetUrls } from "./srcset.js";
+import { crawlUrl, linkCandidates } from "./crawl-scope.js";
+import { createHash } from "node:crypto";
+import { sameSite, slashTwin } from "./network.js";
+import { extractPage, isHtml, resolveLink, markdownEvidence, markdownLinks, textSimilarity, } from "./html.js";
+import { crawlerPolicy, fieldData, xmlParser } from "./site.js";
+import { XMLValidator } from "fast-xml-parser";
+import { httpEvidence, } from "./types.js";
+// Deterministic 5% sample so suffix probing does not triple requests site-wide.
+export function sampledForMarkdown(url) {
+    return createHash("sha256").update(url).digest()[0] < 13;
+}
+// Compact evidence of a fetched HTML destination that is not crawled as a
+// page in its own right (a trailing-slash twin, an external page).
+function htmlSummary(response) {
+    const html = extractPage(response);
+    return {
+        title: html.title,
+        ids: html.ids,
+        canonicals: html.canonicals,
+        robots: html.robots,
+        xRobotsTag: html.xRobotsTag,
+        textHash: html.textHash,
+    };
+}
+export function resourceEvidence(response) {
+    const data = {
+        http: httpEvidence(response),
+        sha256: response.raw
+            ? createHash("sha256").update(response.raw).digest("hex")
+            : null,
+        hashScope: response.truncated ? "prefix-only" : "entire-decoded-response",
+        declaredLength: response.headers["content-length"] ?? null,
+        cacheControl: response.headers["cache-control"] ?? null,
+        etag: response.headers.etag ?? null,
+        lastModified: response.headers["last-modified"] ?? null,
+        sniffedType: response.raw?.subarray(0, 5).toString() === "%PDF-"
+            ? "pdf"
+            : response.raw?.subarray(0, 8).toString("hex") === "89504e470d0a1a0a"
+                ? "png"
+                : response.raw?.subarray(0, 3).toString("hex") === "ffd8ff"
+                    ? "jpeg"
+                    : response.raw?.subarray(0, 4).toString("hex") === "52494646"
+                        ? "webp"
+                        : response.raw?.subarray(0, 4).toString("hex") === "47494638"
+                            ? "gif"
+                            : null,
+    };
+    if (response.outcome === "ok" && isHtml(response))
+        data.html = htmlSummary(response);
+    return data;
+}
+export async function collectPage(scan, url, http, probeMarkdown = true) {
+    const response = await http.get(url);
+    if (response.outcome !== "ok" || !isHtml(response))
+        return {
+            observations: [
+                {
+                    kind: "page",
+                    key: url,
+                    data: { http: httpEvidence(response), html: false },
+                },
+            ],
+        };
+    const page = extractPage(response);
+    const observations = [
+        { kind: "page", key: url, data: page },
+    ];
+    const candidates = [];
+    const twin = slashTwin(url);
+    if (twin) {
+        const alternate = await http.get(twin);
+        const alternatePage = alternate.outcome === "ok" && isHtml(alternate)
+            ? extractPage(alternate)
+            : null;
+        const canonical = (value) => value.canonicals.flatMap((entry) => entry.href ? [crawlUrl(entry.href, scan)] : []);
+        let result;
+        if ([404, 410].includes(alternate.status ?? 0))
+            result = "absent";
+        else if (alternate.outcome !== "ok")
+            result = "unknown";
+        else if (alternate.finalUrl === response.finalUrl &&
+            (alternate.redirects.length || response.redirects.length))
+            result = "redirected";
+        else if (response.truncated || alternate.truncated || !alternatePage)
+            result = "unknown";
+        else if (alternate.redirects.length || response.redirects.length)
+            result = "different-redirect-target";
+        else if (!page.mainText || !alternatePage.mainText)
+            result = "unknown";
+        else if (page.textHash !== alternatePage.textHash)
+            result = "different-content";
+        else {
+            const originals = canonical(page), alternates = canonical(alternatePage);
+            result =
+                originals.length === 1 &&
+                    alternates.length === 1 &&
+                    originals[0] === alternates[0] &&
+                    [url, twin].includes(originals[0])
+                    ? "canonicalized"
+                    : "duplicate";
+        }
+        observations.push({
+            kind: "slash-variant",
+            key: url.endsWith("/") ? url.slice(0, -1) : url,
+            data: {
+                url,
+                alternateUrl: twin,
+                result,
+                original: {
+                    http: httpEvidence(response),
+                    canonicals: canonical(page),
+                    textHash: page.textHash,
+                },
+                alternate: {
+                    http: httpEvidence(alternate),
+                    canonicals: alternatePage ? canonical(alternatePage) : [],
+                    textHash: alternatePage?.textHash ?? null,
+                },
+            },
+        }, 
+        // The twin was fetched here; its resource record spares a second
+        // request when another page links to it.
+        { kind: "resource", key: twin, data: resourceEvidence(alternate) });
+    }
+    for (const link of page.links) {
+        // Cloudflare rewrites obfuscated mailto links to /cdn-cgi/l/email-protection,
+        // which answers 404 to anything but a browser.
+        if (!link.href || new URL(link.href).pathname.startsWith("/cdn-cgi/"))
+            continue;
+        candidates.push(...linkCandidates(link.href, scan, `link:${url}`));
+    }
+    for (const link of page.declarations)
+        if (link.href) {
+            const mapped = /canonical|alternate|next|prev/.test(link.rel);
+            const target = mapped ? crawlUrl(link.href, scan) : link.href;
+            if (mapped && !/markdown/i.test(link.type ?? ""))
+                candidates.push(...linkCandidates(link.href, scan, `${link.rel}:${url}`));
+            else
+                candidates.push({
+                    url: target,
+                    source: `${link.rel}:${url}`,
+                    kind: "resource",
+                });
+        }
+    for (const image of page.images) {
+        if (image.src)
+            candidates.push({
+                url: image.src,
+                source: `image:${url}`,
+                kind: "resource",
+            });
+        for (const entry of srcsetUrls(image.srcset ?? "")) {
+            const src = resolveLink(entry, page.base);
+            if (src)
+                candidates.push({
+                    url: src,
+                    source: `srcset:${url}`,
+                    kind: "resource",
+                });
+        }
+    }
+    for (const script of page.scripts)
+        if (script.src)
+            candidates.push({
+                url: script.src,
+                source: `script:${url}`,
+                kind: "resource",
+            });
+    for (const meta of page.social)
+        if (/^(og:image(?::url)?|twitter:image(?::src)?)$/.test(meta.property ?? meta.name ?? "")) {
+            const src = resolveLink(meta.content, page.base);
+            if (src)
+                candidates.push({
+                    url: src,
+                    source: `social-image:${url}`,
+                    kind: "resource",
+                });
+        }
+    if (new URL(response.finalUrl).pathname === "/")
+        candidates.push({
+            url: new URL("/favicon.ico", response.finalUrl).href,
+            source: `favicon-fallback:${url}`,
+            kind: "resource",
+        });
+    const robots = await http.getRobots(response.finalUrl);
+    observations.push({
+        kind: "crawler-policy",
+        key: url,
+        data: crawlerPolicy(robots, response.finalUrl),
+    });
+    // Real-user data for this page when a Google API key is configured; the
+    // origin record is fetched once by discovery and labelled as fallback.
+    const field = await fieldData({ url: response.finalUrl });
+    if (field)
+        observations.push({ kind: "field-data", key: url, data: field });
+    const declared = page.declarations
+        .filter((l) => /markdown/i.test(l.type ?? "") || /\.md(?:\?|$)/i.test(l.href ?? ""))
+        .map((l) => l.href)
+        .filter(Boolean);
+    const parsed = new URL(response.finalUrl);
+    const appended = new URL(parsed);
+    appended.pathname = parsed.pathname.endsWith("/")
+        ? `${parsed.pathname}index.md`
+        : `${parsed.pathname}.md`;
+    const replaced = new URL(parsed);
+    replaced.pathname = /\.html?$/.test(parsed.pathname)
+        ? parsed.pathname.replace(/\.html?$/, ".md")
+        : parsed.pathname.endsWith("/")
+            ? `${parsed.pathname}index.html.md`
+            : `${parsed.pathname}/index.md`;
+    // Declared links are always checked; negotiation and suffix guesses only
+    // on the homepage, a sample, or once the site has shown Markdown support.
+    const attempts = probeMarkdown
+        ? [
+            ...new Set([...declared.slice(0, 2), appended.href, replaced.href]),
+        ].slice(0, 3)
+        : declared.slice(0, 1);
+    const alternatives = [];
+    if (probeMarkdown)
+        alternatives.push({
+            url: response.finalUrl,
+            method: "accept-header",
+            response: await http.get(response.finalUrl, {
+                accept: "text/markdown",
+                retry: false,
+            }),
+        });
+    observations[0].data.markdownProbe = probeMarkdown
+        ? "full"
+        : attempts.length
+            ? "declared-only"
+            : "skipped";
+    if (!alternatives.some((a) => markdownEvidence(a.response.body, a.response.headers["content-type"])
+        .usable)) {
+        for (const candidate of attempts) {
+            const alternative = await http.get(candidate, { retry: false });
+            alternatives.push({
+                url: candidate,
+                method: declared.includes(candidate) ? "declared-link" : "suffix-probe",
+                response: alternative,
+            });
+            if (alternative.outcome === "ok" &&
+                markdownEvidence(alternative.body, alternative.headers["content-type"])
+                    .usable)
+                break;
+        }
+    }
+    for (const alternative of alternatives) {
+        const evidence = markdownEvidence(alternative.response.body, alternative.response.headers["content-type"]);
+        const usable = alternative.response.outcome === "ok" &&
+            evidence.usable &&
+            !alternative.response.truncated;
+        observations.push({
+            kind: "markdown",
+            key: `${url}|${alternative.method}|${alternative.url}`,
+            data: {
+                pageUrl: url,
+                method: alternative.method,
+                http: httpEvidence(alternative.response),
+                ...evidence,
+                usable,
+                comparison: usable
+                    ? {
+                        method: "token-set Jaccard; heuristic, not semantic equivalence",
+                        similarity: textSimilarity(page.mainText, alternative.response.body),
+                        htmlTextCharacters: page.mainText.length,
+                        markdownCharacters: alternative.response.body.length,
+                    }
+                    : null,
+            },
+        });
+        if (usable)
+            for (const link of markdownLinks(alternative.response.body, alternative.response.finalUrl))
+                if (link.url) {
+                    candidates.push(...linkCandidates(link.url, scan, `markdown:${url}`));
+                }
+    }
+    return { observations, candidates };
+}
+export async function collectResource(url, http, scan) {
+    const response = await http.get(url, {
+        maxBytes: http.options.maxResourceBytes,
+    });
+    const contentType = response.headers["content-type"] ?? "";
+    const data = resourceEvidence(response);
+    const observations = [
+        { kind: "resource", key: url, data },
+    ];
+    const candidates = [];
+    // A file link that turns out to be an HTML page of the scanned site is
+    // crawled as a page after all.
+    if (scan &&
+        data.html &&
+        sameSite(url, scan.url) &&
+        crawlUrl(url, scan) === url)
+        candidates.push({
+            url: crawlUrl(url, scan),
+            source: `html-resource:${url}`,
+            kind: "page",
+        });
+    if (response.outcome === "ok" &&
+        !response.truncated &&
+        (/(?:xml|rss|atom)/i.test(contentType) ||
+            /<(rss|feed)[\s>]/i.test(response.body))) {
+        try {
+            const valid = !/<!DOCTYPE|<!ENTITY/i.test(response.body) &&
+                XMLValidator.validate(response.body) === true;
+            const xml = valid ? xmlParser.parse(response.body) : null;
+            if (xml?.rss || xml?.feed) {
+                const entries = [
+                    xml.rss?.channel?.item ?? xml.feed?.entry ?? [],
+                ].flat();
+                const links = entries.flatMap((entry) => {
+                    const values = [entry.link ?? []].flat();
+                    return values
+                        .map((l) => resolveLink(typeof l === "string" ? l : l["@_href"], url))
+                        .filter(Boolean);
+                });
+                observations.push({
+                    kind: "feed",
+                    key: url,
+                    data: {
+                        valid,
+                        format: xml.rss ? "rss" : "atom",
+                        entries: entries.length,
+                        links,
+                    },
+                });
+                for (const link of links)
+                    if (link) {
+                        candidates.push(...(scan
+                            ? linkCandidates(link, scan, `feed:${url}`)
+                            : [
+                                {
+                                    url: link,
+                                    source: `feed:${url}`,
+                                    kind: "resource",
+                                },
+                            ]));
+                    }
+            }
+            else
+                observations.push({
+                    kind: "xml-resource",
+                    key: url,
+                    data: { valid, recognizedFeed: false },
+                });
+        }
+        catch (error) {
+            observations.push({
+                kind: "xml-resource",
+                key: url,
+                data: { valid: false, error: String(error) },
+            });
+        }
+    }
+    return { observations, candidates };
+}

package/dist/engine/crawl-scope.d.ts

@@ -0,0 +1,22 @@
+import type { Candidate, Scan } from "./types.js";
+export declare const assetExtension: RegExp;
+export declare function isPaginationQuery(url: string): boolean;
+export declare function crawlUrl(href: string, scan: Pick<Scan, "url" | "options">): string;
+export declare function linkCandidates(href: string, scan: Pick<Scan, "url" | "options">, source: string): Candidate[];
+export type ScopeSkip = "url-too-long" | "path-too-deep" | "repeating-path" | "external";
+export declare function scopeRule(url: string): ScopeSkip | null;
+export declare function candidateKind(href: string, scan: Pick<Scan, "url" | "options">): "page" | "resource";
+export declare function pageType(url: string, page?: {
+    structuredTypes?: string[];
+    pagination?: {
+        next: string | null;
+        numbered: string[];
+    } | null;
+    productLinks?: number;
+}): "home" | "product" | "collection" | "article" | "search" | "page";
+export declare function templateGroup(url: string): string;
+export declare function socialPlatform(url: string): {
+    platform: string;
+    blocksBots: boolean;
+} | null;
+export declare function isMapLink(url: string): boolean;

package/dist/engine/crawl-scope.js

@@ -0,0 +1,145 @@
+import { sameSite } from "./network.js";
+// Which discovered URLs are crawled as pages, checked as files, or recorded
+// but not followed. Count caps are optional; these rules are what keeps the
+// crawl finite on sites with infinite URL spaces.
+export const assetExtension = /\.(pdf|zip|docx?|xlsx?|pptx?|csv|jpe?g|png|gif|svg|webp|avif|ico|woff2?|ttf|otf|mp4|webm|mp3|css|js|mjs|xml|json|txt|md|rss|atom)$/i;
+// Query parameters that only carry campaign or session identity.
+const trackingParam = /^(utm_\w+|gclid|dclid|fbclid|msclkid|yclid|_ga|_gl|mc_cid|mc_eid|phpsessid|jsessionid|sessionid|session_id)$/i;
+// Parameters that page through one listing.
+const paginationParam = /^(page|p|pg|paged|pagina|pagination|seite|side|start|offset|from)$/i;
+export function isPaginationQuery(url) {
+    const params = [...new URL(url).searchParams.keys()];
+    return params.length > 0 && params.every((k) => paginationParam.test(k));
+}
+// Same URL space without campaign or session parameters, with parameters
+// sorted so the same page is discovered once. Only applied to pages of the
+// scanned site; external destinations are checked exactly as linked.
+export function crawlUrl(href, scan) {
+    let url;
+    try {
+        url = new URL(href);
+    }
+    catch {
+        return href;
+    }
+    if (!sameSite(url.href, scan.url))
+        return href;
+    const kept = [...url.searchParams]
+        .filter(([k]) => !trackingParam.test(k))
+        .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+    url.search = kept.length ? `?${new URLSearchParams(kept).toString()}` : "";
+    return url.href;
+}
+// Keep page deduplication separate from destination verification: a tracking
+// variant can return a broken response even when its clean page works.
+export function linkCandidates(href, scan, source) {
+    const original = href;
+    const target = crawlUrl(href, scan);
+    const candidates = [
+        { url: target, source, kind: candidateKind(target, scan) },
+    ];
+    if (original !== target)
+        candidates.push({ url: original, source, kind: "resource" });
+    return candidates;
+}
+// Rules that never need the rest of the crawl: URL shape alone.
+export function scopeRule(url) {
+    if (url.length > 2000)
+        return "url-too-long";
+    const segments = new URL(url).pathname.split("/").filter(Boolean);
+    if (segments.length > 12)
+        return "path-too-deep";
+    const counts = new Map();
+    for (const s of segments)
+        counts.set(s, (counts.get(s) ?? 0) + 1);
+    if ([...counts.values()].some((n) => n >= 3))
+        return "repeating-path";
+    return null;
+}
+export function candidateKind(href, scan) {
+    const url = crawlUrl(href, scan);
+    return sameSite(url, scan.url) && !assetExtension.test(new URL(url).pathname)
+        ? "page"
+        : "resource";
+}
+// Coarse page type from URL shape and page evidence; used to compare pages
+// with their peers, never to infer business intent.
+export function pageType(url, page) {
+    const parsed = new URL(url);
+    if (parsed.pathname === "/" && !parsed.search)
+        return "home";
+    const types = page?.structuredTypes ?? [];
+    if (types.some((t) => /^Product(Group)?$/.test(t)))
+        return "product";
+    if (types.some((t) => /^(Article|NewsArticle|BlogPosting)$/.test(t)))
+        return "article";
+    if (types.some((t) => /^(CollectionPage|ItemList|SearchResultsPage)$/.test(t)))
+        return "collection";
+    if (/^\/(search|zoeken|suche|recherche)(\/|$)/i.test(parsed.pathname))
+        return "search";
+    if (/\/(products?|producten|produkte|produits|item|artikel)\/[^/]+\/?$/i.test(parsed.pathname))
+        return "product";
+    if (/\/(collections?|categor(y|ies|ie|ieen)|shop|kategorie|catalog(ue)?|tags?)(\/|$)/i.test(parsed.pathname) ||
+        page?.pagination?.next ||
+        (page?.pagination?.numbered.length ?? 0) > 0 ||
+        (page?.productLinks ?? 0) >= 6)
+        return "collection";
+    if (/\/(blog|news|nieuws|artikelen|articles|posts?|journal|magazine)\/[^/]+/i.test(parsed.pathname))
+        return "article";
+    return "page";
+}
+// URL template group: sibling pages under the same parent path.
+export function templateGroup(url) {
+    const path = new URL(url).pathname;
+    const parts = path.split("/").filter(Boolean);
+    return parts.length < 2 ? path : parts.slice(0, -1).join("/") + "/*";
+}
+// Public profile platforms and map services linked from pages. Platforms
+// that block automated requests make a link unverifiable, not broken.
+const socialPlatforms = [
+    [/(^|\.)facebook\.com$/, "Facebook", true],
+    [/(^|\.)instagram\.com$/, "Instagram", true],
+    [/(^|\.)linkedin\.com$/, "LinkedIn", true],
+    [/(^|\.)(x|twitter)\.com$/, "X", true],
+    [/(^|\.)youtube\.com$/, "YouTube", false],
+    [/(^|\.)tiktok\.com$/, "TikTok", true],
+    [/(^|\.)pinterest\.[a-z.]+$/, "Pinterest", true],
+    [/(^|\.)threads\.(net|com)$/, "Threads", true],
+    [/(^|\.)github\.com$/, "GitHub", false],
+    [/(^|\.)vimeo\.com$/, "Vimeo", false],
+    [/(^|\.)bsky\.app$/, "Bluesky", false],
+    [/(^|\.)wa\.me$/, "WhatsApp", true],
+    [/(^|\.)snapchat\.com$/, "Snapchat", true],
+];
+const mapHosts = /(^|\.)(maps\.google\.[a-z.]+|google\.[a-z.]+|maps\.app\.goo\.gl|goo\.gl|openstreetmap\.org|maps\.apple\.com|waze\.com)$/;
+// Profile addresses only: one path segment (two for /in/, /company/,
+// /channel/, /user/ and /c/ forms). Deeper paths are posts or repositories.
+export function socialPlatform(url) {
+    try {
+        const parsed = new URL(url);
+        const host = parsed.hostname.replace(/^www\./, "");
+        const match = socialPlatforms.find(([pattern]) => pattern.test(host));
+        if (!match)
+            return null;
+        const segments = parsed.pathname.split("/").filter(Boolean);
+        const profile = segments.length === 1 ||
+            (segments.length === 2 &&
+                /^(in|company|school|channel|user|c|pages|groups)$/.test(segments[0]));
+        return profile ? { platform: match[1], blocksBots: match[2] } : null;
+    }
+    catch {
+        return null;
+    }
+}
+export function isMapLink(url) {
+    try {
+        const parsed = new URL(url);
+        return (mapHosts.test(parsed.hostname) &&
+            (/maps/.test(parsed.hostname) ||
+                /\/maps/.test(parsed.pathname) ||
+                /goo\.gl/.test(parsed.hostname)));
+    }
+    catch {
+        return false;
+    }
+}

package/dist/engine/env.d.ts

@@ -0,0 +1 @@
+export declare function env(name: string): string | undefined;

package/dist/engine/env.js

@@ -0,0 +1,3 @@
+export function env(name) {
+    return process.env[name];
+}