@fulldotdev/scan 0.1.0

package/dist/engine/analysis.js

@@ -0,0 +1,747 @@
+import { crawlUrl, isMapLink, pageType, socialPlatform, templateGroup, } from "./crawl-scope.js";
+import { detectLanguage, languageLinks, languageOfTag } from "./language.js";
+import { structuredTypes } from "./html.js";
+import { compareOffers, entities, offerFacts, richResultChecks, } from "./structured-data.js";
+import { sameSite } from "./network.js";
+import { setImmediate } from "node:timers/promises";
+const vagueAnchor = /^(click here|here|read more|more|learn more|details|link|this|this page|view|see more|go|continue|lees meer|meer|klik hier|hier|bekijk|mehr|weiterlesen|hier klicken|en savoir plus|lire la suite|ici|leer más|más|ver más)\.?$/i;
+const digitsOf = (phone) => phone.replace(/\D/g, "").replace(/^00/, "");
+// FNV-1a over a string, then a cheap per-seed mix, for MinHash signatures.
+function fnv(text) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < text.length; i++) {
+        h ^= text.charCodeAt(i);
+        h = Math.imul(h, 0x01000193) >>> 0;
+    }
+    return h;
+}
+const seeds = Array.from({ length: 64 }, (_, i) => fnv(`seed-${i}`));
+function signature(tokens) {
+    const shingles = new Set();
+    for (let i = 0; i + 2 < tokens.length; i++)
+        shingles.add(`${tokens[i]} ${tokens[i + 1]} ${tokens[i + 2]}`);
+    if (!shingles.size)
+        for (const t of tokens)
+            shingles.add(t);
+    const bases = [...shingles].map(fnv);
+    return seeds.map((seed) => {
+        let min = 0xffffffff;
+        for (const base of bases) {
+            const h = Math.imul(base ^ seed, 0x9e3779b1) >>> 0;
+            if (h < min)
+                min = h;
+        }
+        return min;
+    });
+}
+export async function analyze(scan, records, urls) {
+    // The analysis job shares the worker's job time limit. Pairwise phases
+    // stop at this budget and the coverage says so, rather than pretending.
+    const started = Date.now();
+    const budgetMs = 240000;
+    const outOfTime = () => Date.now() - started > budgetMs;
+    // HTML pages carry the content analysis; every page record, including
+    // broken, blocked and non-HTML answers, feeds the HTTP lookups so link,
+    // canonical, hreflang and pagination targets keep their observed status.
+    const pageRecords = records.filter((r) => r.kind === "page");
+    const pages = pageRecords.filter((r) => r.data.html !== false);
+    const byUrl = new Map();
+    for (const p of pages) {
+        byUrl.set(p.key, p.data);
+        byUrl.set(p.data.http.finalUrl, p.data);
+    }
+    const httpByUrl = new Map();
+    for (const p of pageRecords) {
+        httpByUrl.set(p.key, p.data.http);
+        if (p.data.http?.finalUrl)
+            httpByUrl.set(p.data.http.finalUrl, p.data.http);
+    }
+    const rendered = new Map(records.filter((r) => r.kind === "rendered").map((r) => [r.key, r.data]));
+    const resources = new Map(records.filter((r) => r.kind === "resource").map((r) => [r.key, r.data]));
+    // Observed HTTP outcome of any URL the crawl touched, page or resource.
+    const httpOf = (url) => url ? (httpByUrl.get(url) ?? resources.get(url)?.http ?? null) : null;
+    const statusOf = (url) => httpOf(url)?.status ?? null;
+    const outcomeOf = (url) => httpOf(url)?.outcome ?? null;
+    // Link graph: every internal link from original and rendered HTML,
+    // separated into navigation (header, nav, footer, aside) and content.
+    const inlinks = new Map();
+    const originalInlinks = new Set();
+    const contextualInlinks = new Map();
+    const outlinks = new Map();
+    const anchors = [];
+    const vague = [];
+    for (const page of pages) {
+        const renderedLinks = rendered.get(page.key)?.links ?? [];
+        const consider = (link, fromRendered) => {
+            if (!link.href)
+                return;
+            const targetUrl = crawlUrl(link.href, scan);
+            if (!sameSite(targetUrl, scan.url))
+                return;
+            if (targetUrl === page.key || targetUrl === page.data.http.finalUrl)
+                return;
+            if (!fromRendered)
+                originalInlinks.add(targetUrl);
+            (inlinks.get(targetUrl) ??
+                inlinks.set(targetUrl, new Set()).get(targetUrl)).add(page.key);
+            if (link.location === "content")
+                (contextualInlinks.get(targetUrl) ??
+                    contextualInlinks.set(targetUrl, new Set()).get(targetUrl)).add(page.key);
+            (outlinks.get(page.key) ??
+                outlinks.set(page.key, new Set()).get(page.key)).add(targetUrl);
+        };
+        for (const link of page.data.links) {
+            consider(link, false);
+            if (link.href &&
+                link.location === "content" &&
+                (vagueAnchor.test(link.name ?? link.text ?? "") ||
+                    !(link.name ?? link.text)))
+                vague.push({
+                    source: page.key,
+                    target: link.href,
+                    text: link.name ?? link.text ?? "",
+                });
+            if (link.fragment && link.href) {
+                const targetUrl = crawlUrl(link.href, scan);
+                const target = byUrl.get(targetUrl) ??
+                    resources.get(targetUrl)?.html ??
+                    resources.get(link.href)?.html;
+                let fragment = link.fragment;
+                try {
+                    fragment = decodeURIComponent(fragment);
+                }
+                catch { }
+                anchors.push({
+                    source: page.key,
+                    target: link.href,
+                    scanTarget: targetUrl,
+                    fragment,
+                    present: target ? target.ids.includes(fragment) : null,
+                    coverage: target ? "checked" : "target-not-inspected",
+                });
+            }
+        }
+        for (const link of renderedLinks)
+            consider(link, true);
+    }
+    const home = pages.find((p) => p.key === scan.url) ??
+        pages.find((p) => new URL(p.key).pathname === "/");
+    const depth = new Map();
+    if (home) {
+        depth.set(home.key, 0);
+        depth.set(home.data.http.finalUrl, 0);
+    }
+    const queue = home ? [home.key, home.data.http.finalUrl] : [];
+    for (let i = 0; i < queue.length; i++) {
+        const from = queue[i];
+        for (const to of outlinks.get(from) ?? [])
+            if (!depth.has(to)) {
+                depth.set(to, depth.get(from) + 1);
+                queue.push(to);
+            }
+    }
+    const productLinkCount = (page) => page.links.filter((l) => l.href &&
+        l.location === "content" &&
+        /\/(products?|producten|produkte|produits|item|artikel|p)\/[^/]+/i.test(new URL(l.href).pathname)).length;
+    const typeOf = new Map();
+    for (const page of pages) {
+        const types = (page.data.structuredData ?? []).flatMap((s) => s.parseValid ? structuredTypes(s.data) : []);
+        typeOf.set(page.key, pageType(page.key, {
+            structuredTypes: types,
+            pagination: page.data.pagination ?? rendered.get(page.key)?.pagination ?? null,
+            productLinks: productLinkCount(page.data),
+        }));
+    }
+    const graph = urls
+        .filter((u) => u.kind === "page")
+        .map((u) => {
+        const page = byUrl.get(u.url);
+        const inSitemap = u.sources.some((s) => s.startsWith("sitemap:"));
+        return {
+            url: u.url,
+            sources: u.sources,
+            scheduled: u.scheduled,
+            skipReason: u.skipReason ?? null,
+            status: page?.http?.status ?? statusOf(u.url),
+            outcome: page?.http?.outcome ?? outcomeOf(u.url),
+            type: typeOf.get(u.url) ?? (page ? "page" : null),
+            group: templateGroup(u.url),
+            language: page?.language ?? null,
+            detectedLanguage: page ? detectLanguage(page.mainText ?? "") : null,
+            inlinkCount: inlinks.get(u.url)?.size ?? 0,
+            contextualInlinkCount: contextualInlinks.get(u.url)?.size ?? 0,
+            outlinkCount: outlinks.get(u.url)?.size ?? 0,
+            clickDepth: depth.get(u.url) ?? null,
+            inSitemap,
+            linkedFromSite: (inlinks.get(u.url)?.size ?? 0) > 0,
+            // Linked only by markup that JavaScript added.
+            renderedOnly: inlinks.has(u.url) && !originalInlinks.has(u.url),
+            orphanCandidate: inSitemap && !inlinks.has(u.url) && u.url !== home?.key,
+        };
+    });
+    // Peer comparison: pages of one template group and type with far fewer
+    // contextual incoming links than the group's median.
+    const peerGroups = new Map();
+    for (const row of graph)
+        if (row.status === 200 && row.type && row.type !== "home") {
+            const key = `${row.type}|${row.group}`;
+            peerGroups.set(key, [...(peerGroups.get(key) ?? []), row]);
+        }
+    const fewContextual = [];
+    for (const [key, rows] of peerGroups) {
+        if (rows.length < 5)
+            continue;
+        const sorted = rows
+            .map((r) => r.contextualInlinkCount)
+            .sort((a, b) => a - b);
+        const median = sorted[Math.floor(sorted.length / 2)];
+        if (median < 2)
+            continue;
+        for (const row of rows)
+            if (row.contextualInlinkCount === 0)
+                fewContextual.push({
+                    url: row.url,
+                    group: key,
+                    peers: rows.length,
+                    peerMedian: median,
+                    contextualInlinks: 0,
+                    navigationInlinks: row.inlinkCount,
+                });
+    }
+    // Exact duplicates, then near-duplicates through MinHash candidate
+    // selection and exact word-set Jaccard on the candidates.
+    const duplicates = [];
+    for (const field of ["title", "description", "textHash"]) {
+        const groups = new Map();
+        for (const page of pages) {
+            const value = field === "description"
+                ? page.data.descriptions.join("|")
+                : page.data[field];
+            if (value)
+                groups.set(value, [...(groups.get(value) ?? []), page.key]);
+        }
+        for (const [value, urls] of groups)
+            if (urls.length > 1)
+                duplicates.push({
+                    field,
+                    value: field === "textHash" ? value : value.slice(0, 1000),
+                    urls,
+                });
+    }
+    const eligible = pages.filter((p) => (p.data.wordCount ?? 0) >= 50);
+    const tokenLists = eligible.map((page) => page.data.mainText
+        .slice(0, 20000)
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}\s]/gu, " ")
+        .split(/\s+/)
+        .filter(Boolean)
+        .slice(0, 4000));
+    const tokenSets = tokenLists.map((t) => new Set(t));
+    // Band buckets per page: bucketKeys[i] names the 16 buckets page i is in.
+    const buckets = new Map();
+    const bucketKeys = [];
+    for (let i = 0; i < eligible.length; i++) {
+        const sig = signature(tokenLists[i]);
+        const keys = [];
+        for (let band = 0; band < 16; band++) {
+            const key = `${band}:${sig.slice(band * 4, band * 4 + 4).join(",")}`;
+            keys.push(key);
+            const members = buckets.get(key);
+            if (members)
+                members.push(i);
+            else
+                buckets.set(key, [i]);
+        }
+        bucketKeys.push(keys);
+        if (i % 50 === 0)
+            await setImmediate();
+    }
+    const nearDuplicates = [];
+    let comparisons = 0;
+    let candidatePairs = 0;
+    let pagesCompared = 0;
+    // Sampled text is a known bound; running out of time is reported as such.
+    const sampled = pages.some((p) => p.data.mainTextTotalLength > 20000);
+    let limited = false;
+    let limitReason = null;
+    // Candidates are enumerated per left page (bucket members with a higher
+    // index), so memory stays at one page's candidate set and the time budget
+    // is checked as the work proceeds.
+    left: for (let a = 0; a < eligible.length; a++) {
+        if (outOfTime()) {
+            limited = true;
+            limitReason = "analysis time budget reached during text comparison";
+            break;
+        }
+        const candidates = new Set();
+        for (const key of bucketKeys[a])
+            for (const b of buckets.get(key))
+                if (b > a)
+                    candidates.add(b);
+        candidatePairs += candidates.size;
+        pagesCompared++;
+        for (const b of candidates) {
+            if (comparisons % 200 === 0) {
+                await setImmediate();
+                if (outOfTime()) {
+                    limited = true;
+                    limitReason = "analysis time budget reached during text comparison";
+                    break left;
+                }
+            }
+            if (eligible[a].data.textHash === eligible[b].data.textHash)
+                continue;
+            comparisons++;
+            const one = tokenSets[a], two = tokenSets[b];
+            let intersection = 0;
+            for (const token of one)
+                if (two.has(token))
+                    intersection++;
+            const similarity = intersection / (one.size + two.size - intersection || 1);
+            if (similarity >= 0.9) {
+                const onlyLeft = [...one].filter((t) => !two.has(t));
+                const onlyRight = [...two].filter((t) => !one.has(t));
+                nearDuplicates.push({
+                    urls: [eligible[a].key, eligible[b].key],
+                    similarity,
+                    sameGroup: templateGroup(eligible[a].key) === templateGroup(eligible[b].key),
+                    differences: {
+                        onlyLeft: onlyLeft.slice(0, 12),
+                        onlyRight: onlyRight.slice(0, 12),
+                    },
+                    // Few differing words between otherwise identical texts: the
+                    // pattern of a place or product name swapped into a template.
+                    substitution: onlyLeft.length > 0 &&
+                        onlyRight.length > 0 &&
+                        onlyLeft.length <= 8 &&
+                        onlyRight.length <= 8,
+                    method: "token-set Jaccard >=0.9 on MinHash candidates, minimum 50 words; heuristic, bounded text sample",
+                });
+            }
+        }
+    }
+    // Collections listing the same items: content links of collection pages
+    // in the same template group.
+    const collections = pages.filter((p) => typeOf.get(p.key) === "collection");
+    const itemSets = new Map();
+    for (const page of collections) {
+        const set = new Set();
+        for (const link of [
+            ...page.data.links,
+            ...(rendered.get(page.key)?.links ?? []),
+        ])
+            if (link.href && link.location === "content") {
+                const target = crawlUrl(link.href, scan);
+                if (sameSite(target, scan.url) && typeOf.get(target) === "product")
+                    set.add(target);
+            }
+        if (set.size >= 4)
+            itemSets.set(page.key, set);
+    }
+    const overlappingCollections = [];
+    let collectionComparisons = 0;
+    const collectionKeys = [...itemSets.keys()];
+    outer: for (let a = 0; a < collectionKeys.length; a++)
+        for (let b = a + 1; b < collectionKeys.length; b++) {
+            if (collectionComparisons++ % 500 === 0) {
+                await setImmediate();
+                if (outOfTime()) {
+                    limited = true;
+                    limitReason ??=
+                        "analysis time budget reached during collection comparison";
+                    break outer;
+                }
+            }
+            const one = itemSets.get(collectionKeys[a]), two = itemSets.get(collectionKeys[b]);
+            let shared = 0;
+            for (const item of one)
+                if (two.has(item))
+                    shared++;
+            const overlap = shared / Math.max(one.size, two.size);
+            if (overlap >= 0.9)
+                overlappingCollections.push({
+                    urls: [collectionKeys[a], collectionKeys[b]],
+                    items: [one.size, two.size],
+                    shared,
+                    overlap: Math.round(overlap * 100) / 100,
+                    method: "shared product links in content, largest-set ratio >=0.9",
+                });
+        }
+    // Language: hreflang reciprocity and reachability, language-switch links
+    // and the written language of each page.
+    const hreflangs = pages.flatMap((page) => page.data.hreflangs.map((link) => {
+        const targetUrl = link.href ? crawlUrl(link.href, scan) : null;
+        const target = targetUrl ? byUrl.get(targetUrl) : null;
+        let languageValid = false;
+        try {
+            languageValid =
+                link.hreflang === "x-default" || !!new Intl.Locale(link.hreflang);
+        }
+        catch { }
+        return {
+            source: page.key,
+            href: link.href,
+            hreflang: link.hreflang,
+            scanTarget: targetUrl,
+            languageValid,
+            targetObserved: !!target,
+            targetStatus: statusOf(targetUrl),
+            targetOutcome: outcomeOf(targetUrl),
+            targetLanguage: target?.language ?? null,
+            targetLanguageMatches: target && link.hreflang !== "x-default"
+                ? languageOfTag(target.language) === languageOfTag(link.hreflang)
+                : null,
+            reciprocal: target
+                ? target.hreflangs.some((h) => {
+                    const href = h.href ? crawlUrl(h.href, scan) : null;
+                    return href === page.key || href === page.data.http.finalUrl;
+                })
+                : null,
+        };
+    }));
+    const switches = new Map();
+    for (const page of pages)
+        for (const link of languageLinks(page.data.links)) {
+            const targetUrl = crawlUrl(link.url, scan);
+            if (!sameSite(targetUrl, scan.url) ||
+                switches.has(`${targetUrl}|${link.expected}`))
+                continue;
+            const target = byUrl.get(targetUrl);
+            switches.set(`${targetUrl}|${link.expected}`, {
+                source: page.key,
+                url: targetUrl,
+                expected: link.expected,
+                text: link.text,
+                status: statusOf(targetUrl),
+                outcome: outcomeOf(targetUrl),
+                targetLanguage: target?.language ?? null,
+                declaredMatches: target
+                    ? languageOfTag(target.language) === link.expected
+                    : null,
+                detectedLanguage: target ? detectLanguage(target.mainText ?? "") : null,
+            });
+        }
+    const canonicals = pages.flatMap((page) => page.data.canonicals.map((link) => {
+        const scanTarget = link.href ? crawlUrl(link.href, scan) : null;
+        const target = (scanTarget ? byUrl.get(scanTarget) : null) ??
+            (scanTarget ? resources.get(scanTarget) : null) ??
+            resources.get(link.href);
+        return {
+            source: page.key,
+            target: link.href,
+            scanTarget,
+            targetHttp: target?.http ?? httpOf(scanTarget) ?? null,
+            targetRobots: target?.robots ?? target?.html?.robots ?? null,
+            targetXRobots: target?.xRobotsTag ?? target?.html?.xRobotsTag ?? null,
+        };
+    }));
+    // Pagination: next targets, canonical relationships, indexing directives
+    // and whether the paginated URLs exist without JavaScript.
+    const noindexOf = (page) => /\b(noindex|none)\b/i.test([
+        page.xRobotsTag,
+        ...(page.robots ?? [])
+            .filter((r) => /^(robots|googlebot)$/i.test(r.name ?? ""))
+            .map((r) => r.content),
+    ].join(","));
+    const paginated = pages.flatMap((page) => {
+        const original = page.data.pagination;
+        const fromRendered = rendered.get(page.key)?.pagination;
+        const pagination = original ?? fromRendered;
+        if (!pagination)
+            return [];
+        const next = pagination.next ? crawlUrl(pagination.next, scan) : null;
+        const canonical = page.data.canonicals[0]?.href
+            ? crawlUrl(page.data.canonicals[0].href, scan)
+            : null;
+        const self = page.data.http.finalUrl;
+        const isFirst = !pagination.prev && !/[?&](page|p|pg|paged|pagina)=(\d+)/.test(self);
+        const numbered = (pagination.numbered ?? []).map((u) => crawlUrl(u, scan));
+        return [
+            {
+                url: page.key,
+                type: typeOf.get(page.key),
+                next,
+                prev: pagination.prev ? crawlUrl(pagination.prev, scan) : null,
+                numbered,
+                source: pagination.source,
+                renderedOnly: !original && !!fromRendered,
+                nextStatus: statusOf(next),
+                nextOutcome: outcomeOf(next),
+                nextObserved: !!next && httpOf(next) !== null,
+                numberedBroken: numbered.filter((u) => (statusOf(u) ?? 0) >= 400),
+                canonical,
+                canonicalRelation: !canonical
+                    ? "none"
+                    : canonical === self || canonical === page.key
+                        ? "self"
+                        : !isFirst &&
+                            numbered.includes(canonical) &&
+                            !/[?&](page|p|pg|paged|pagina)=/.test(canonical)
+                            ? "first-page"
+                            : "other",
+                noindex: noindexOf(page.data),
+                isFirst,
+                loadMore: page.data.loadMore ?? [],
+            },
+        ];
+    });
+    // Structured data per page: entities, documented rich-result properties
+    // and the declared offers against the visible page.
+    const structured = pages.flatMap((page) => {
+        const original = entities(page.data.structuredData ?? []);
+        const renderedBlocks = rendered.get(page.key)?.structuredData ?? [];
+        const renderedEntities = entities(renderedBlocks);
+        const found = original.length ? original : renderedEntities;
+        if (!found.length && !(page.data.structuredData ?? []).length)
+            return [];
+        const evidence = rendered.get(page.key);
+        const products = offerFacts(found);
+        return [
+            {
+                url: page.key,
+                types: [...new Set(found.map((e) => e.type))],
+                source: original.length
+                    ? "original-html"
+                    : renderedEntities.length
+                        ? "rendered-only"
+                        : "none",
+                parseErrors: (page.data.structuredData ?? []).filter((s) => s.parseValid === false).length,
+                structuralErrors: (page.data.structuredData ?? [])
+                    .flatMap((s) => s.structuralErrors ?? [])
+                    .slice(0, 20),
+                richResults: richResultChecks(found),
+                offers: compareOffers(products, {
+                    prices: evidence?.prices?.length
+                        ? evidence.prices
+                        : (page.data.prices ?? []),
+                    availability: evidence?.availability ??
+                        page.data.availability ?? {
+                        inStock: false,
+                        outOfStock: false,
+                        inStockText: null,
+                        outOfStockText: null,
+                    },
+                    headings: evidence?.headings ?? page.data.headings ?? [],
+                    title: page.data.title ?? "",
+                }),
+                visibleSource: evidence?.prices?.length
+                    ? "rendered-dom"
+                    : "original-html",
+            },
+        ];
+    });
+    // Business and social identity: what the site says about itself, where it
+    // says it, and whether its public profiles answer.
+    const organizations = pages.flatMap((page) => entities(page.data.structuredData ?? [])
+        .filter((e) => e.types.some((t) => /(Organization|LocalBusiness|Store|Restaurant|Dentist|Physician|MedicalBusiness|ProfessionalService|Corporation|Hotel|Attorney|RealEstateAgent|AutoRepair|Gym|HealthClub|Person)$/.test(t)))
+        .map((e) => ({ page: page.key, type: e.type, node: e.node })));
+    const asList = (v) => (Array.isArray(v) ? v : v ? [v] : []);
+    const declaredNames = new Map();
+    const declaredPhones = new Map();
+    const declaredAddresses = new Map();
+    const declaredHours = new Map();
+    const sameAs = new Map();
+    const addTo = (map, key, page) => (map.get(key) ?? map.set(key, new Set()).get(key)).add(page);
+    for (const org of organizations) {
+        if (typeof org.node.name === "string")
+            addTo(declaredNames, org.node.name.trim(), org.page);
+        for (const phone of asList(org.node.telephone))
+            if (typeof phone === "string")
+                addTo(declaredPhones, phone, org.page);
+        for (const address of asList(org.node.address)) {
+            const text = typeof address === "string"
+                ? address
+                : [
+                    address?.streetAddress,
+                    address?.postalCode,
+                    address?.addressLocality,
+                    address?.addressCountry,
+                ]
+                    .filter(Boolean)
+                    .join(", ");
+            if (text)
+                addTo(declaredAddresses, text, org.page);
+        }
+        for (const hours of [
+            ...asList(org.node.openingHours),
+            ...asList(org.node.openingHoursSpecification),
+        ])
+            addTo(declaredHours, typeof hours === "string"
+                ? hours
+                : `${asList(hours?.dayOfWeek)
+                    .map((d) => String(d).replace(/^.*\//, ""))
+                    .join(",")} ${hours?.opens ?? ""}-${hours?.closes ?? ""}`, org.page);
+        for (const url of asList(org.node.sameAs))
+            if (typeof url === "string")
+                addTo(sameAs, url, org.page);
+    }
+    const siteNames = new Set();
+    for (const page of pages)
+        for (const m of page.data.social ?? [])
+            if (m.property === "og:site_name" && m.content)
+                siteNames.add(m.content.trim());
+    const visiblePhones = new Map();
+    const visibleEmails = new Map();
+    for (const page of pages) {
+        const contacts = page.data.contacts ?? rendered.get(page.key)?.contacts;
+        for (const phone of contacts?.phones ?? [])
+            addTo(visiblePhones, phone, page.key);
+        for (const email of contacts?.emails ?? [])
+            addTo(visibleEmails, email, page.key);
+    }
+    const profiles = new Map();
+    const maps = new Map();
+    for (const page of pages)
+        for (const link of page.data.links) {
+            if (!link.href)
+                continue;
+            const social = socialPlatform(link.href);
+            if (social) {
+                const entry = profiles.get(link.href) ??
+                    profiles
+                        .set(link.href, {
+                        url: link.href,
+                        platform: social.platform,
+                        blocksBots: social.blocksBots,
+                        pages: new Set(),
+                        locations: new Set(),
+                    })
+                        .get(link.href);
+                entry.pages.add(page.key);
+                entry.locations.add(link.location);
+            }
+            else if (isMapLink(link.href))
+                addTo(maps, link.href, page.key);
+        }
+    const profileRows = [...profiles.values()].map((p) => {
+        const http = resources.get(p.url)?.http;
+        const status = http?.status ?? null;
+        const outcome = http?.outcome ?? null;
+        return {
+            url: p.url,
+            platform: p.platform,
+            pages: p.pages.size,
+            locations: [...p.locations],
+            inSameAs: sameAs.has(p.url),
+            status,
+            outcome,
+            verification: !http
+                ? "not-checked"
+                : outcome === "ok" && !p.blocksBots
+                    ? "ok"
+                    : outcome === "ok"
+                        ? "responded"
+                        : p.blocksBots
+                            ? "unverifiable"
+                            : "broken",
+            note: p.blocksBots
+                ? "This platform answers automated requests with blocks, logins or rate limits; a non-200 answer is not evidence that the profile is missing"
+                : null,
+        };
+    });
+    const platformVariants = new Map();
+    for (const row of profileRows)
+        platformVariants.set(row.platform, [
+            ...(platformVariants.get(row.platform) ?? []),
+            row.url,
+        ]);
+    const normalizedDeclaredPhones = new Set([...declaredPhones.keys()].map(digitsOf));
+    const business = {
+        names: {
+            structured: [...declaredNames].map(([name, pages]) => ({
+                name,
+                pages: pages.size,
+            })),
+            siteName: [...siteNames],
+            homepageTitle: home?.data.title ?? null,
+        },
+        phones: {
+            structured: [...declaredPhones].map(([phone, pages]) => ({
+                phone,
+                pages: pages.size,
+            })),
+            visible: [...visiblePhones]
+                .map(([phone, pages]) => ({ phone, pages: pages.size }))
+                .slice(0, 30),
+            visibleNotDeclared: [...visiblePhones.keys()]
+                .filter((p) => normalizedDeclaredPhones.size &&
+                !normalizedDeclaredPhones.has(digitsOf(p)))
+                .slice(0, 10),
+            declaredNotVisible: [...declaredPhones.keys()].filter((p) => ![...visiblePhones.keys()].some((v) => digitsOf(v) === digitsOf(p))),
+        },
+        emails: [...visibleEmails]
+            .map(([email, pages]) => ({ email, pages: pages.size }))
+            .slice(0, 20),
+        addresses: [...declaredAddresses].map(([address, pages]) => ({
+            address,
+            pages: pages.size,
+        })),
+        hours: [...declaredHours]
+            .map(([hours, pages]) => ({ hours, pages: pages.size }))
+            .slice(0, 30),
+        maps: [...maps].map(([url, pages]) => ({
+            url,
+            pages: pages.size,
+            status: statusOf(url),
+        })),
+        sameAs: [...sameAs].map(([url, pages]) => ({
+            url,
+            pages: pages.size,
+            linkedOnSite: profiles.has(url),
+            platform: socialPlatform(url)?.platform ?? null,
+        })),
+        profiles: profileRows,
+        platformVariants: [...platformVariants]
+            .filter(([, urls]) => urls.length > 1)
+            .map(([platform, urls]) => ({ platform, urls })),
+        scope: "Public website evidence only: JSON-LD organization data, visible contact details and linked profiles with their HTTP answers. No listing or search services were queried.",
+    };
+    return {
+        observations: [
+            {
+                kind: "link-graph",
+                key: "site",
+                data: {
+                    pages: graph,
+                    fewContextual,
+                    vagueAnchors: vague,
+                    coverage: {
+                        pages: pages.length,
+                        renderedPages: rendered.size,
+                        scope: "observed crawl only; missing links and unknown depth are not proof of an orphan page",
+                    },
+                },
+            },
+            { kind: "anchors", key: "site", data: anchors },
+            {
+                kind: "duplicates",
+                key: "site",
+                data: {
+                    exact: duplicates,
+                    near: nearDuplicates,
+                    collections: overlappingCollections,
+                    coverage: {
+                        pages: eligible.length,
+                        candidatePairs,
+                        pagesCompared,
+                        comparisons,
+                        collectionsCompared: itemSets.size,
+                        collectionComparisons,
+                        maxCharacters: 20000,
+                        maxTokens: 4000,
+                        textSampled: sampled,
+                        method: "MinHash (64 hashes, 16 bands) selects candidate pairs; each candidate is compared exactly",
+                        limited,
+                        limitReason,
+                    },
+                },
+            },
+            { kind: "hreflang-links", key: "site", data: hreflangs },
+            { kind: "language-links", key: "site", data: [...switches.values()] },
+            { kind: "canonical-targets", key: "site", data: canonicals },
+            { kind: "pagination", key: "site", data: paginated },
+            { kind: "structured-data", key: "site", data: structured },
+            { kind: "business-identity", key: "site", data: business },
+        ],
+    };
+}