@brandlab-digital/conformance 3.3.3 → 3.4.1

package/dist/estate.js

@@ -17,10 +17,32 @@
  * by defineBrandlabPageQualityTests: a backend release keeps production's pages
  * as they are, which the comparison harness proves. Counts are always derived
  * from the current catalogue, never a historic constant.
+ *
+ * From 3.4.0 (the Brandlab on-page and location standard) the estate journey
+ * also fails on: a sitemap that is not split by page family once the catalogue
+ * spans families, or whose lastmod is not the route's content date; an
+ * indexable page without a title, meta description or JSON-LD in its server HTML; FAQPage or
+ * HowTo markup, or a rating on the business itself; a homepage without a
+ * WebSite node (name and url); an unknown slug that does not answer 404 to GET
+ * and HEAD; a nearby list on a location page that is not ranked by distance
+ * between suburb centres; and a catalogue that adds indexable location pages
+ * beyond the recorded capacity (`locationCapacity`). The page-quality journey
+ * reports title, heading, description and HTML-size warnings, orphan pages,
+ * links from indexable pages to noindex location pages, centerpiece
+ * similarity (warn at 0.5, review list at 0.7) and the location review list
+ * (location-review.json); pages on that list stay indexed.
  */
 import { request as httpRequest } from 'node:http';
 import { expect, test } from '@playwright/test';
+import { buildLocationReviewList, centerpieceSimilarity, checkLocationCapacity, descriptionLengthProblem, HTML_MAX_BYTES, headingTexts, loadBrandlabGeography, neighbourOrderProblems, parseSitemapEntries, SIMILARITY_REVIEW, SIMILARITY_WARN, sitemapFamilyProblems, sitemapLastmodProblems, structuredDataProblems, textBlocks, titleFormatProblems, titleHeadingProblems, } from './page-standard.js';
 import { extractSearchCertificationHtmlSnapshot } from './search-certification.js';
+const LOCATION_KINDS = new Set(['location', 'service-location']);
+/** A location route's place slug: the last segment of its path. */
+function routePlace(route) {
+    if (!route || !LOCATION_KINDS.has(route.kind ?? ''))
+        return undefined;
+    return normalisePath(route.path).split('/').filter(Boolean).at(-1);
+}
 const BUSINESS_TYPES = new Set([
     'LocalBusiness', 'HomeAndConstructionBusiness', 'GeneralContractor', 'Plumber', 'Electrician',
     'RoofingContractor', 'HousePainter', 'Locksmith', 'MovingCompany', 'HVACBusiness', 'ProfessionalService',
@@ -44,6 +66,8 @@ export function defineBrandlabEstateTests(options) {
     const timeout = options.timeout ?? 240_000;
     const families = options.families ?? [];
     const familyPaths = new Set(families.flatMap((family) => family.paths.map(normalisePath)));
+    const routeByPath = new Map(routes.map((route) => [normalisePath(route.path), route]));
+    const homePath = normalisePath(routes.find((route) => route.kind === 'home')?.path ?? '/');
     test.describe('Brandlab technical estate', () => {
         test('the sitemap lists exactly the indexable catalogue routes', async ({ request }) => {
             test.setTimeout(timeout);
@@ -54,8 +78,27 @@ export function defineBrandlabEstateTests(options) {
             expect(listed.length, 'the sitemap repeats a URL').toBe(new Set(listed).size);
             expect([...listed].sort()).toEqual([...new Set(expected)].sort());
         });
+        test('the sitemap is split by page family and dated by content, never by the build', async ({ request }, testInfo) => {
+            test.setTimeout(timeout);
+            const { index, files } = await readSitemapFiles(request, '/sitemap.xml', origin);
+            const problems = [
+                ...sitemapFamilyProblems({ index, files, routes, origin }),
+                ...sitemapLastmodProblems(files.flatMap((file) => file.entries), routes, origin),
+            ];
+            await testInfo.attach('sitemap-files.json', {
+                body: JSON.stringify(files.map((file) => ({ path: file.path, urls: file.entries.length })), null, 2),
+                contentType: 'application/json',
+            });
+            expect(problems).toEqual([]);
+        });
+        test('location pages stay within the recorded capacity', async ({}, testInfo) => {
+            const result = checkLocationCapacity(routes, options.locationCapacity);
+            testInfo.annotations.push({ type: result.limit === null ? 'warning' : 'location-capacity', description: result.detail });
+            expect(result.blocked, result.detail).toBe(false);
+        });
         test('every catalogue route serves its canonical identity and links only to live pages', async ({ request }, testInfo) => {
             test.setTimeout(timeout);
+            const centreOf = options.centreOf ?? (await loadBrandlabGeography()).centreOf;
             const failures = [];
             const internalLinks = new Map();
             const checked = [];
@@ -93,6 +136,34 @@ export function defineBrandlabEstateTests(options) {
                     failures.push(`${route.path}: ${route.indexing} route is indexable`);
                 for (const problem of breadcrumbListProblems(snapshot.jsonLd))
                     failures.push(`${route.path}: ${problem}`);
+                if (route.indexing === 'index') {
+                    if (!snapshot.title)
+                        failures.push(`${route.path}: no <title> in the server HTML`);
+                    if (!snapshot.description)
+                        failures.push(`${route.path}: no meta description in the server HTML`);
+                    if (snapshot.jsonLd.length === 0)
+                        failures.push(`${route.path}: no JSON-LD in the server HTML`);
+                }
+                if (snapshot.jsonLd.some((payload) => payload.parseError))
+                    failures.push(`${route.path}: invalid JSON-LD`);
+                for (const problem of structuredDataProblems(snapshot.jsonLd, {
+                    origin,
+                    home: normalisePath(route.path) === homePath,
+                    businessId: options.businessId,
+                    business: options.business,
+                }))
+                    failures.push(`${route.path}: ${problem}`);
+                const place = routePlace(route);
+                if (place && centreOf(place)) {
+                    for (const problem of neighbourOrderProblems({
+                        html,
+                        origin: place,
+                        pageUrl: expectedCanonical,
+                        placeOf: (path) => routePlace(routeByPath.get(path)),
+                        centreOf,
+                    }))
+                        failures.push(`${route.path}: ${problem}`);
+                }
                 for (const href of extractLinks(html)) {
                     const path = internalPath(href, expectedCanonical, origin);
                     if (path && !internalLinks.has(path))
@@ -216,9 +287,14 @@ export function defineBrandlabEstateTests(options) {
                 testInfo.annotations.push({ type: 'skipped', description: `${path}: a redirect rule answers it` });
             const failures = [];
             for (const path of paths) {
-                const response = await request.get(path, { maxRedirects: 0 });
-                if (response.status() !== 404)
-                    failures.push(`${path}: ${response.status()} (expected 404)`);
+                const [get, head] = await Promise.all([
+                    request.get(path, { maxRedirects: 0 }),
+                    request.head(path, { maxRedirects: 0 }),
+                ]);
+                if (get.status() !== 404)
+                    failures.push(`${path}: GET ${get.status()} (expected 404)`);
+                if (head.status() !== 404)
+                    failures.push(`${path}: HEAD ${head.status()} (expected 404)`);
             }
             expect(failures).toEqual([]);
         });
@@ -315,6 +391,29 @@ export async function readSitemap(request, path, origin) {
     }
     return nested;
 }
+/**
+ * The sitemap's URL files with their entries: the sitemap itself, or every
+ * file a sitemap index lists (one level, same origin, no redirects).
+ */
+export async function readSitemapFiles(request, path, origin) {
+    const read = async (target) => {
+        const response = await request.get(target, { maxRedirects: 0 });
+        if (response.status() !== 200)
+            throw new Error(`${target} answers ${response.status()}; a sitemap must answer 200 without redirecting.`);
+        return response.text();
+    };
+    const xml = await read(path);
+    if (!/<sitemapindex\b/i.test(xml))
+        return { index: false, files: [{ path, entries: parseSitemapEntries(xml) }] };
+    const files = [];
+    for (const location of extractSitemapLocations(xml)) {
+        const url = new URL(location);
+        if (url.origin !== new URL(origin).origin)
+            throw new Error(`The sitemap index lists ${location}, which is not on ${new URL(origin).origin}.`);
+        files.push({ path: url.pathname, entries: parseSitemapEntries(await read(`${url.pathname}${url.search}`)) });
+    }
+    return { index: true, files };
+}
 export function extractSitemapLocations(xml) {
     return [...xml.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/gi)].map((match) => new URL(decodeXml(match[1])).href);
 }
@@ -424,10 +523,17 @@ export function defineBrandlabPageQualityTests(options) {
     const businessId = options.businessId ?? `${origin}/#business`;
     const timeout = options.timeout ?? 240_000;
     const routes = options.routes.filter((route) => normalisePath(route.path) !== NOT_FOUND_ROUTE);
+    const routeByPath = new Map(routes.map((route) => [normalisePath(route.path), route]));
     test.describe('Brandlab page quality (advisory)', () => {
         test('reports page and structured-data quality without failing', async ({ request }, testInfo) => {
             test.setTimeout(timeout);
             const findings = [];
+            const geography = await loadBrandlabGeography();
+            const centreOf = options.centreOf ?? geography.centreOf;
+            const titles = new Map();
+            const descriptions = new Map();
+            const similarityPages = [];
+            const linksFrom = new Map();
             await eachWithConcurrency(routes, options.concurrency ?? 8, async (route) => {
                 const response = await request.get(route.path, { maxRedirects: 0 });
                 // The estate journey owns status codes; quality is read from the pages that load.
@@ -436,13 +542,35 @@ export function defineBrandlabPageQualityTests(options) {
                 const html = await response.text();
                 const snapshot = extractSearchCertificationHtmlSnapshot(html);
                 const add = (kind, detail) => findings.push({ path: route.path, kind, detail });
-                const headings = html.match(/<h1(?:\s|>)/gi)?.length ?? 0;
-                if (headings !== 1)
-                    add('headings', `${headings} h1 headings`);
+                const h1s = headingTexts(html);
+                if (h1s.length !== 1)
+                    add('headings', `${h1s.length} h1 headings`);
                 if (!snapshot.title)
                     add('title', 'no title');
                 if (!snapshot.description)
                     add('description', 'no meta description');
+                const indexable = route.indexing === 'index';
+                if (indexable && snapshot.title) {
+                    const formatProblems = titleFormatProblems(snapshot.title);
+                    if (formatProblems.length > 0)
+                        add('title-format', `"${snapshot.title}": ${formatProblems.join('; ')}`);
+                    titles.set(snapshot.title, [...(titles.get(snapshot.title) ?? []), route.path]);
+                    const place = routePlace(route);
+                    if (h1s.length === 1) {
+                        const problems = titleHeadingProblems(snapshot.title, h1s[0], place ? geography.placeNames.get(place) : undefined);
+                        if (problems.length > 0)
+                            add('title-h1', problems.join('; '));
+                    }
+                }
+                if (indexable && snapshot.description) {
+                    const problem = descriptionLengthProblem(snapshot.description);
+                    if (problem)
+                        add('description-length', problem);
+                    descriptions.set(snapshot.description, [...(descriptions.get(snapshot.description) ?? []), route.path]);
+                }
+                const bytes = Buffer.byteLength(html);
+                if (bytes > HTML_MAX_BYTES)
+                    add('html-size', `${(bytes / 1_048_576).toFixed(2)} MB of HTML (investigate anything over 1 MB)`);
                 const expectedCanonical = canonical(route.canonicalPath ?? route.path);
                 const ogUrl = snapshot.openGraph['og:url'];
                 if (ogUrl && new URL(ogUrl, origin).href !== expectedCanonical)
@@ -452,13 +580,80 @@ export function defineBrandlabPageQualityTests(options) {
                 for (const failure of inspectStructuredData(route, snapshot.jsonLd, businessId)) {
                     add('structured-data', failure.slice(route.path.length + 2));
                 }
+                const place = routePlace(route);
+                if (indexable && place)
+                    similarityPages.push({ path: route.path, place, blocks: textBlocks(html).map(geography.maskPlaces) });
+                const targets = new Set();
+                for (const href of extractLinks(html)) {
+                    const path = internalPath(href, expectedCanonical, origin);
+                    if (path && path !== normalisePath(route.path))
+                        targets.add(path);
+                }
+                linksFrom.set(normalisePath(route.path), targets);
             });
+            // Internal links: every indexable page needs a link from another page, and indexable pages link no noindex location page.
+            const inbound = new Map();
+            for (const [from, targets] of linksFrom) {
+                const source = routeByPath.get(from);
+                for (const target of targets) {
+                    inbound.set(target, (inbound.get(target) ?? 0) + 1);
+                    const linked = routeByPath.get(target);
+                    if (source?.indexing === 'index' && linked && linked.indexing !== 'index' && LOCATION_KINDS.has(linked.kind ?? '')) {
+                        findings.push({ path: from, kind: 'noindex-link', detail: `links to ${target}, a noindex location page (list unindexed places as plain text)` });
+                    }
+                }
+            }
+            for (const route of routes) {
+                const path = normalisePath(route.path);
+                if (route.indexing === 'index' && route.kind !== 'home' && path !== '/' && linksFrom.has(path) && !inbound.get(path)) {
+                    findings.push({ path: route.path, kind: 'orphan', detail: 'no other catalogue page links to it' });
+                }
+            }
+            for (const [title, paths] of titles) {
+                if (paths.length > 1)
+                    for (const path of paths)
+                        findings.push({ path, kind: 'title-duplicate', detail: `"${title}" is also the title of ${paths.filter((other) => other !== path).slice(0, 3).join(', ')}${paths.length > 4 ? ' and others' : ''}` });
+            }
+            for (const [description, paths] of descriptions) {
+                if (paths.length > 1)
+                    for (const path of paths)
+                        findings.push({ path, kind: 'description-duplicate', detail: `the meta description is shared with ${paths.length - 1} other page(s) ("${description.slice(0, 60)}…")` });
+            }
+            // Location pages: centerpiece similarity against near siblings, by route kind.
+            const similarity = [...new Set(similarityPages.map((page) => routeByPath.get(normalisePath(page.path))?.kind))]
+                .flatMap((kind) => centerpieceSimilarity(similarityPages.filter((page) => routeByPath.get(normalisePath(page.path))?.kind === kind), centreOf));
+            for (const score of similarity) {
+                if (score.score < SIMILARITY_WARN)
+                    continue;
+                findings.push({
+                    path: score.path,
+                    kind: 'similarity',
+                    detail: `${score.score.toFixed(2)} of its page-specific phrases also appear on ${score.sibling ?? 'a near sibling'}${score.score >= SIMILARITY_REVIEW ? ' (at or above 0.7: on the review list; the page stays indexed)' : ' (at or above 0.5: add page-specific evidence)'}`,
+                });
+            }
+            const reviewList = buildLocationReviewList(routes);
+            const flagged = new Set(similarity.filter((score) => score.score >= SIMILARITY_REVIEW).map((score) => score.path));
+            const review = [
+                ...reviewList.map((item) => ({ ...item, reasons: [...item.reasons, ...(flagged.has(item.path) ? ['centerpiece similarity at or above 0.7'] : [])] })),
+                ...[...flagged].filter((path) => !reviewList.some((item) => item.path === path)).map((path) => ({ path, evidence: routeByPath.get(normalisePath(path))?.locationDecision?.evidence ?? [], reasons: ['centerpiece similarity at or above 0.7'] })),
+            ].sort((a, b) => a.path.localeCompare(b.path));
+            for (const item of review)
+                findings.push({ path: item.path, kind: 'location-review', detail: item.reasons.join('; ') });
             findings.sort((a, b) => a.path.localeCompare(b.path) || a.kind.localeCompare(b.kind));
             const byKind = Object.fromEntries([...new Set(findings.map((finding) => finding.kind))].sort().map((kind) => [kind, findings.filter((finding) => finding.kind === kind).length]));
             await testInfo.attach('page-quality.json', {
                 body: JSON.stringify({ pages: routes.length, findings: findings.length, byKind, details: findings }, null, 2),
                 contentType: 'application/json',
             });
+            await testInfo.attach('location-review.json', {
+                body: JSON.stringify({
+                    reviewed: review.length,
+                    note: 'Pages on the review list stay indexed. Each review ends in adding evidence, merging into the nearest owner or hiding with noindex, follow; merging and hiding are Romie\'s decision.',
+                    similarity: similarity.sort((a, b) => b.score - a.score),
+                    review,
+                }, null, 2),
+                contentType: 'application/json',
+            });
             testInfo.annotations.push({
                 type: 'advisory',
                 description: `${findings.length} page-quality finding(s) across ${routes.length} page(s)${findings.length ? `: ${Object.entries(byKind).map(([kind, count]) => `${kind} ${count}`).join(', ')}` : ''}; see page-quality.json.`,