@nurkamol/seo-audit 1.31.0 → 1.32.0

package/README.md

@@ -30,9 +30,19 @@
 
 ```bash
 # Or anywhere, with nothing installed
+npx @nurkamol/seo-audit https://example.com
+
+# Or from the repository, pinned to a major version. Same tool — but this clones
+# 16 MB of app sources and tests to get at a 115 kB crawler.
 npx github:nurkamol/seo-audit@v1 https://example.com
+
+# Or keep it, if you would rather just type `seo-audit`
+npm install -g @nurkamol/seo-audit
 ```
 
+Every `npx github:…` in the examples below works as `npx @nurkamol/seo-audit`
+or as a plain `seo-audit`; they are the same bytes, fetched differently.
+
 <p align="center"><img src="docs/terminal.svg" alt="Example output" width="820"></p>
 
 > ### Not a terminal person? There is a window.
@@ -623,6 +633,7 @@ Findings come at three levels: **error** (wrong, and costing traffic), **warning
 | Check | Level |
 |---|---|
 | Page returns 200 and is not a redirect listed in the sitemap | error |
+| The body is actually HTML — an XML or JSON file served as `text/html` is crawled and indexed as a page | warning |
 | A page the server answers `429` to is reported as rate limited, never as a page that failed — the crawl waits, slows down and comes back first | note |
 | `noindex` on a page the sitemap advertises | error |
 | `X-Robots-Tag: noindex` — the same instruction as a header, invisible in the HTML | error |
@@ -640,7 +651,9 @@ Findings come at three levels: **error** (wrong, and costing traffic), **warning
 | The canonical target isn't `noindex` — a page that hands its indexing to one leaves the index with it | error |
 | Page 2 of an archive names itself, not page 1 — Google's guidance is "Don't use the first page of a paginated sequence as the canonical page", and a sitemap almost never lists these pages, so they're read where they're linked | error |
 | The canonical target isn't itself canonicalised elsewhere — Google needn't follow a chain | warning |
-| `og:title`, `og:description`, `og:image` present | warning |
+| `og:title` present — shared links otherwise fall back to the page title | warning |
+| `og:description` present — shared links otherwise preview whatever the platform scrapes | warning |
+| `og:image` present — shared links otherwise preview with no picture | warning |
 | `og:image` is an absolute URL — a scraper has no page to resolve a relative one against | error |
 | `og:image` is not WebP — LinkedIn won't render it, WhatsApp is unreliable | warning |
 | `og:image` declares width and height | note |
@@ -716,6 +729,7 @@ Findings come at three levels: **error** (wrong, and costing traffic), **warning
 | No page is linked but missing from the sitemap | warning |
 | The link and image sweeps say so when they stop at their cap rather than implying they checked everything | note |
 | Every `og:image` actually loads, and isn't too heavy to scrape | error / warning |
+| A declared `twitter:image` loads, when it differs from `og:image` — its *absence* is fine, X falls back to Open Graph | error |
 
 ---
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nurkamol/seo-audit",
-  "version": "1.31.0",
+  "version": "1.32.0",
   "description": "Crawl a site's sitemap and check every page for SEO, metadata and structured-data problems that single-page graders miss. Zero dependencies.",
   "type": "module",
   "bin": {

package/src/areas.mjs

@@ -29,7 +29,7 @@ export const CATEGORIES = [
 const CATEGORY_OF = {
   // Indexability
   noindex: 'Indexability', 'x-robots-noindex': 'Indexability', 'nofollow-page': 'Indexability',
-  'page-status': 'Indexability', unreachable: 'Indexability', 'nothing-crawlable': 'Indexability',
+  'page-status': 'Indexability', 'body-not-html': 'Indexability', unreachable: 'Indexability', 'nothing-crawlable': 'Indexability',
   'crawl-rate-limited': 'Indexability',
   'rate-limited': 'Indexability',
   'canonical-missing': 'Indexability', 'canonical-multiple': 'Indexability',
@@ -78,8 +78,9 @@ const CATEGORY_OF = {
   'img-lazy-priority': 'Images',
 
   // Social
-  'og-missing': 'Social', 'og-webp': 'Social', 'og-no-dimensions': 'Social',
-  'og-image-relative': 'Social', 'og-image-broken': 'Social', 'og-image-heavy': 'Social',
+  'og-title-missing': 'Social', 'og-description-missing': 'Social',
+  'og-image-missing': 'Social', 'og-webp': 'Social', 'og-no-dimensions': 'Social',
+  'og-image-relative': 'Social', 'og-image-broken': 'Social', 'twitter-image-broken': 'Social', 'og-image-heavy': 'Social',
 
   // Structured data
   'jsonld-invalid': 'Structured data', 'jsonld-no-type': 'Structured data',

package/src/audit.mjs

@@ -1,10 +1,10 @@
 // Orchestration: find the pages, fetch them, run the checks.
 import { Fetcher, mapLimit } from './http.mjs';
-import { parseHtml, parseSitemap } from './parse.mjs';
+import { bodyKind, parseHtml, parseSitemap } from './parse.mjs';
 import { parseRobots, robotsVerdict } from './robots.mjs';
 import { redirectChecks } from './redirects.mjs';
 import { pageChecks, crossPageChecks, sitemapChecks } from './checks.mjs';
-import { siteChecks } from './site.mjs';
+import { certificateExpiry, siteChecks } from './site.mjs';
 import { linkGraph } from './graph.mjs';
 import { compareAgents } from './compare.mjs';
 import { searchConsole } from './console.mjs';
@@ -125,7 +125,8 @@ async function crawlByLinks(origin, fetcher, { limit, concurrency, robotsGroups,
         url: final.url,
         res: final,
         html: final.body,
-        doc: final.ok && isHtml ? parseHtml(final.body, final.url) : null,
+        // Same as the sitemap path: the header is a claim, the body is the fact.
+        doc: final.ok && isHtml && !bodyKind(final.body) ? parseHtml(final.body, final.url) : null,
       };
     });
 
@@ -222,17 +223,51 @@ export async function audit(target, opts = {}) {
   // A host that never answered is not a sitemap problem, and following links
   // from a page that does not load would find nothing either.
   if (!urls.length && !fetcher.reachable) {
-    findings.push({
-      level: 'error',
-      id: 'unreachable',
-      title: 'The site did not answer a single request',
-      detail:
-        `Tried: ${tried.join(', ')}. The TLS connection succeeds but no response arrives, which ` +
-        'usually means a bot-protection rule is stalling non-browser clients — Cloudflare Bot ' +
-        "Fight Mode does exactly this. If it is your site, allow this crawler's user agent, or " +
-        'pass --user-agent to present a different one.',
-      url: origin,
-    });
+    // Read the certificate before blaming bot protection.
+    //
+    // A browser refuses an expired certificate and so does `fetch`, so "nothing
+    // answered" and "the certificate lapsed years ago" are indistinguishable
+    // from here — except that one of them is knowable, over a socket that does
+    // not validate. This used to assert "the TLS connection succeeds" and point
+    // at Cloudflare Bot Fight Mode, which for expired.badssl.com was wrong
+    // twice over and sends somebody to the wrong dashboard. The site checks
+    // that would have caught it never run, because the crawl gives up first.
+    const expiresAt =
+      url.protocol === 'https:'
+        ? await (opts.readCertificateExpiry ?? certificateExpiry)(url.hostname)
+        : null;
+    const lapsed = expiresAt ? Math.floor(((opts.now ?? Date.now()) - expiresAt) / 86_400_000) : 0;
+
+    findings.push(
+      lapsed > 0
+        ? {
+            level: 'error',
+            id: 'tls-expired',
+            title: `The TLS certificate expired ${lapsed} day(s) ago`,
+            detail:
+              `It ran out on ${new Date(expiresAt).toISOString().slice(0, 10)}, which is why nothing here ` +
+              'could be fetched — browsers refuse the site outright. Renew it; nothing else about this ' +
+              'site can be measured until then.',
+            url: origin,
+          }
+        : {
+            level: 'error',
+            id: 'unreachable',
+            title: 'The site did not answer a single request',
+            detail:
+              `Tried: ${tried.join(', ')}. ` +
+              // Only said where it was actually established. The hosted Worker
+              // has no socket to read a certificate over, so it gets null here
+              // and must not be made to assert that the certificate is fine —
+              // a runtime that cannot run a check says so rather than implying
+              // a result.
+              (expiresAt ? 'The certificate is valid, so this is ' : 'This is ') +
+              'usually a bot-protection rule stalling non-browser clients — Cloudflare Bot Fight Mode ' +
+              "does exactly this. If it is your site, allow this crawler's user agent, or pass " +
+              '--user-agent to present a different one.',
+            url: origin,
+          },
+    );
     return {
       findings,
       meta: {
@@ -326,7 +361,10 @@ export async function audit(target, opts = {}) {
         url: pageUrl,
         res,
         html: res.body,
-        doc: res.ok && isHtml ? parseHtml(res.body, pageUrl) : null,
+        // `isHtml` is the server's claim; `bodyKind` is what actually came
+        // back. A sitemap that lists an XML file the server labels text/html
+        // otherwise gets parsed as a page and reported for having no h1.
+        doc: res.ok && isHtml && !bodyKind(res.body) ? parseHtml(res.body, pageUrl) : null,
       };
     });
   } else {
@@ -411,7 +449,7 @@ export async function audit(target, opts = {}) {
   let home = null;
   if (!pages.some((p) => p.doc && p.res.ok && new URL(p.url).pathname.replace(/\/$/, '') === '')) {
     const { final } = await fetcher.chain(origin);
-    if (final.ok && /text\/html/i.test(final.headers.get('content-type') ?? '')) {
+    if (final.ok && /text\/html/i.test(final.headers.get('content-type') ?? '') && !bodyKind(final.body)) {
       home = { url: final.url, doc: parseHtml(final.body, final.url) };
     }
   }

package/src/checks.mjs

@@ -12,7 +12,7 @@
 
 import { linkGraph, key as graphKey } from './graph.mjs';
 
-import { attr, stripMarkupInAttributes } from './parse.mjs';
+import { attr, bodyKind, stripMarkupInAttributes } from './parse.mjs';
 import { cluster } from './dupes.mjs';
 
 // Defaults, overridable per site under `limits` in the config file. A
@@ -265,6 +265,23 @@ export function pageChecks(page, limits = DEFAULT_LIMITS) {
       res.error ? `Request failed: ${res.error}` : `HTTP ${res.status}`, url));
     return out;
   }
+  // A URL the server calls HTML that is not HTML. The body was already read, so
+  // this costs no request — and it matters, because Google indexes whatever
+  // comes back under `text/html` as a page.
+  //
+  // Raised *instead of* the page checks below, not alongside them. A sitemap
+  // listing an XML file labelled text/html was reported for having no title, no
+  // h1, no viewport, no charset, thin content and three missing Open Graph tags
+  // — thirteen true statements about a document that was never a page, and not
+  // one of them the thing to fix.
+  if (!doc && res.ok) {
+    const kind = bodyKind(page.html);
+    if (kind && /text\/html/i.test(res.headers?.get?.('content-type') ?? '')) {
+      out.push(f('warn', 'body-not-html', `Served as HTML, but the body is ${kind}`,
+        'The server sends Content-Type: text/html, so this is crawled and indexed as a page. ' +
+          'Either serve it with its own content type or take it out of the sitemap.', url));
+    }
+  }
   if (!doc) return out;
 
   // --- Indexability -------------------------------------------------------
@@ -426,9 +443,23 @@ export function pageChecks(page, limits = DEFAULT_LIMITS) {
   }
 
   // --- Social -------------------------------------------------------------
-  for (const tag of ['og:title', 'og:description', 'og:image']) {
-    if (!doc.og[tag]) out.push(f('warn', 'og-missing', `Missing ${tag}`,
-      'Shared links will preview with whatever the platform scrapes.', url));
+  // One id per tag, not one for all three. They shared `og-missing` until a run
+  // against a real site printed "Missing og:description ×6" over four pages —
+  // three of which were missing only the description, and one missing all
+  // three. The grouping takes its title from whichever finding it saw first, so
+  // a row named one tag and counted another two. Every finding was true and the
+  // summary above them was not, which is the failure grouping exists to avoid.
+  //
+  // They are also three different repairs: a missing og:image is a picture
+  // somebody has to make, and a missing og:title is one line of template. Being
+  // able to `--ignore og-image-missing` and keep the other two is worth an id.
+  const OG_REQUIRED = [
+    ['og:title', 'og-title-missing', 'Shared links fall back to the page title, if the platform finds one.'],
+    ['og:description', 'og-description-missing', 'Shared links preview with whatever text the platform scrapes.'],
+    ['og:image', 'og-image-missing', 'Shared links preview with no picture, which is most of what gets clicked.'],
+  ];
+  for (const [tag, id, consequence] of OG_REQUIRED) {
+    if (!doc.og[tag]) out.push(f('warn', id, `Missing ${tag}`, consequence, url));
   }
   const ogImage = doc.og['og:image'];
   // The Open Graph spec requires an absolute URL. A scraper has no page context

package/src/config.mjs

@@ -41,9 +41,20 @@ export function matchGlob(pattern, path) {
 
 /** A rule matches a finding when the id matches and, if the rule names URLs,
  *  one of them matches the finding's path. */
+// Ids that used to be one check. A config in somebody's repository is written
+// against the ids that existed when they wrote it, and splitting a check is our
+// decision, not theirs — so the old id keeps silencing what it used to silence.
+// The alternative is a check they had accepted starting to fail their build on
+// an upgrade that promised to be compatible.
+const RETIRED_IDS = {
+  // Split in 1.32.0: one id covered three tags, so a group named one of them
+  // and counted all three.
+  'og-missing': ['og-title-missing', 'og-description-missing', 'og-image-missing'],
+};
+
 function ruleMatches(rule, finding) {
   const id = typeof rule === 'string' ? rule : rule.id;
-  if (id !== finding.id) return false;
+  if (id !== finding.id && !RETIRED_IDS[id]?.includes(finding.id)) return false;
   const urls = typeof rule === 'string' ? null : rule.urls;
   if (!urls?.length) return true;
   if (!finding.url) return false;

package/src/parse.mjs

@@ -345,3 +345,34 @@ export function parseSitemap(xml) {
 
   return { urls: locs, sitemaps: [], entries };
 }
+
+/**
+ * What a response body actually is, when the server has already claimed it is
+ * HTML. Returns `null` for anything that might be HTML, and a noun for the
+ * cases that provably are not.
+ *
+ * Deliberately conservative — it answers on positive evidence only. A fragment
+ * with no `<html>` wrapper is still HTML; a page behind a byte-order mark is
+ * still HTML; and XHTML opens with `<?xml` and *is* HTML, which is why the
+ * markers are checked before the prologue. Guessing wrong here would silence
+ * every check on a real page, which is a worse failure than the noise it is
+ * meant to remove.
+ */
+export function bodyKind(body) {
+  if (typeof body !== 'string') return null;
+  const head = body.replace(/^/, '').trimStart().slice(0, 2048);
+  if (!head) return null;
+  // Any of these and it is HTML, whatever it opened with.
+  if (/<!doctype\s+html|<html[\s>]|<head[\s>]|<body[\s>]/i.test(head)) return null;
+  if (/^%PDF-/.test(head)) return 'a PDF';
+  if (/^<\?xml[\s?]/i.test(head) || /^<(urlset|sitemapindex|rss|feed|kml|svg)[\s>]/i.test(head)) return 'XML';
+  if (/^[[{]/.test(head)) {
+    try {
+      JSON.parse(body);
+      return 'JSON';
+    } catch {
+      return null;
+    }
+  }
+  return null;
+}

package/src/site.mjs

@@ -710,5 +710,34 @@ export async function siteChecks(origin, fetcher, pages, opts = {}) {
     }
   }
 
+  // twitter:image, and only when it is a different picture from og:image.
+  //
+  // The *absence* of a Twitter card is deliberately not a finding and stays
+  // that way: X falls back to Open Graph correctly, so reporting it would
+  // invent a defect. A twitter:image that is declared and does not load is the
+  // opposite case — nothing falls back to anything, and the one platform that
+  // was handed its own tag previews blank. Same conservatism as above: a 403 is
+  // hotlink protection working.
+  const twitterImages = new Map();
+  for (const page of pages) {
+    const src = page.doc?.twitter?.['twitter:image'];
+    if (!src || !/^(https?:)?\/\//i.test(src)) continue;
+    // Identical to og:image means the sweep above already judged this picture.
+    // Two findings about one file is the noise this project keeps refusing.
+    if (src === page.doc?.og?.['og:image']) continue;
+    twitterImages.set(src, page.url);
+  }
+  const twitterResults = await mapLimit([...twitterImages.keys()], 4, async (src) => {
+    const { final } = await fetcher.chain(src);
+    return { src, final };
+  });
+  for (const { src, final } of twitterResults) {
+    if (final.status === 404 || final.status === 410 || final.status === 0) {
+      out.push(f('error', 'twitter-image-broken', 'twitter:image does not load',
+        `HTTP ${final.status || final.error} for ${src} — it differs from og:image, so X has nothing to fall back to.`,
+        twitterImages.get(src)));
+    }
+  }
+
   return out;
 }