blume 2.0.1 → 2.0.3

package/src/audit/checks/i18n.ts

@@ -3,7 +3,8 @@ import type { Diagnostic } from "../../core/types.ts";
 import { finding } from "../catalog.ts";
 import { pageSite } from "../locate.ts";
 import type { AuditContext, CheckModule, PageSnapshot } from "../types.ts";
-import { normalizePath } from "../url.ts";
+import { decodePath, normalizePath } from "../url.ts";
+import { isArchivedLatestCanonical } from "./indexability.ts";
 
 /** The normalized `deployment.base` — hreflang hrefs carry it, page URLs don't. */
 const deployBaseOf = (context: AuditContext): string =>
@@ -21,15 +22,52 @@ const isValidBcp47 = (tag: string): boolean => {
   }
 };
 
-/** The site path an hreflang `href` points at, or null when it isn't parseable. */
+/**
+ * The site path an hreflang `href` points at, or null when it isn't parseable.
+ * Percent-decoded like every other URL the audit compares, since page URLs
+ * come from raw on-disk names (`/ガイド`) and `URL#pathname` is encoded.
+ */
 const alternatePath = (href: string, deployBase: string): string | null => {
   try {
-    return normalizePath(stripBasePath(deployBase, new URL(href).pathname));
+    return normalizePath(
+      stripBasePath(deployBase, decodePath(new URL(href).pathname))
+    );
   } catch {
     return null;
   }
 };
 
+/**
+ * An i18n fallback copy renders another locale's page at this locale's URL.
+ * Blume gives it hreflang links to the real translations, which by design
+ * never link back to it (it isn't one), and leaves it out of the sitemap —
+ * so the hreflang cluster checks don't apply to it.
+ */
+const isFallbackCopy = (page: PageSnapshot): boolean =>
+  page.route?.fallback === true;
+
+/**
+ * Whether `target` names a URL other than its own as canonical — unless that
+ * is Blume's archived-version default pointing at the latest equivalent,
+ * which the canonical checks already accept.
+ */
+const canonicalizesElsewhere = (
+  context: AuditContext,
+  target: PageSnapshot,
+  deployBase: string
+): boolean => {
+  if (!target.canonical) {
+    return false;
+  }
+  const canonical = alternatePath(target.canonical, deployBase);
+  if (canonical === normalizePath(target.url)) {
+    return false;
+  }
+  return !(
+    canonical !== null && isArchivedLatestCanonical(context, target, canonical)
+  );
+};
+
 const langChecks = (
   context: AuditContext,
   page: PageSnapshot
@@ -115,10 +153,7 @@ const hreflangChecks = (
           `hreflang="${alternate.lang}" points at ${path}, which the build does not serve.`
         )
       );
-    } else if (
-      target.canonical &&
-      alternatePath(target.canonical, deployBase) !== normalizePath(target.url)
-    ) {
+    } else if (canonicalizesElsewhere(context, target, deployBase)) {
       found.push(
         finding(
           "BLUME_AUDIT_HREFLANG_BAD_TARGET",
@@ -190,6 +225,9 @@ const returnTagChecks = (context: AuditContext): Diagnostic[] => {
   const found: Diagnostic[] = [];
   const deployBase = deployBaseOf(context);
   for (const page of context.pages) {
+    if (isFallbackCopy(page)) {
+      continue;
+    }
     for (const alternate of page.hreflang) {
       if (alternate.lang === X_DEFAULT) {
         continue;
@@ -235,7 +273,7 @@ export const i18nChecks: CheckModule = {
     const found: Diagnostic[] = [];
     for (const page of context.pages) {
       found.push(...langChecks(context, page));
-      if (page.hreflang.length > 0) {
+      if (page.hreflang.length > 0 && !isFallbackCopy(page)) {
         found.push(...hreflangChecks(context, page));
       }
     }

package/src/audit/checks/indexability.ts

@@ -16,6 +16,33 @@ const parseCanonical = (canonical: string): URL | null => {
   }
 };
 
+/**
+ * Whether `page` naming `target` as its canonical is Blume's own default for
+ * an archived-version page (`versions.archived[].canonical: "latest"`)
+ * pointing at its latest equivalent — the live page is the authoritative one
+ * by design, not a defect. `target` is a base-less, normalized site path.
+ */
+export const isArchivedLatestCanonical = (
+  context: AuditContext,
+  page: PageSnapshot,
+  target: string
+): boolean => {
+  const version = page.route?.version;
+  const archived = version
+    ? context.project.config.versions?.archived.find(
+        (entry) => entry.id === version
+      )
+    : undefined;
+  const latestAlternate = page.route?.versionAlternates.find(
+    (alternate) => alternate.version === ""
+  );
+  return (
+    archived?.canonical === "latest" &&
+    latestAlternate !== undefined &&
+    normalizePath(latestAlternate.path) === target
+  );
+};
+
 const canonicalChecks = (
   context: AuditContext,
   page: PageSnapshot
@@ -33,8 +60,10 @@ const canonicalChecks = (
   if (!page.canonical) {
     // Without `deployment.site` Blume has no absolute URL to canonicalize to, so
     // report the root cause once (in the sitemap/robots checks) rather than
-    // flagging every page for a config field they can't fix individually.
-    return site
+    // flagging every page for a config field they can't fix individually. A
+    // noindex page omits its canonical on purpose — the layouts drop it, since
+    // Google treats the pairing as contradictory (see CANONICAL_ON_NOINDEX).
+    return site && page.indexable
       ? [
           finding(
             "BLUME_AUDIT_CANONICAL_MISSING",
@@ -101,23 +130,14 @@ const canonicalChecks = (
       )
     );
   } else if (context.byUrl.has(target)) {
-    // An archived-version page pointing at its latest equivalent is Blume's
-    // own default (versions.archived[].canonical: "latest"), not a defect —
-    // the live page is the authoritative one by design.
-    const version = page.route?.version;
-    const archived = version
-      ? context.project.config.versions?.archived.find(
-          (entry) => entry.id === version
-        )
-      : undefined;
-    const latestAlternate = page.route?.versionAlternates.find(
-      (alternate) => alternate.version === ""
-    );
-    const intentional =
-      archived?.canonical === "latest" &&
-      latestAlternate !== undefined &&
-      normalizePath(latestAlternate.path) === target;
-    if (!intentional) {
+    // An i18n fallback copy canonicalizes to the page it copies on purpose,
+    // like an archived page to its latest equivalent.
+    if (
+      !(
+        page.route?.fallback === true ||
+        isArchivedLatestCanonical(context, page, target)
+      )
+    ) {
       found.push(
         finding(
           "BLUME_AUDIT_CANONICAL_NOT_SELF",

package/src/audit/checks/links.ts

@@ -14,6 +14,9 @@ import { isServed, normalizePath, resolveHref, siteOrigin } from "../url.ts";
 /** Browser-magic fragments that scroll without needing a matching id. */
 const MAGIC_FRAGMENTS = new Set(["", "top"]);
 
+/** Where a fragment's directives (text fragments) start. */
+const FRAGMENT_DIRECTIVE = ":~:";
+
 /** Whether a fragment lands on an id of the target page. */
 const anchorResolves = (target: PageSnapshot, fragment: string): boolean => {
   if (MAGIC_FRAGMENTS.has(fragment)) {
@@ -65,8 +68,12 @@ export const linkChecks: CheckModule = {
       page: PageSnapshot,
       link: SnapshotLink,
       target: PageSnapshot,
-      fragment: string
+      hash: string
     ): void => {
+      // A text fragment (`#:~:text=…`) is a directive the browser strips
+      // before it looks for an id; only the part in front of `:~:` names an
+      // element, and an empty one scrolls to the text instead.
+      const [fragment = ""] = hash.split(FRAGMENT_DIRECTIVE);
       if (anchorResolves(target, fragment)) {
         return;
       }
@@ -100,6 +107,20 @@ export const linkChecks: CheckModule = {
       if (resolved.kind === "external" || resolved.kind === "ignored") {
         return;
       }
+      if (resolved.kind === "outside-base") {
+        if (link.content) {
+          found.push(
+            finding(
+              "BLUME_AUDIT_LINK_TO_BROKEN",
+              pageSite(context, page),
+              `Link to ${link.href} is missing deployment.base (${deployBase}), so the deployed site does not serve it.`
+            )
+          );
+        } else if (!brokenChrome.has(resolved.path)) {
+          brokenChrome.set(resolved.path, page);
+        }
+        return;
+      }
 
       const anchorTarget = resolved.hash
         ? context.byUrl.get(resolved.path)

package/src/audit/checks/llms.ts

@@ -113,15 +113,18 @@ export const llmsChecks: CheckModule = {
 
     // The reverse direction, mirroring INDEXABLE_PAGE_NOT_IN_SITEMAP: a page
     // that is built, indexable, and in the nav belongs in the index. Pages the
-    // generator deliberately skips — hidden, drafts, error routes, API
-    // reference when `agents.llmsTxt.openapi` is off, custom pages with no
-    // manifest route — are skipped here for the same reasons.
+    // generator deliberately skips — hidden, drafts, `ai.exclude`, i18n
+    // fallback copies, error routes, API reference when
+    // `agents.llmsTxt.openapi` is off, custom pages with no manifest route —
+    // are skipped here for the same reasons.
     for (const page of context.pages) {
       const { route } = page;
       if (
         !route ||
         route.hidden ||
         route.draft ||
+        route.aiExclude ||
+        route.fallback ||
         !page.indexable ||
         ERROR_ROUTES.has(page.url) ||
         (!openapi && route.source.name === "openapi") ||

package/src/audit/checks/network.ts

@@ -180,8 +180,10 @@ export const networkChecks: CheckModule = {
       found.push(...servedPageChecks(context, page, result, origin));
     }
 
+    // With `seo.robots: false` Blume writes no robots.txt, so its absence on
+    // the live site is the configured outcome, not a defect.
     const robots = results.get(robotsUrl);
-    if (robots && !robots.ok) {
+    if (context.project.config.seo.robots && robots && !robots.ok) {
       found.push(
         finding(
           "BLUME_AUDIT_ROBOTS_NOT_ACCESSIBLE",

package/src/audit/checks/og-image.ts

@@ -71,6 +71,16 @@ export const ogImageChecks: CheckModule = {
       if (resolved.kind === "external" || resolved.kind === "ignored") {
         continue;
       }
+      if (resolved.kind === "outside-base") {
+        found.push(
+          finding(
+            "BLUME_AUDIT_OG_IMAGE_BROKEN",
+            pageSite(context, page, ["seo", "image"]),
+            `og:image points at ${resolved.path}, which is missing deployment.base (${deployBase}), so the deployed site does not serve it.`
+          )
+        );
+        continue;
+      }
 
       const { path } = resolved;
       if (

package/src/audit/checks/robots.ts

@@ -47,7 +47,12 @@ export const robotsChecks: CheckModule = {
       )
     );
 
-    if (site && robots.sitemaps.length === 0) {
+    // Blume only writes the Sitemap line when there is a sitemap to point at.
+    if (
+      site &&
+      context.project.config.seo.sitemap &&
+      robots.sitemaps.length === 0
+    ) {
       found.push(
         finding(
           "BLUME_AUDIT_ROBOTS_SITEMAP_MISSING",

package/src/audit/checks/sitemap.ts

@@ -138,6 +138,40 @@ const checkListedUrl = (
   return found;
 };
 
+/**
+ * The other direction: a page that was built, is indexable, and should be
+ * findable — but never made it into the sitemap. A page that names another
+ * URL as its canonical (an archived-version page pointing at the latest docs)
+ * and an i18n fallback copy are left out of the sitemap on purpose: it lists
+ * canonical URLs only (see NON_CANONICAL_IN_SITEMAP).
+ */
+const unlistedPages = (
+  context: AuditContext,
+  listed: Map<string, string>,
+  deployBase: string
+): Diagnostic[] =>
+  context.pages.flatMap((page) => {
+    const canonical = page.canonical
+      ? canonicalPath(page.canonical, deployBase)
+      : null;
+    if (
+      !page.indexable ||
+      page.route?.fallback ||
+      (canonical && canonical !== normalizePath(page.url)) ||
+      ERROR_ROUTES.has(page.url) ||
+      listed.has(normalizePath(page.url))
+    ) {
+      return [];
+    }
+    return [
+      finding(
+        "BLUME_AUDIT_INDEXABLE_PAGE_NOT_IN_SITEMAP",
+        pageSite(context, page),
+        `${page.url} is built and indexable but is not listed in sitemap.xml.`
+      ),
+    ];
+  });
+
 /**
  * The sitemap, cross-checked against what was actually built.
  *
@@ -175,23 +209,32 @@ export const sitemapChecks: CheckModule = {
         finding(
           "BLUME_AUDIT_SITEMAP_INVALID",
           { file: sitemap.file, url: "/sitemap.xml" },
-          `sitemap.xml is not a valid urlset: ${sitemap.error}`
+          `sitemap.xml is not a valid urlset or sitemap index: ${sitemap.error}`
         )
       );
       return found;
     }
 
-    if (
-      sitemap.bytes > MAX_SITEMAP_BYTES ||
-      sitemap.urls.length > MAX_SITEMAP_URLS
-    ) {
-      found.push(
-        finding(
-          "BLUME_AUDIT_SITEMAP_TOO_LARGE",
-          { file: sitemap.file, url: "/sitemap.xml" },
-          `sitemap.xml holds ${sitemap.urls.length} URLs in ${Math.round(sitemap.bytes / 1024 / 1024)} MB.`
-        )
-      );
+    // The limits are per file: a sitemap index splits a large site across
+    // urlsets, each of which must fit on its own.
+    const parts = sitemap.parts ?? [
+      {
+        bytes: sitemap.bytes,
+        file: sitemap.file,
+        url: "/sitemap.xml",
+        urls: sitemap.urls.length,
+      },
+    ];
+    for (const part of parts) {
+      if (part.bytes > MAX_SITEMAP_BYTES || part.urls > MAX_SITEMAP_URLS) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_SITEMAP_TOO_LARGE",
+            { file: part.file, url: part.url },
+            `${part.url.slice(1)} holds ${part.urls} URLs in ${Math.round(part.bytes / 1024 / 1024)} MB.`
+          )
+        );
+      }
     }
 
     // A `<lastmod>` that lies — malformed, or claiming the future — teaches
@@ -230,25 +273,7 @@ export const sitemapChecks: CheckModule = {
       );
     }
 
-    // The other direction: a page that was built, is indexable, and should be
-    // findable — but never made it into the sitemap.
-    for (const page of context.pages) {
-      if (
-        !page.indexable ||
-        ERROR_ROUTES.has(page.url) ||
-        listed.has(normalizePath(page.url))
-      ) {
-        continue;
-      }
-      found.push(
-        finding(
-          "BLUME_AUDIT_INDEXABLE_PAGE_NOT_IN_SITEMAP",
-          pageSite(context, page),
-          `${page.url} is built and indexable but is not listed in sitemap.xml.`
-        )
-      );
-    }
-
+    found.push(...unlistedPages(context, listed, deployBase));
     return found;
   },
   tier: "static",

package/src/audit/checks/social.ts

@@ -1,7 +1,8 @@
+import { normalizeBasePath, stripBasePath } from "../../core/base-path.ts";
 import type { Diagnostic } from "../../core/types.ts";
 import { finding } from "../catalog.ts";
 import { pageSite } from "../locate.ts";
-import type { CheckModule, PageSnapshot } from "../types.ts";
+import type { AuditContext, CheckModule, PageSnapshot } from "../types.ts";
 import { normalizePath } from "../url.ts";
 
 /** The Open Graph properties a share card is unusable without. */
@@ -175,6 +176,23 @@ const URL_STYLE: { name: string; test: RegExp }[] = [
   { name: "spaces", test: /%20| /u },
 ];
 
+/**
+ * The part of a page URL its author named: the URL minus a leading i18n locale
+ * segment. A locale code like `pt-BR` is BCP 47 casing set once in config,
+ * not an untidy slug, so it must not trip the uppercase check on every page
+ * of that locale. The segment follows `basePath`, which page URLs carry.
+ */
+const authoredPath = (context: AuditContext, url: string): string => {
+  const { basePath, i18n } = context.project.config;
+  if (!i18n) {
+    return url;
+  }
+  const base = normalizeBasePath(basePath);
+  const [, first = "", ...rest] = stripBasePath(base, url).split("/");
+  const isLocale = i18n.locales.some((locale) => locale.code === first);
+  return isLocale ? `${base}/${rest.join("/")}` : url;
+};
+
 /** A protocol-relative URL names a dotted host (`//cdn.example.com/x`). */
 const DOTTED_HOST = /^\/\/[^/]*\./u;
 
@@ -220,7 +238,8 @@ export const urlChecks: CheckModule = {
         }
       }
 
-      const untidy = URL_STYLE.filter((style) => style.test.test(page.url));
+      const authored = authoredPath(context, page.url);
+      const untidy = URL_STYLE.filter((style) => style.test.test(authored));
       if (untidy.length > 0) {
         found.push(
           finding(

package/src/audit/crawl.ts

@@ -125,6 +125,11 @@ type XmlValue =
 const isUrlsetElement = (value: XmlValue): value is { url?: XmlValue } =>
   typeof value === "object" && value !== null;
 
+const isSitemapIndexElement = (
+  value: XmlValue
+): value is { sitemap?: XmlValue } =>
+  typeof value === "object" && value !== null;
+
 const isUrlEntry = (
   value: XmlValue
 ): value is { lastmod?: XmlValue; loc?: XmlValue } =>
@@ -134,7 +139,9 @@ const isText = (value: XmlValue): value is string => typeof value === "string";
 
 /**
  * Parse `sitemap.xml`. Deliberately shallow: the checks only need the `<loc>`
- * list, each loc's `<lastmod>`, and whether the document is a urlset at all.
+ * list, each loc's `<lastmod>`, and whether the document is a urlset at all —
+ * or, for a sitemap index, the child sitemaps it lists (`sitemaps`), which
+ * the crawler reads from the build.
  */
 export const parseSitemap = (
   file: string,
@@ -149,10 +156,20 @@ export const parseSitemap = (
     doc.error = "no <urlset> element";
     return doc;
   }
+  if (Object.hasOwn(parsed, "sitemapindex")) {
+    const { sitemapindex } = parsed;
+    const children = isSitemapIndexElement(sitemapindex)
+      ? [sitemapindex.sitemap].flat()
+      : [];
+    doc.sitemaps = children.flatMap((child) =>
+      isUrlEntry(child) && isText(child.loc) && child.loc.trim()
+        ? [child.loc.trim()]
+        : []
+    );
+    return doc;
+  }
   if (!Object.hasOwn(parsed, "urlset")) {
-    doc.error = Object.hasOwn(parsed, "sitemapindex")
-      ? "sitemap is an index, not a urlset"
-      : "no <urlset> element";
+    doc.error = "no <urlset> element";
     return doc;
   }
   const { urlset } = parsed;
@@ -252,6 +269,69 @@ const readIfPresent = async (file: string): Promise<string | null> => {
   }
 };
 
+/**
+ * Read `sitemap.xml`, following it through a sitemap index: each child
+ * sitemap is read from the build (its `<loc>` carries the site origin and the
+ * deployment base, the file tree neither) and its URLs merged into the one
+ * document the checks see. A child that can't be resolved, is missing, or
+ * isn't a urlset makes the whole document invalid.
+ */
+const readSitemap = async (
+  staticDir: string,
+  files: Map<string, number>,
+  deployBase: string
+): Promise<SitemapDoc | null> => {
+  const file = join(staticDir, "sitemap.xml");
+  const xml = await readIfPresent(file);
+  if (xml === null) {
+    return null;
+  }
+  const doc = parseSitemap(
+    file,
+    xml,
+    files.get("/sitemap.xml") ?? Buffer.byteLength(xml, "utf-8")
+  );
+  if (!doc.sitemaps) {
+    return doc;
+  }
+  doc.parts = [];
+  for (const loc of doc.sitemaps) {
+    let url: string;
+    try {
+      url = stripBasePath(deployBase, decodeURI(new URL(loc).pathname));
+    } catch {
+      doc.error = `the sitemap index lists "${loc}", which is not a valid absolute URL`;
+      return doc;
+    }
+    const childFile = join(staticDir, url);
+    // oxlint-disable-next-line no-await-in-loop -- one read per child, in index order
+    const childXml = await readIfPresent(childFile);
+    const child =
+      childXml === null
+        ? null
+        : parseSitemap(
+            childFile,
+            childXml,
+            files.get(url) ?? Buffer.byteLength(childXml, "utf-8")
+          );
+    if (!child || child.error || child.sitemaps) {
+      doc.error = `the sitemap index lists ${loc}, which is not a urlset in the build`;
+      return doc;
+    }
+    doc.urls.push(...child.urls);
+    for (const [key, value] of child.lastmod ?? []) {
+      doc.lastmod?.set(key, value);
+    }
+    doc.parts.push({
+      bytes: child.bytes,
+      file: childFile,
+      url,
+      urls: child.urls.length,
+    });
+  }
+  return doc;
+};
+
 /**
  * Read the built site: every HTML page reduced to a snapshot, the full file
  * index (for resolving subresource references), plus sitemap.xml and robots.txt.
@@ -260,8 +340,10 @@ export const crawlStaticDir = async (options: {
   staticDir: string;
   manifest: BlumeManifest;
   basePath: string;
+  /** The normalized `deployment.base`, which sitemap-index `<loc>`s carry. */
+  deployBase?: string;
 }): Promise<CrawlResult> => {
-  const { staticDir, manifest, basePath } = options;
+  const { staticDir, manifest, basePath, deployBase = "" } = options;
   const routes = routeIndex(manifest, basePath);
 
   const htmlFiles = await glob("**/*.html", { absolute: true, cwd: staticDir });
@@ -287,8 +369,6 @@ export const crawlStaticDir = async (options: {
   );
   const pages = snapshots.filter((page) => page !== null);
 
-  const sitemapFile = join(staticDir, "sitemap.xml");
-  const sitemapXml = await readIfPresent(sitemapFile);
   const robotsFile = join(staticDir, "robots.txt");
   const robotsTxt = await readIfPresent(robotsFile);
   const llmsFile = join(staticDir, "llms.txt");
@@ -300,13 +380,6 @@ export const crawlStaticDir = async (options: {
     llms: llmsText === null ? null : parseLlms(llmsFile, llmsText),
     pages,
     robots: robotsTxt === null ? null : parseRobots(robotsFile, robotsTxt),
-    sitemap:
-      sitemapXml === null
-        ? null
-        : parseSitemap(
-            sitemapFile,
-            sitemapXml,
-            files.get("/sitemap.xml") ?? Buffer.byteLength(sitemapXml, "utf-8")
-          ),
+    sitemap: await readSitemap(staticDir, files, deployBase),
   };
 };

package/src/audit/graph.ts

@@ -58,7 +58,8 @@ export const buildGraph = (
  * Non-indexable pages are excluded — a `noindex` page is *meant* to be
  * unreachable — as is the home page, which is nobody's job to link to. Under a
  * `basePath` the home page's built URL is the base itself, so the caller names
- * it via `homeUrl`.
+ * it via `homeUrl`. An i18n fallback copy is excluded too: it stands in for a
+ * translation nobody has written yet, so the page to link to is the original.
  */
 export const orphanPages = (
   pages: PageSnapshot[],
@@ -68,6 +69,7 @@ export const orphanPages = (
   pages.filter(
     (page) =>
       page.indexable &&
+      page.route?.fallback !== true &&
       page.url !== "/" &&
       page.url !== homeUrl &&
       (graph.contentIn.get(page.url)?.size ?? 0) === 0