blume 1.0.3 → 1.1.0

package/src/audit/checks/robots.ts

@@ -0,0 +1,114 @@
+import type { Diagnostic } from "../../core/types.ts";
+import { finding } from "../catalog.ts";
+import type { CheckModule } from "../types.ts";
+import { normalizePath } from "../url.ts";
+
+/**
+ * Whether a robots.txt `Disallow` value covers a path. robots.txt matching is
+ * prefix-based, with `*` as a wildcard and `$` anchoring the end.
+ */
+export const disallowMatches = (rule: string, path: string): boolean => {
+  const anchored = rule.endsWith("$");
+  const pattern = anchored ? rule.slice(0, -1) : rule;
+  const parts = pattern.split("*");
+
+  let cursor = 0;
+  for (const [index, part] of parts.entries()) {
+    if (part === "") {
+      continue;
+    }
+    // The first segment is anchored to the start of the path (robots.txt rules
+    // are prefix matches); every later segment may appear anywhere after the
+    // previous one, which is what makes `*` a wildcard.
+    let at: number;
+    if (index === 0) {
+      at = path.startsWith(part) ? 0 : -1;
+    } else {
+      at = path.indexOf(part, cursor);
+    }
+    if (at === -1) {
+      return false;
+    }
+    cursor = at + part.length;
+  }
+  // A wildcard just before `$` (`/docs*$`) absorbs the rest of the path, so
+  // the anchor is already satisfied by any prefix match.
+  return anchored && !pattern.endsWith("*") ? cursor === path.length : true;
+};
+
+/**
+ * robots.txt: is it there, is it well-formed, does it point at the sitemap, and
+ * — the one that matters — does it block a page the sitemap is advertising?
+ *
+ * Ahrefs also tracks "robots.txt has too many redirects". A static host serves
+ * the file directly, so that is effectively unreachable here and isn't checked.
+ */
+export const robotsChecks: CheckModule = {
+  category: "robots",
+  run(context) {
+    const { robots } = context;
+    const { site } = context.project.config.deployment;
+
+    if (!context.project.config.seo.robots) {
+      return [];
+    }
+
+    if (!robots) {
+      return [
+        finding(
+          "BLUME_AUDIT_ROBOTS_MISSING",
+          { url: "/robots.txt" },
+          "The build has no robots.txt."
+        ),
+      ];
+    }
+
+    const found: Diagnostic[] = robots.invalid.map((line) =>
+      finding(
+        "BLUME_AUDIT_ROBOTS_INVALID",
+        { file: robots.file, line: line.line, url: "/robots.txt" },
+        `robots.txt line ${line.line} is not a directive: "${line.text}"`
+      )
+    );
+
+    if (site && robots.sitemaps.length === 0) {
+      found.push(
+        finding(
+          "BLUME_AUDIT_ROBOTS_SITEMAP_MISSING",
+          { file: robots.file, url: "/robots.txt" },
+          "robots.txt does not declare a Sitemap."
+        )
+      );
+    }
+
+    // A page can't be both blocked from crawling and advertised for indexing.
+    // Checking the disallow rules against the sitemap (rather than against every
+    // built file) keeps this to the pages the site actually wants indexed.
+    for (const loc of context.sitemap?.urls ?? []) {
+      let pathname: string;
+      try {
+        ({ pathname } = new URL(loc));
+      } catch {
+        continue;
+      }
+      // Match the pathname as served: robots.txt rules are literal prefixes,
+      // so `Disallow: /page/` must see the trailing slash to match.
+      const path = normalizePath(pathname);
+      const rule = robots.disallow.find((entry) =>
+        disallowMatches(entry, pathname)
+      );
+      if (rule) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_ROBOTS_DISALLOWS_INDEXABLE",
+            { file: robots.file, url: path },
+            `robots.txt "Disallow: ${rule}" blocks ${path}, which sitemap.xml advertises.`
+          )
+        );
+      }
+    }
+
+    return found;
+  },
+  tier: "static",
+};

package/src/audit/checks/sitemap.ts

@@ -0,0 +1,229 @@
+import { normalizeBasePath, stripBasePath } from "../../core/base-path.ts";
+import type { Diagnostic } from "../../core/types.ts";
+import { finding } from "../catalog.ts";
+import { pageSite } from "../locate.ts";
+import type { AuditContext, CheckModule } from "../types.ts";
+import { normalizePath, siteOrigin } from "../url.ts";
+
+const MAX_SITEMAP_BYTES = 50 * 1024 * 1024;
+const MAX_SITEMAP_URLS = 50_000;
+
+/**
+ * Slack for future `<lastmod>` values. A date-only stamp written in a timezone
+ * ahead of UTC parses as "tomorrow" from behind it; a day of grace keeps that
+ * from being reported as a lie.
+ */
+const LASTMOD_SLACK_MS = 24 * 60 * 60 * 1000;
+
+/** Error routes are never crawlable destinations, so they belong out of the sitemap. */
+const ERROR_ROUTES = new Set(["/404", "/500"]);
+
+/** Site paths listed in the sitemap, normalized for comparison against page URLs. */
+const sitemapPaths = (context: AuditContext): Map<string, string> => {
+  const paths = new Map<string, string>();
+  for (const loc of context.sitemap?.urls ?? []) {
+    try {
+      paths.set(normalizePath(new URL(loc).pathname), loc);
+    } catch {
+      // A malformed <loc> is reported by SITEMAP_INVALID, not here.
+    }
+  }
+  return paths;
+};
+
+/** The path of a canonical URL, or null when it isn't parseable (CANONICAL_BAD_TARGET reports that). */
+const canonicalPath = (canonical: string): string | null => {
+  try {
+    return normalizePath(new URL(canonical).pathname);
+  } catch {
+    return null;
+  }
+};
+
+/** Validate one `<loc>` against the build: does it exist, and is it indexable? */
+const checkListedUrl = (
+  context: AuditContext,
+  loc: string,
+  origin: string | null,
+  file: string
+): Diagnostic[] => {
+  let parsed: URL;
+  try {
+    parsed = new URL(loc);
+  } catch {
+    return [
+      finding(
+        "BLUME_AUDIT_SITEMAP_INVALID",
+        { file, url: "/sitemap.xml" },
+        `sitemap.xml lists "${loc}", which is not a valid absolute URL.`
+      ),
+    ];
+  }
+
+  if (origin && parsed.origin !== origin) {
+    return [
+      finding(
+        "BLUME_AUDIT_SITEMAP_OUT_OF_SCOPE",
+        { file, url: loc },
+        `sitemap.xml lists ${loc}, which is on another origin.`
+      ),
+    ];
+  }
+
+  // `<loc>`s carry the deployment base; page URLs (from the file tree) don't.
+  const path = normalizePath(
+    stripBasePath(
+      normalizeBasePath(context.project.config.deployment.base),
+      parsed.pathname
+    )
+  );
+  const page = context.byUrl.get(path);
+  if (!page) {
+    const redirect = context.redirects.find(
+      (entry) => normalizePath(entry.from) === path
+    );
+    return [
+      finding(
+        "BLUME_AUDIT_SITEMAP_BAD_URL",
+        { file, url: loc },
+        redirect
+          ? `sitemap.xml lists ${path}, which redirects to ${redirect.to}.`
+          : `sitemap.xml lists ${path}, which the build does not serve.`
+      ),
+    ];
+  }
+
+  const found: Diagnostic[] = [];
+  if (!page.indexable) {
+    found.push(
+      finding(
+        "BLUME_AUDIT_NOINDEX_IN_SITEMAP",
+        pageSite(context, page, ["noindex"]),
+        `${path} is in the sitemap but declares robots "${page.robots}".`
+      )
+    );
+  }
+
+  const canonical = page.canonical && canonicalPath(page.canonical);
+  if (canonical && canonical !== path) {
+    found.push(
+      finding(
+        "BLUME_AUDIT_NON_CANONICAL_IN_SITEMAP",
+        pageSite(context, page, ["seo", "canonical"]),
+        `${path} is in the sitemap but canonicalizes to ${canonical}.`
+      )
+    );
+  }
+  return found;
+};
+
+/**
+ * The sitemap, cross-checked against what was actually built.
+ *
+ * The highest-value check here is the one Ahrefs buries at info severity:
+ * `INDEXABLE_PAGE_NOT_IN_SITEMAP`. A page that a stray `draft: true` or
+ * `sidebar.hidden` quietly kept out of the sitemap is invisible to search, and
+ * nothing else in the toolchain tells you.
+ */
+export const sitemapChecks: CheckModule = {
+  category: "sitemap",
+  run(context) {
+    const { sitemap } = context;
+    const { site } = context.project.config.deployment;
+
+    // Without `deployment.site` Blume can't emit a sitemap at all (absolute URLs
+    // are required), and that's a config choice, not a defect. Stay quiet.
+    if (!(site && context.project.config.seo.sitemap)) {
+      return [];
+    }
+
+    if (!sitemap) {
+      return [
+        finding(
+          "BLUME_AUDIT_SITEMAP_INVALID",
+          { url: "/sitemap.xml" },
+          "The build has no sitemap.xml.",
+          "Set `seo.sitemap: true` and `deployment.site` in blume.config.ts."
+        ),
+      ];
+    }
+
+    const found: Diagnostic[] = [];
+    if (sitemap.error) {
+      found.push(
+        finding(
+          "BLUME_AUDIT_SITEMAP_INVALID",
+          { file: sitemap.file, url: "/sitemap.xml" },
+          `sitemap.xml is not a valid urlset: ${sitemap.error}`
+        )
+      );
+      return found;
+    }
+
+    if (
+      sitemap.bytes > MAX_SITEMAP_BYTES ||
+      sitemap.urls.length > MAX_SITEMAP_URLS
+    ) {
+      found.push(
+        finding(
+          "BLUME_AUDIT_SITEMAP_TOO_LARGE",
+          { file: sitemap.file, url: "/sitemap.xml" },
+          `sitemap.xml holds ${sitemap.urls.length} URLs in ${Math.round(sitemap.bytes / 1024 / 1024)} MB.`
+        )
+      );
+    }
+
+    // A `<lastmod>` that lies — malformed, or claiming the future — teaches
+    // search engines to distrust every lastmod in the file, which throws away
+    // the recrawl-priority signal the field exists to provide.
+    for (const [loc, lastmod] of sitemap.lastmod ?? []) {
+      const time = Date.parse(lastmod);
+      if (Number.isNaN(time)) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_SITEMAP_LASTMOD_INVALID",
+            { file: sitemap.file, url: loc },
+            `sitemap.xml gives ${loc} a lastmod of "${lastmod}", which is not a valid W3C date.`
+          )
+        );
+      } else if (time > Date.now() + LASTMOD_SLACK_MS) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_SITEMAP_LASTMOD_INVALID",
+            { file: sitemap.file, url: loc },
+            `sitemap.xml gives ${loc} a lastmod of ${lastmod}, which is in the future.`
+          )
+        );
+      }
+    }
+
+    const origin = siteOrigin(site);
+    const listed = sitemapPaths(context);
+
+    for (const loc of sitemap.urls) {
+      found.push(...checkListedUrl(context, loc, origin, sitemap.file));
+    }
+
+    // The other direction: a page that was built, is indexable, and should be
+    // findable — but never made it into the sitemap.
+    for (const page of context.pages) {
+      if (
+        !page.indexable ||
+        ERROR_ROUTES.has(page.url) ||
+        listed.has(normalizePath(page.url))
+      ) {
+        continue;
+      }
+      found.push(
+        finding(
+          "BLUME_AUDIT_INDEXABLE_PAGE_NOT_IN_SITEMAP",
+          pageSite(context, page),
+          `${page.url} is built and indexable but is not listed in sitemap.xml.`
+        )
+      );
+    }
+
+    return found;
+  },
+  tier: "static",
+};

package/src/audit/checks/social.ts

@@ -0,0 +1,238 @@
+import type { Diagnostic } from "../../core/types.ts";
+import { finding } from "../catalog.ts";
+import { pageSite } from "../locate.ts";
+import type { CheckModule, PageSnapshot } from "../types.ts";
+import { normalizePath } from "../url.ts";
+
+/** The Open Graph properties a share card is unusable without. */
+const OG_REQUIRED = ["og:title", "og:type", "og:description"];
+
+/**
+ * `og:url` must be an absolute URL, so Blume can only emit it once
+ * `deployment.site` is known. Requiring it on a site that hasn't set one would
+ * report the same missing config on every page; SITE_NOT_SET says it once.
+ */
+const ogRequired = (hasSite: boolean): string[] =>
+  hasSite ? [...OG_REQUIRED, "og:url"] : OG_REQUIRED;
+
+/**
+ * Open Graph and X (Twitter) card tags.
+ *
+ * Ahrefs splits each of these into "missing" and "incomplete". That's a
+ * distinction without a difference — a card with no tags and a card with half
+ * its tags both fail to render — so each is one check that names exactly which
+ * properties are absent.
+ */
+export const socialChecks: CheckModule = {
+  category: "social",
+  run(context) {
+    const found: Diagnostic[] = [];
+    const hasSite = Boolean(context.project.config.deployment.site);
+    const required = ogRequired(hasSite);
+
+    for (const page of context.pages) {
+      // Error pages are never shared, so their cards don't matter.
+      if (!page.indexable) {
+        continue;
+      }
+
+      const missing = required.filter((property) => !page.og[property]);
+      if (missing.length > 0) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_OG_INCOMPLETE",
+            pageSite(context, page, ["description"]),
+            missing.length === required.length
+              ? "Page has no Open Graph tags."
+              : `Open Graph is missing ${missing.join(", ")}.`
+          )
+        );
+      }
+
+      // The generated OG card is served from an absolute URL, so it too depends
+      // on `deployment.site`; SITE_NOT_SET already covers that case.
+      if (hasSite && !page.og["og:image"]) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_OG_IMAGE_MISSING",
+            pageSite(context, page, ["seo", "image"]),
+            "Page has no og:image — shares will render without a preview card."
+          )
+        );
+      }
+
+      const ogUrl = page.og["og:url"];
+      if (ogUrl && page.canonical && ogUrl !== page.canonical) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_OG_URL_MISMATCH",
+            pageSite(context, page),
+            `og:url is ${ogUrl} but the canonical is ${page.canonical}.`
+          )
+        );
+      }
+
+      // X reads title/description/image from the Open Graph tags, so the only
+      // thing it can't infer is the card type and the account attribution.
+      if (!page.twitter["twitter:card"]) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_TWITTER_CARD_INCOMPLETE",
+            pageSite(context, page),
+            "Page has no twitter:card — X will render a plain link, not a card."
+          )
+        );
+      }
+    }
+    return found;
+  },
+  tier: "static",
+};
+
+/**
+ * What's missing from one JSON-LD block.
+ *
+ * A block may either be a single node (`{@context, @type, …}`) or a `@graph`
+ * container (`{@context, @graph: [{@type, …}, …]}`), which is the shape Blume
+ * itself emits. In the container form `@context` is declared once at the root
+ * and each entry carries its own `@type` — so demanding `@type` on the root, or
+ * `@context` on each entry, would flag perfectly valid structured data.
+ */
+const jsonLdProblems = (node: unknown): string[] => {
+  if (typeof node !== "object" || node === null) {
+    return ["it is not an object"];
+  }
+  const record = node as Record<string, unknown>;
+  const problems: string[] = [];
+  if (!record["@context"]) {
+    problems.push("@context");
+  }
+
+  const graph = record["@graph"];
+  if (Array.isArray(graph)) {
+    const untyped = graph.filter(
+      (entry) =>
+        typeof entry !== "object" ||
+        entry === null ||
+        !(entry as Record<string, unknown>)["@type"]
+    ).length;
+    if (untyped > 0) {
+      problems.push(`@type on ${untyped} of its ${graph.length} @graph nodes`);
+    }
+  } else if (!record["@type"]) {
+    problems.push("@type");
+  }
+
+  return problems;
+};
+
+/** Structured data. We validate what Blume emits, and don't pretend to do more. */
+export const structuredDataChecks: CheckModule = {
+  category: "structured-data",
+  run(context) {
+    const found: Diagnostic[] = [];
+    for (const page of context.pages) {
+      for (const error of page.jsonldErrors) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_JSONLD_INVALID",
+            pageSite(context, page),
+            `A JSON-LD block failed to parse: ${error}`
+          )
+        );
+      }
+
+      for (const node of page.jsonld) {
+        const problems = jsonLdProblems(node);
+        if (problems.length > 0) {
+          found.push(
+            finding(
+              "BLUME_AUDIT_JSONLD_INCOMPLETE",
+              pageSite(context, page),
+              `A JSON-LD block is missing ${problems.join(" and ")}.`
+            )
+          );
+        }
+      }
+    }
+    return found;
+  },
+  tier: "static",
+};
+
+/** What makes a slug untidy, with the human name for each offense. */
+const URL_STYLE: { name: string; test: RegExp }[] = [
+  { name: "uppercase letters", test: /[A-Z]/u },
+  { name: "underscores", test: /_/u },
+  { name: "spaces", test: /%20| /u },
+];
+
+/** A protocol-relative URL names a dotted host (`//cdn.example.com/x`). */
+const DOTTED_HOST = /^\/\/[^/]*\./u;
+
+/**
+ * Whether an href carries a doubled slash from a trailing-slash `basePath` /
+ * `deployment.base`. `//docs/x` is the telltale: a browser reads it as a
+ * protocol-relative URL with host `docs`, so the link silently leaves the site
+ * — and every link checker skips it as external. An interior `//` (`/docs//x`)
+ * is the same mistake composed mid-path.
+ */
+const doubledSlash = (href: string): boolean => {
+  if (href.startsWith("//")) {
+    return !DOTTED_HOST.test(href);
+  }
+  if (!href.startsWith("/")) {
+    return false;
+  }
+  // Only the path — a query param may legitimately carry a URL.
+  const [path] = href.split(/[?#]/u);
+  return (path ?? "").includes("//");
+};
+
+export const urlChecks: CheckModule = {
+  category: "links",
+  run(context) {
+    const found: Diagnostic[] = [];
+    /** Doubled hrefs are site-wide (a bad base) — one finding per target. */
+    const doubled = new Map<string, PageSnapshot>();
+    for (const page of context.pages) {
+      if (page.url.includes("//")) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_DOUBLE_SLASH_URL",
+            pageSite(context, page),
+            `URL ${normalizePath(page.url)} contains a double slash.`
+          )
+        );
+      }
+      for (const link of page.links) {
+        const href = link.href.trim();
+        if (doubledSlash(href) && !doubled.has(href)) {
+          doubled.set(href, page);
+        }
+      }
+
+      const untidy = URL_STYLE.filter((style) => style.test.test(page.url));
+      if (untidy.length > 0) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_URL_STYLE",
+            pageSite(context, page),
+            `URL ${page.url} contains ${untidy.map((style) => style.name).join(" and ")}.`
+          )
+        );
+      }
+    }
+    for (const [href, page] of doubled) {
+      found.push(
+        finding(
+          "BLUME_AUDIT_DOUBLE_SLASH_URL",
+          pageSite(context, page),
+          `Link to ${href} contains a double slash.`
+        )
+      );
+    }
+    return found;
+  },
+  tier: "static",
+};