blume 1.0.4 → 1.1.0

package/src/audit/checks/llms.ts

@@ -0,0 +1,135 @@
+import { normalizeBasePath, stripBasePath } from "../../core/base-path.ts";
+import type { Diagnostic } from "../../core/types.ts";
+import { finding } from "../catalog.ts";
+import { pageSite } from "../locate.ts";
+import { ERROR_ROUTES } from "../types.ts";
+import type { AuditContext, CheckModule } from "../types.ts";
+import { normalizePath, siteOrigin } from "../url.ts";
+
+/** The `ai.llmsTxt` config normalized to what the checks need. */
+const llmsConfig = (
+  context: AuditContext
+): { enabled: boolean; openapi: boolean } => {
+  const value = context.project.config.ai?.llmsTxt;
+  if (typeof value === "object" && value !== null) {
+    return { enabled: value.enabled, openapi: value.openapi };
+  }
+  return { enabled: value !== false, openapi: true };
+};
+
+/**
+ * An llms.txt link target reduced to a site path, or null when it's off-site.
+ * Entry URLs carry the deployment base; page URLs (from the file tree) don't.
+ */
+const entryPath = (
+  url: string,
+  origin: string | null,
+  deployBase: string
+): string | null => {
+  if (/^https?:\/\//iu.test(url)) {
+    try {
+      const parsed = new URL(url);
+      if (origin && parsed.origin !== origin) {
+        return null;
+      }
+      return normalizePath(
+        stripBasePath(deployBase, decodeURI(parsed.pathname))
+      );
+    } catch {
+      return null;
+    }
+  }
+  if (!url.startsWith("/")) {
+    return null;
+  }
+  try {
+    return normalizePath(stripBasePath(deployBase, decodeURI(url)));
+  } catch {
+    return normalizePath(stripBasePath(deployBase, url));
+  }
+};
+
+/**
+ * The `llms.txt` index, held to the sitemap's standard.
+ *
+ * SEO crawlers audit for Google and stop there. But Blume's promise is
+ * AI-ready docs, and `llms.txt` is the sitemap of that surface: a stale entry
+ * sends an agent to a page that is not there, and an unlisted page is
+ * invisible to every tool that starts from the index.
+ */
+export const llmsChecks: CheckModule = {
+  category: "ai",
+  run(context) {
+    const { enabled, openapi } = llmsConfig(context);
+    if (!enabled) {
+      return [];
+    }
+
+    const { llms } = context;
+    if (!llms) {
+      return [
+        finding(
+          "BLUME_AUDIT_LLMS_TXT_MISSING",
+          { url: "/llms.txt" },
+          "The build has no llms.txt."
+        ),
+      ];
+    }
+
+    const found: Diagnostic[] = [];
+    const origin = siteOrigin(context.project.config.deployment.site);
+    const deployBase = normalizeBasePath(
+      context.project.config.deployment.base
+    );
+
+    const listed = new Set<string>();
+    for (const entry of llms.entries) {
+      const path = entryPath(entry.url, origin, deployBase);
+      if (path === null) {
+        // Off-site or unparseable targets aren't pages this build can vouch
+        // for; external link health is the `--external` tier's job.
+        continue;
+      }
+      listed.add(path);
+      if (!context.byUrl.has(path)) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_LLMS_TXT_STALE_ENTRY",
+            { file: llms.file, line: entry.line, url: entry.url },
+            `llms.txt lists ${path}, which the build does not serve.`
+          )
+        );
+      }
+    }
+
+    // The reverse direction, mirroring INDEXABLE_PAGE_NOT_IN_SITEMAP: a page
+    // that is built, indexable, and in the nav belongs in the index. Pages the
+    // generator deliberately skips — hidden, drafts, error routes, API
+    // reference when `ai.llmsTxt.openapi` is off, custom pages with no
+    // manifest route — are skipped here for the same reasons.
+    for (const page of context.pages) {
+      const { route } = page;
+      if (
+        !route ||
+        route.hidden ||
+        route.draft ||
+        !page.indexable ||
+        ERROR_ROUTES.has(page.url) ||
+        (!openapi && route.source.name === "openapi") ||
+        listed.has(normalizePath(page.url))
+      ) {
+        continue;
+      }
+      found.push(
+        finding(
+          "BLUME_AUDIT_LLMS_TXT_PAGE_MISSING",
+          pageSite(context, page),
+          `${page.url} is built and indexable but is not listed in llms.txt.`
+        )
+      );
+    }
+
+    return found;
+  },
+  tier: "static",
+};

package/src/audit/checks/network.ts

@@ -0,0 +1,272 @@
+import { normalizeBasePath } from "../../core/base-path.ts";
+import { gradeExternal, probeAll } from "../../core/probe.ts";
+import type { ProbeResult } from "../../core/probe.ts";
+import type { Diagnostic } from "../../core/types.ts";
+import { finding } from "../catalog.ts";
+import { pageSite } from "../locate.ts";
+import type { AuditContext, CheckModule, PageSnapshot } from "../types.ts";
+import { resolveHref, siteOrigin } from "../url.ts";
+
+const CLIENT_ERROR = 400;
+const SERVER_ERROR = 500;
+/** Past this, a page is slow enough that it costs you crawl budget and readers. */
+const SLOW_MS = 1500;
+
+/** The live URL a built page is served at, under the `--url` origin. */
+const liveUrl = (origin: string, page: PageSnapshot): string =>
+  new URL(page.url, origin).toString();
+
+/**
+ * Whether the response failed outright, and how. Null when the page is served.
+ *
+ * Exported so the grading can be unit-tested against synthetic responses: a
+ * timeout, an HTTPS-to-HTTP downgrade, and a slow first byte are all awkward to
+ * provoke from a local test server, and faking the network to test them would
+ * only be testing the fake.
+ */
+export const badResponse = (
+  context: AuditContext,
+  page: PageSnapshot,
+  result: ProbeResult
+): Diagnostic | null => {
+  const site = pageSite(context, page);
+
+  if (result.timedOut) {
+    return finding(
+      "BLUME_AUDIT_HTTP_TIMEOUT",
+      site,
+      `${page.url} did not respond within the timeout.`
+    );
+  }
+
+  const { status } = result;
+  if (status !== undefined && status >= SERVER_ERROR) {
+    return finding(
+      "BLUME_AUDIT_HTTP_5XX",
+      site,
+      `${page.url} returned HTTP ${status}.`
+    );
+  }
+  if (status !== undefined && status >= CLIENT_ERROR) {
+    return finding(
+      "BLUME_AUDIT_HTTP_4XX",
+      site,
+      `${page.url} is in the build but returned HTTP ${status}.`
+    );
+  }
+  if (!result.ok) {
+    return finding(
+      "BLUME_AUDIT_HTTP_5XX",
+      site,
+      `${page.url} is unreachable: ${result.error ?? "no response"}.`
+    );
+  }
+  return null;
+};
+
+/** What a *successful* response still gets wrong: headers, timing, protocol. */
+export const servedPageChecks = (
+  context: AuditContext,
+  page: PageSnapshot,
+  result: ProbeResult,
+  origin: string
+): Diagnostic[] => {
+  const site = pageSite(context, page);
+  const found: Diagnostic[] = [];
+
+  // A page served over HTTPS that redirects down to HTTP hands the reader to an
+  // insecure connection — the opposite of the usual upgrade.
+  if (
+    result.redirected &&
+    result.finalUrl?.startsWith("http://") &&
+    origin.startsWith("https://")
+  ) {
+    found.push(
+      finding(
+        "BLUME_AUDIT_REDIRECT_TO_HTTP",
+        site,
+        `${page.url} redirects to ${result.finalUrl}, downgrading to HTTP.`
+      )
+    );
+  }
+
+  if (!result.encoding) {
+    found.push(
+      finding(
+        "BLUME_AUDIT_NOT_COMPRESSED",
+        site,
+        `${page.url} is served without gzip or brotli compression.`
+      )
+    );
+  }
+
+  if (result.ms !== undefined && result.ms > SLOW_MS) {
+    found.push(
+      finding(
+        "BLUME_AUDIT_SLOW_RESPONSE",
+        site,
+        `${page.url} took ${result.ms}ms to respond.`
+      )
+    );
+  }
+
+  // The header wins over the meta tag, so a stray `X-Robots-Tag: noindex` —
+  // Vercel sets one on password-protected and preview deploys — silently
+  // deindexes a page whose HTML looks perfectly indexable.
+  const tag = result.robotsTag;
+  if (tag?.includes("noindex") && page.indexable) {
+    found.push(
+      finding(
+        "BLUME_AUDIT_ROBOTS_HEADER_CONFLICT",
+        site,
+        `${page.url} sends "X-Robots-Tag: ${tag}" but its HTML has no noindex — the header wins.`
+      )
+    );
+  }
+
+  return found;
+};
+
+/**
+ * The built site, checked against a live deployment.
+ *
+ * Everything here needs the network, which is why it only runs with `--url`: a
+ * page that exists in `dist/` can still 404 in production behind a bad rewrite,
+ * and only the real response carries the headers (`Content-Encoding`,
+ * `X-Robots-Tag`) that decide whether the page is compressed and indexable.
+ */
+export const networkChecks: CheckModule = {
+  category: "network",
+  async run(context) {
+    const { origin } = context;
+    if (!origin) {
+      return [];
+    }
+
+    const found: Diagnostic[] = [];
+    const targets = context.pages.map((page) => liveUrl(origin, page));
+    // robots.txt and sitemap.xml are fetched alongside the pages: they're the
+    // two files a crawler asks for first, and a deploy that hides them silently
+    // undoes everything else the audit checks.
+    const robotsUrl = new URL("/robots.txt", origin).toString();
+    const sitemapUrl = new URL("/sitemap.xml", origin).toString();
+
+    const results = await probeAll([...targets, robotsUrl, sitemapUrl]);
+
+    for (const page of context.pages) {
+      const result = results.get(liveUrl(origin, page));
+      if (!result) {
+        continue;
+      }
+      const failure = badResponse(context, page, result);
+      if (failure) {
+        found.push(failure);
+        continue;
+      }
+      found.push(...servedPageChecks(context, page, result, origin));
+    }
+
+    const robots = results.get(robotsUrl);
+    if (robots && !robots.ok) {
+      found.push(
+        finding(
+          "BLUME_AUDIT_ROBOTS_NOT_ACCESSIBLE",
+          { url: "/robots.txt" },
+          `robots.txt is not reachable at ${robotsUrl}.`
+        )
+      );
+    }
+
+    const sitemap = results.get(sitemapUrl);
+    if (context.sitemap && sitemap && !sitemap.ok) {
+      found.push(
+        finding(
+          "BLUME_AUDIT_SITEMAP_NOT_ACCESSIBLE",
+          { url: "/sitemap.xml" },
+          `sitemap.xml is in the build but is not reachable at ${sitemapUrl}.`
+        )
+      );
+    }
+
+    return found;
+  },
+  tier: "network",
+};
+
+/**
+ * Outbound links, probed over the network (`--external`).
+ *
+ * Severity is graded rather than flat: a 404 is the author's bug, but a 403 or a
+ * 5xx is usually rate limiting or someone else's outage, and failing a build on
+ * that would make the check useless.
+ */
+export const externalChecks: CheckModule = {
+  category: "network",
+  async run(context) {
+    const origin = siteOrigin(context.project.config.deployment.site);
+    const deployBase = normalizeBasePath(
+      context.project.config.deployment.base
+    );
+
+    /** Every outbound URL, and the pages that link to it. */
+    const linkers = new Map<string, PageSnapshot[]>();
+    for (const page of context.pages) {
+      for (const link of page.links) {
+        const resolved = resolveHref(page.url, link.href, origin, deployBase);
+        if (resolved.kind !== "external") {
+          continue;
+        }
+        const pages = linkers.get(resolved.url);
+        if (pages) {
+          if (!pages.includes(page)) {
+            pages.push(page);
+          }
+        } else {
+          linkers.set(resolved.url, [page]);
+        }
+      }
+    }
+
+    if (linkers.size === 0) {
+      return [];
+    }
+    const results = await probeAll([...linkers.keys()]);
+
+    const found: Diagnostic[] = [];
+    for (const [url, pages] of linkers) {
+      const result = results.get(url);
+      if (!result) {
+        continue;
+      }
+      const site = pageSite(context, pages[0] as PageSnapshot);
+
+      const grade = gradeExternal(result);
+      if (grade) {
+        // Severity is graded, not flat. A 404 is a bug the author can fix; a 403
+        // or 5xx is usually rate limiting or someone else's outage, and failing
+        // a build on that would get `--external` switched off for good.
+        found.push({
+          ...finding(
+            "BLUME_AUDIT_EXTERNAL_LINK_BROKEN",
+            site,
+            `${url} is unreachable (${grade.detail}), linked from ${pages.length} page(s).`
+          ),
+          severity: grade.severity,
+        });
+        continue;
+      }
+
+      if (result.redirected && result.finalUrl && result.finalUrl !== url) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_EXTERNAL_LINK_REDIRECT",
+            site,
+            `${url} redirects to ${result.finalUrl}.`
+          )
+        );
+      }
+    }
+    return found;
+  },
+  tier: "external",
+};

package/src/audit/checks/og-image.ts

@@ -0,0 +1,113 @@
+import { readFile } from "node:fs/promises";
+
+import { join } from "pathe";
+
+import { normalizeBasePath } from "../../core/base-path.ts";
+import type { Diagnostic } from "../../core/types.ts";
+import { finding } from "../catalog.ts";
+import { imageSize } from "../image-size.ts";
+import type { ImageSize } from "../image-size.ts";
+import { pageSite } from "../locate.ts";
+import type { CheckModule, PageSnapshot } from "../types.ts";
+import { resolveHref, siteOrigin } from "../url.ts";
+
+/**
+ * Below this, large social cards render the image blurry or fall back to a
+ * small-card layout: Facebook's floor for a full-width card is 600×315
+ * (1200×630 recommended), X's for `summary_large_image` is 300×157.
+ */
+const MIN_WIDTH = 600;
+const MIN_HEIGHT = 315;
+
+/**
+ * The Open Graph image as bytes, not just a tag. `OG_IMAGE_MISSING` proves the
+ * meta tag exists; only the build can prove the tag points at a real file of a
+ * shareable size — a crawler needs the live site for either.
+ */
+export const ogImageChecks: CheckModule = {
+  category: "social",
+  async run(context) {
+    const found: Diagnostic[] = [];
+    /** Pending reads per file path — the site-wide default OG image is read once. */
+    const sizes = new Map<string, Promise<ImageSize | null>>();
+
+    const read = async (path: string): Promise<ImageSize | null> => {
+      try {
+        return imageSize(await readFile(join(context.staticDir, path)));
+      } catch {
+        // Unreadable bytes: existence was already established via the file
+        // index, and an unknown format is not a finding.
+        return null;
+      }
+    };
+
+    // Cache the promise, not the result, so concurrent pages sharing one
+    // image (the site-wide default) share a single read.
+    const measure = (path: string): Promise<ImageSize | null> => {
+      const cached = sizes.get(path);
+      if (cached) {
+        return cached;
+      }
+      const pending = read(path);
+      sizes.set(path, pending);
+      return pending;
+    };
+
+    const origin = siteOrigin(context.project.config.deployment.site);
+    const deployBase = normalizeBasePath(
+      context.project.config.deployment.base
+    );
+    const candidates: { page: PageSnapshot; path: string }[] = [];
+
+    for (const page of context.pages) {
+      const src = page.og["og:image"];
+      if (!src) {
+        continue;
+      }
+      // An og:image on another origin (a CDN, an external host) is outside the
+      // build — its existence can't be proven without the network, so it's the
+      // network tier's business, not this check's.
+      const resolved = resolveHref(page.url, src, origin, deployBase);
+      if (resolved.kind === "external" || resolved.kind === "ignored") {
+        continue;
+      }
+
+      const { path } = resolved;
+      if (
+        !(context.files.has(path) || context.files.has(`${path}/index.html`))
+      ) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_OG_IMAGE_BROKEN",
+            pageSite(context, page, ["seo", "image"]),
+            `og:image points at ${path}, which is not in the build.`
+          )
+        );
+        continue;
+      }
+
+      candidates.push({ page, path });
+    }
+
+    const measured = await Promise.all(
+      candidates.map(async (candidate) => ({
+        ...candidate,
+        size: await measure(candidate.path),
+      }))
+    );
+    for (const { page, path, size } of measured) {
+      if (size && (size.width < MIN_WIDTH || size.height < MIN_HEIGHT)) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_OG_IMAGE_SMALL",
+            pageSite(context, page, ["seo", "image"]),
+            `og:image ${path} is ${size.width}×${size.height} — below the ${MIN_WIDTH}×${MIN_HEIGHT} floor for large social cards.`
+          )
+        );
+      }
+    }
+
+    return found;
+  },
+  tier: "static",
+};

package/src/audit/checks/redirects.ts

@@ -0,0 +1,87 @@
+import type { Diagnostic } from "../../core/types.ts";
+import { finding } from "../catalog.ts";
+import { pageSite } from "../locate.ts";
+import type { CheckModule } from "../types.ts";
+import { normalizePath } from "../url.ts";
+
+/**
+ * Configured redirects, walked through to their destinations.
+ *
+ * Ahrefs also lists "3XX redirect" and "302 redirect" as issues. Having a
+ * redirect is inventory, not a defect, and "HTTP to HTTPS redirect" — which it
+ * also flags — is correct behavior. None of those are reported. What is: a
+ * redirect that loops, dead-ends, takes the long way round, or is shadowed by a
+ * real page and therefore never fires at all.
+ */
+export const redirectChecks: CheckModule = {
+  category: "redirects",
+  run(context) {
+    const found: Diagnostic[] = [];
+    const configFile = context.project.context.configFile ?? undefined;
+    const site = { file: configFile, url: "/" };
+
+    for (const redirect of context.redirects) {
+      const from = normalizePath(redirect.from);
+
+      // A redirect whose source is also a built page never fires — the page
+      // wins. Ahrefs can't see this: it only observes the served response, which
+      // looks perfectly healthy.
+      if (context.byUrl.has(from)) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_REDIRECT_SOURCE_IS_PAGE",
+            { ...site, url: from },
+            `A redirect is configured from ${from}, but ${from} is also a real page — the redirect never fires.`
+          )
+        );
+        continue;
+      }
+
+      if (redirect.outcome === "loop") {
+        found.push(
+          finding(
+            "BLUME_AUDIT_REDIRECT_LOOP",
+            { ...site, url: from },
+            `Redirect loop: ${redirect.chain.join(" → ")}`
+          )
+        );
+      } else if (redirect.outcome === "broken") {
+        found.push(
+          finding(
+            "BLUME_AUDIT_REDIRECT_BROKEN",
+            { ...site, url: from },
+            `Redirect from ${from} lands on ${redirect.chain.at(-1)}, which the build does not serve.`
+          )
+        );
+      } else if (redirect.outcome === "chain") {
+        const hops = redirect.chain.length - 1;
+        const severity =
+          hops > context.thresholds.maxRedirectHops ? " (too long)" : "";
+        found.push(
+          finding(
+            "BLUME_AUDIT_REDIRECT_CHAIN",
+            { ...site, url: from },
+            `Redirect from ${from} passes through ${hops} hops${severity}: ${redirect.chain.join(" → ")}`
+          )
+        );
+      }
+    }
+
+    // A meta refresh is a redirect the framework doesn't know about, so it can't
+    // be validated, cached, or followed reliably by crawlers.
+    for (const page of context.pages) {
+      if (page.metaRefresh) {
+        found.push(
+          finding(
+            "BLUME_AUDIT_META_REFRESH",
+            pageSite(context, page),
+            `Page uses a meta refresh redirect ("${page.metaRefresh}").`
+          )
+        );
+      }
+    }
+
+    return found;
+  },
+  tier: "static",
+};