blume 1.0.4 → 1.1.0

package/src/audit/crawl.ts

@@ -0,0 +1,259 @@
+import { readFile, stat } from "node:fs/promises";
+
+import { join, relative } from "pathe";
+import { glob } from "tinyglobby";
+
+import { examplesRouteBase } from "../astro/templates.ts";
+import { stripBasePath } from "../core/base-path.ts";
+import type { BlumeManifest, RouteManifestEntry } from "../core/types.ts";
+import { buildSnapshot } from "./snapshot.ts";
+import type { LlmsDoc, PageSnapshot, RobotsDoc, SitemapDoc } from "./types.ts";
+
+/**
+ * The route prefix `<Component />` preview frames live under. They are bare
+ * documents rendered for iframes — deliberately noindex, no title worth
+ * grading, no front matter to fix — so auditing them as pages only produces
+ * findings nobody can act on.
+ */
+const EXAMPLES_PREFIX = `${examplesRouteBase("")}/`;
+
+/** Everything read off disk in one pass over the built site. */
+export interface CrawlResult {
+  pages: PageSnapshot[];
+  /** Every file in the static dir: URL path -> size in bytes. */
+  files: Map<string, number>;
+  sitemap: SitemapDoc | null;
+  robots: RobotsDoc | null;
+  llms: LlmsDoc | null;
+}
+
+/**
+ * The URL a built HTML file is served at. Astro's directory build format emits
+ * `docs/api/index.html` for `/docs/api`, so the `index.html` leaf collapses;
+ * a flat `404.html` keeps its name.
+ */
+export const fileToUrl = (staticDir: string, file: string): string => {
+  const rel = relative(staticDir, file).replaceAll("\\", "/");
+  const path = rel.replace(/(?:^|\/)index\.html$/u, "").replace(/\.html$/u, "");
+  return path === "" ? "/" : `/${path}`;
+};
+
+/**
+ * Every file in the static dir, keyed the way an HTML `src`/`href` would name
+ * it, with its size — so a reference can be resolved and weighed in one pass.
+ */
+const indexFiles = async (staticDir: string): Promise<Map<string, number>> => {
+  const found = await glob("**/*", { cwd: staticDir, dot: true });
+  const sized = await Promise.all(
+    found.map(async (file) => {
+      const path = `/${file.replaceAll("\\", "/")}`;
+      const info = await stat(join(staticDir, file));
+      return [path, info.size] as const;
+    })
+  );
+  return new Map(sized);
+};
+
+/**
+ * Look a built URL up in the route manifest. Routes carry `basePath` while the
+ * built file tree may not, so both spellings are tried before giving up — a page
+ * that fails to join is still audited, it just can't name a source file to fix.
+ */
+const routeIndex = (
+  manifest: BlumeManifest,
+  basePath: string
+): Map<string, RouteManifestEntry> => {
+  const index = new Map<string, RouteManifestEntry>();
+  for (const route of manifest.routes) {
+    index.set(route.path, route);
+    const stripped = stripBasePath(basePath, route.path);
+    if (!index.has(stripped)) {
+      index.set(stripped, route);
+    }
+  }
+  return index;
+};
+
+const SITEMAP_URL = /<url>(?<block>[\s\S]*?)<\/url>/gu;
+const SITEMAP_LOC = /<loc>(?<loc>[\s\S]*?)<\/loc>/gu;
+const SITEMAP_LASTMOD = /<lastmod>(?<date>[\s\S]*?)<\/lastmod>/u;
+const XML_ENTITIES: Record<string, string> = {
+  "&amp;": "&",
+  "&apos;": "'",
+  "&gt;": ">",
+  "&lt;": "<",
+  "&quot;": '"',
+};
+
+const unescapeXml = (value: string): string =>
+  value.replaceAll(
+    /&(?:amp|apos|gt|lt|quot);/gu,
+    (entity) => XML_ENTITIES[entity] ?? entity
+  );
+
+/**
+ * Parse `sitemap.xml`. Deliberately shallow: we only need the `<loc>` list and
+ * whether the document is a well-formed urlset, and pulling in an XML parser to
+ * learn that would be a dependency for one regex.
+ */
+export const parseSitemap = (
+  file: string,
+  xml: string,
+  bytes: number
+): SitemapDoc => {
+  const doc: SitemapDoc = { bytes, file, lastmod: new Map(), urls: [] };
+  if (!xml.includes("<urlset")) {
+    doc.error = xml.includes("<sitemapindex")
+      ? "sitemap is an index, not a urlset"
+      : "no <urlset> element";
+    return doc;
+  }
+  for (const match of xml.matchAll(SITEMAP_LOC)) {
+    const loc = unescapeXml((match.groups?.loc ?? "").trim());
+    if (loc) {
+      doc.urls.push(loc);
+    }
+  }
+  // `<lastmod>` is scoped per `<url>` block so it stays attached to its `<loc>`
+  // — the flat loc scan above deliberately isn't, so a sitemap with stray text
+  // between blocks still yields its URL list.
+  for (const match of xml.matchAll(SITEMAP_URL)) {
+    const block = match.groups?.block ?? "";
+    const loc = unescapeXml(
+      (
+        new RegExp(SITEMAP_LOC.source, "u").exec(block)?.groups?.loc ?? ""
+      ).trim()
+    );
+    const lastmod = SITEMAP_LASTMOD.exec(block)?.groups?.date?.trim();
+    if (loc && lastmod) {
+      doc.lastmod?.set(loc, lastmod);
+    }
+  }
+  return doc;
+};
+
+/**
+ * Parse the `llms.txt` index into its Markdown link targets. Deliberately
+ * shallow, like {@link parseSitemap}: the checks only need "which pages does
+ * this file claim exist", not a Markdown AST.
+ */
+export const parseLlms = (file: string, text: string): LlmsDoc => {
+  const entries: LlmsDoc["entries"] = [];
+  const link = /\]\((?<url>[^)\s]+)\)/gu;
+  for (const [index, line] of text.split(/\r?\n/u).entries()) {
+    for (const match of line.matchAll(link)) {
+      const url = match.groups?.url;
+      if (url) {
+        entries.push({ line: index + 1, url });
+      }
+    }
+  }
+  return { entries, file };
+};
+
+const ROBOTS_DIRECTIVE = /^(?<field>[a-z-]+)\s*:\s*(?<value>.*)$/iu;
+
+/** Parse `robots.txt` into the directives the audit cares about. */
+export const parseRobots = (file: string, text: string): RobotsDoc => {
+  const doc: RobotsDoc = { disallow: [], file, invalid: [], sitemaps: [] };
+  // Only `User-agent: *` rules bind the crawlers we're auditing for; a block
+  // scoped to some other agent isn't a finding about our indexable pages.
+  let appliesToAll = false;
+  for (const [index, raw] of text.split(/\r?\n/u).entries()) {
+    const line = raw.trim();
+    if (line === "" || line.startsWith("#")) {
+      continue;
+    }
+    const match = ROBOTS_DIRECTIVE.exec(line);
+    if (!match) {
+      doc.invalid.push({ line: index + 1, text: line });
+      continue;
+    }
+    const field = (match.groups?.field ?? "").toLowerCase();
+    const value = (match.groups?.value ?? "").trim();
+    if (field === "user-agent") {
+      appliesToAll = value === "*";
+    } else if (field === "disallow" && appliesToAll && value) {
+      doc.disallow.push(value);
+    } else if (field === "sitemap" && value) {
+      doc.sitemaps.push(value);
+    }
+  }
+  return doc;
+};
+
+/**
+ * Whether an emitted `.html` file is a real page rather than a fragment.
+ *
+ * Astro writes standalone HTML for some components (`_home/Footer/index.html`
+ * and friends) — markup with no `<html>` or `<head>`, never served as a route.
+ * Auditing those as pages reports every one of them as missing a title, a
+ * viewport, and a `lang` attribute, which is noise about markup nobody visits.
+ * An SEO audit is about documents, so that's what we keep.
+ */
+const isDocument = (html: string): boolean => /<head[\s>]/iu.test(html);
+
+const readIfPresent = async (file: string): Promise<string | null> => {
+  try {
+    return await readFile(file, "utf-8");
+  } catch {
+    return null;
+  }
+};
+
+/**
+ * Read the built site: every HTML page reduced to a snapshot, the full file
+ * index (for resolving subresource references), plus sitemap.xml and robots.txt.
+ */
+export const crawlStaticDir = async (options: {
+  staticDir: string;
+  manifest: BlumeManifest;
+  basePath: string;
+}): Promise<CrawlResult> => {
+  const { staticDir, manifest, basePath } = options;
+  const routes = routeIndex(manifest, basePath);
+
+  const htmlFiles = await glob("**/*.html", { absolute: true, cwd: staticDir });
+  const snapshots = await Promise.all(
+    htmlFiles.toSorted().map(async (file) => {
+      const url = fileToUrl(staticDir, file);
+      if (stripBasePath(basePath, url).startsWith(EXAMPLES_PREFIX)) {
+        return null;
+      }
+      const html = await readFile(file, "utf-8");
+      if (!isDocument(html)) {
+        return null;
+      }
+      return buildSnapshot({
+        file,
+        html,
+        route: routes.get(url) ?? routes.get(stripBasePath(basePath, url)),
+        url,
+      });
+    })
+  );
+  const pages = snapshots.filter((page) => page !== null);
+
+  const sitemapFile = join(staticDir, "sitemap.xml");
+  const sitemapXml = await readIfPresent(sitemapFile);
+  const robotsFile = join(staticDir, "robots.txt");
+  const robotsTxt = await readIfPresent(robotsFile);
+  const llmsFile = join(staticDir, "llms.txt");
+  const llmsText = await readIfPresent(llmsFile);
+  const files = await indexFiles(staticDir);
+
+  return {
+    files,
+    llms: llmsText === null ? null : parseLlms(llmsFile, llmsText),
+    pages,
+    robots: robotsTxt === null ? null : parseRobots(robotsFile, robotsTxt),
+    sitemap:
+      sitemapXml === null
+        ? null
+        : parseSitemap(
+            sitemapFile,
+            sitemapXml,
+            files.get("/sitemap.xml") ?? Buffer.byteLength(sitemapXml, "utf-8")
+          ),
+  };
+};

package/src/audit/graph.ts

@@ -0,0 +1,74 @@
+import type { LinkGraph, PageSnapshot } from "./types.ts";
+import { resolveHref } from "./url.ts";
+
+const add = (
+  map: Map<string, Set<string>>,
+  key: string,
+  value: string
+): void => {
+  const existing = map.get(key);
+  if (existing) {
+    existing.add(value);
+  } else {
+    map.set(key, new Set([value]));
+  }
+};
+
+/**
+ * Build the internal link graph, keeping prose links and chrome links apart.
+ *
+ * The split is the whole point. Blume's sidebar links every navigable page from
+ * every page, so a graph that lumps the two together concludes that every page
+ * has hundreds of inbound links — which makes "orphan page" unfireable and makes
+ * a single broken sidebar link look like N broken links, one per page.
+ */
+export const buildGraph = (
+  pages: PageSnapshot[],
+  origin: string | null,
+  deployBase = ""
+): LinkGraph => {
+  const graph: LinkGraph = {
+    chromeIn: new Map(),
+    chromeOut: new Map(),
+    contentIn: new Map(),
+    contentOut: new Map(),
+  };
+
+  for (const page of pages) {
+    for (const link of page.links) {
+      const resolved = resolveHref(page.url, link.href, origin, deployBase);
+      if (resolved.kind !== "internal" && resolved.kind !== "self-origin") {
+        continue;
+      }
+      const out = link.content ? graph.contentOut : graph.chromeOut;
+      const incoming = link.content ? graph.contentIn : graph.chromeIn;
+      add(out, page.url, resolved.path);
+      add(incoming, resolved.path, page.url);
+    }
+  }
+
+  return graph;
+};
+
+/**
+ * Pages nothing links to from prose — reachable only through the sidebar.
+ *
+ * This is what an SEO means by "orphan": the page ships, but no other page's
+ * body ever points a reader (or a crawler following editorial links) at it.
+ * Non-indexable pages are excluded — a `noindex` page is *meant* to be
+ * unreachable — as is the home page, which is nobody's job to link to. Under a
+ * `basePath` the home page's built URL is the base itself, so the caller names
+ * it via `homeUrl`.
+ */
+export const orphanPages = (
+  pages: PageSnapshot[],
+  graph: LinkGraph,
+  homeUrl = "/"
+): PageSnapshot[] =>
+  pages.filter(
+    (page) =>
+      page.indexable &&
+      page.url !== "/" &&
+      page.url !== homeUrl &&
+      (graph.contentIn.get(page.url)?.size ?? 0) === 0
+  );

package/src/audit/html.ts

@@ -0,0 +1,54 @@
+import { parse } from "node-html-parser";
+import type { HTMLElement } from "node-html-parser";
+
+/**
+ * A parsed HTML document. Re-exported so the rest of the audit types against
+ * this module rather than the parser: `node-html-parser` is deliberately
+ * quarantined here, so swapping it stays a one-file change.
+ */
+export type HtmlDocument = HTMLElement;
+
+/**
+ * Parse built HTML into a queryable tree.
+ *
+ * `blockTextElements` keeps the raw text of `<script>`/`<style>`/`<pre>` intact
+ * instead of parsing it as markup — the audit reads `<script
+ * type="application/ld+json">` bodies verbatim to validate structured data.
+ */
+export const parseHtml = (html: string): HtmlDocument =>
+  parse(html, {
+    blockTextElements: { noscript: true, pre: true, script: true, style: true },
+    comment: false,
+  });
+
+/** Trimmed attribute value, or null when absent or empty. */
+export const attr = (element: HTMLElement, name: string): string | null => {
+  const value = element.getAttribute(name)?.trim();
+  return value || null;
+};
+
+/** `content` of every `<meta>` matching a selector, in document order. */
+export const metaContents = (
+  document: HtmlDocument,
+  selector: string
+): string[] =>
+  document
+    .querySelectorAll(selector)
+    .map((element) => element.getAttribute("content")?.trim() ?? "")
+    .filter((value) => value.length > 0);
+
+/**
+ * Collapse an element's visible text. Script/style bodies and code blocks are
+ * dropped: a fenced code sample isn't prose, and counting it would let a page
+ * that is 90% code pass the word-count check on the strength of its snippets.
+ */
+export const visibleText = (element: HTMLElement): string => {
+  const clone = parse(element.outerHTML, {
+    blockTextElements: { noscript: true, pre: true, script: true, style: true },
+    comment: false,
+  });
+  for (const node of clone.querySelectorAll("script, style, pre, code")) {
+    node.remove();
+  }
+  return clone.structuredText.replaceAll(/\s+/gu, " ").trim();
+};

package/src/audit/image-size.ts

@@ -0,0 +1,63 @@
+/**
+ * Pixel dimensions read straight from a PNG, JPEG, or GIF header. A dedicated
+ * image library would be a dependency for three well-documented byte layouts;
+ * anything else (SVG, WebP, AVIF) yields null and its checks simply don't run.
+ */
+export interface ImageSize {
+  width: number;
+  height: number;
+}
+
+const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
+
+const pngSize = (bytes: Buffer): ImageSize | null => {
+  // Signature, then the IHDR chunk is required to come first: width and height
+  // are big-endian u32s at fixed offsets 16 and 20.
+  if (bytes.length < 24 || !bytes.subarray(0, 4).equals(PNG_SIGNATURE)) {
+    return null;
+  }
+  return { height: bytes.readUInt32BE(20), width: bytes.readUInt32BE(16) };
+};
+
+/** JPEG start-of-frame markers (C0–CF minus DHT C4, JPG C8, DAC CC). */
+const isSof = (marker: number): boolean =>
+  marker >= 0xc0 &&
+  marker <= 0xcf &&
+  marker !== 0xc4 &&
+  marker !== 0xc8 &&
+  marker !== 0xcc;
+
+const jpegSize = (bytes: Buffer): ImageSize | null => {
+  if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) {
+    return null;
+  }
+  // Walk the segment list: each is FF <marker> <u16 length> <payload>. The
+  // dimensions live in the first start-of-frame segment's payload, as
+  // big-endian u16s after a one-byte precision field.
+  let offset = 2;
+  while (offset + 9 < bytes.length) {
+    if (bytes[offset] !== 0xff) {
+      return null;
+    }
+    const marker = bytes[offset + 1] ?? 0;
+    if (isSof(marker)) {
+      return {
+        height: bytes.readUInt16BE(offset + 5),
+        width: bytes.readUInt16BE(offset + 7),
+      };
+    }
+    offset += 2 + bytes.readUInt16BE(offset + 2);
+  }
+  return null;
+};
+
+const gifSize = (bytes: Buffer): ImageSize | null => {
+  if (bytes.length < 10 || bytes.subarray(0, 4).toString("latin1") !== "GIF8") {
+    return null;
+  }
+  return { height: bytes.readUInt16LE(8), width: bytes.readUInt16LE(6) };
+};
+
+/** The image's pixel dimensions, or null when the format isn't recognized. */
+export const imageSize = (bytes: Buffer): ImageSize | null =>
+  pngSize(bytes) ?? jpegSize(bytes) ?? gifSize(bytes);

package/src/audit/locate.ts

@@ -0,0 +1,33 @@
+import { locateFrontmatterKey } from "../core/diagnostics.ts";
+import type { FindingSite } from "./catalog.ts";
+import type { AuditContext, PageSnapshot } from "./types.ts";
+
+/**
+ * Where to report a finding about `page`: the built URL, plus — when the page
+ * came from authored content — the source file and the exact front matter line
+ * to edit.
+ *
+ * This is the feature. A crawler can only tell you that `/docs/api` has no
+ * description; Blume knows the page was built from `docs/api.mdx` and can put
+ * the cursor on the line. Pass the front matter key path a fix would touch
+ * (`["description"]`, `["seo", "canonical"]`); omit it when the key doesn't
+ * exist yet, and the finding anchors to the file instead.
+ */
+export const pageSite = (
+  context: AuditContext,
+  page: PageSnapshot,
+  key?: readonly (string | number)[]
+): FindingSite => {
+  const file = page.source;
+  if (!file) {
+    return { url: page.url };
+  }
+  const source = key ? context.sources.get(file) : undefined;
+  const position = source ? locateFrontmatterKey(source, key ?? []) : undefined;
+  return {
+    column: position?.column,
+    file,
+    line: position?.line,
+    url: page.url,
+  };
+};

package/src/audit/redirects.ts

@@ -0,0 +1,74 @@
+import type { RedirectResolution } from "./types.ts";
+import { normalizePath } from "./url.ts";
+
+interface ConfiguredRedirect {
+  from: string;
+  to: string;
+  status: number;
+}
+
+/**
+ * Follow every configured redirect through to its destination, classifying what
+ * it lands on.
+ *
+ * - `loop`   — the chain revisits a hop it has already been to. Never resolves.
+ * - `broken` — the chain ends somewhere that isn't a built page.
+ * - `chain`  — it resolves, but through at least one intermediate redirect.
+ * - `ok`     — one hop, straight to a real page.
+ *
+ * An external destination (`https://…`) is always `ok`: it's outside the site,
+ * so there's no local page to check it against.
+ */
+export const resolveRedirects = (
+  redirects: readonly ConfiguredRedirect[],
+  pageUrls: ReadonlySet<string>
+): RedirectResolution[] => {
+  const byFrom = new Map<string, ConfiguredRedirect>();
+  for (const redirect of redirects) {
+    byFrom.set(normalizePath(redirect.from), redirect);
+  }
+
+  return redirects.map((redirect) => {
+    const from = normalizePath(redirect.from);
+    const chain: string[] = [from];
+    const seen = new Set<string>([from]);
+    let current = redirect.to;
+
+    for (;;) {
+      // An external hop ends the walk — we can't follow it locally.
+      if (/^https?:\/\//iu.test(current)) {
+        chain.push(current);
+        break;
+      }
+      const next = normalizePath(current);
+      if (seen.has(next)) {
+        chain.push(next);
+        return {
+          ...redirect,
+          chain,
+          outcome: "loop" as const,
+        };
+      }
+      chain.push(next);
+      seen.add(next);
+      const hop = byFrom.get(next);
+      if (!hop) {
+        break;
+      }
+      current = hop.to;
+    }
+
+    const destination = chain.at(-1) ?? from;
+    const external = /^https?:\/\//iu.test(destination);
+    if (!(external || pageUrls.has(destination))) {
+      return { ...redirect, chain, outcome: "broken" as const };
+    }
+    // `chain` is [from, …hops, destination]; more than two entries means at
+    // least one intermediate redirect.
+    return {
+      ...redirect,
+      chain,
+      outcome: chain.length > 2 ? ("chain" as const) : ("ok" as const),
+    };
+  });
+};