dogsbay 0.2.0-beta.7 → 0.2.0-beta.71

package/dist/audit/lib/collect-assets.js

@@ -0,0 +1,64 @@
+/**
+ * Walk one or more content directories collecting absolute-path
+ * asset references that the build's `copyAssets` would emit to
+ * `public/`. The audit rule `structure/asset-refs` cross-checks
+ * `<img src>` / `<a href>` references against this set so a
+ * misnamed file shows up as a structured Issue.
+ *
+ * Mirrors the extension set used by `copyAssets` in
+ * `@dogsbay/format-astro/src/project.ts`. Keep in sync — drift
+ * would either flag valid refs as broken or miss real misses.
+ */
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { join, relative } from "node:path";
+// Same sets as format-astro's copyAssets. Keep in sync.
+const OPTIMIZABLE_EXTS = new Set([".png", ".jpg", ".jpeg", ".gif", ".webp"]);
+const PASSTHROUGH_EXTS = new Set([".svg", ".ico", ".pdf"]);
+/**
+ * Return the set of absolute asset paths (leading slash) that
+ * `copyAssets` would emit for the given content directories.
+ *
+ * @param contentDirs Absolute paths, one per source.
+ */
+export function collectAssets(contentDirs) {
+    const out = new Set();
+    for (const dir of contentDirs) {
+        if (!existsSync(dir))
+            continue;
+        walk(dir, dir, out);
+    }
+    return out;
+}
+function walk(rootDir, dir, out) {
+    let entries;
+    try {
+        entries = readdirSync(dir);
+    }
+    catch {
+        return;
+    }
+    for (const entry of entries) {
+        const full = join(dir, entry);
+        let isDir = false;
+        try {
+            isDir = statSync(full).isDirectory();
+        }
+        catch {
+            continue;
+        }
+        if (isDir) {
+            walk(rootDir, full, out);
+            continue;
+        }
+        const dot = entry.lastIndexOf(".");
+        if (dot < 0)
+            continue;
+        const ext = entry.slice(dot).toLowerCase();
+        if (!OPTIMIZABLE_EXTS.has(ext) && !PASSTHROUGH_EXTS.has(ext))
+            continue;
+        const rel = relative(rootDir, full);
+        // Public URL is "/<rel>" with forward slashes — copyAssets
+        // copies rel-to-source verbatim into public/.
+        out.add("/" + rel.replace(/\\/g, "/"));
+    }
+}

package/dist/audit/lib/validate-refs.js

@@ -0,0 +1,132 @@
+const EXTERNAL_RE = /^(?:[a-z][a-z0-9+.-]*:|\/\/)/i;
+const ANCHOR_ONLY_RE = /^#/;
+const DATA_URI_RE = /^data:/i;
+const ASSET_EXT_RE = /\.(png|jpe?g|gif|webp|svg|avif|pdf|zip|csv|tsv|json|yml|yaml|toml|mp4|webm|mp3|ogg|woff2?|ttf|otf)$/i;
+/**
+ * Heuristic for "looks like an asset reference," used to decide
+ * whether an `<a href>` should be validated against the asset
+ * set or the page set. Image src is always asset; href can be
+ * either. Anything ending in a known asset extension is treated
+ * as asset.
+ */
+function looksLikeAsset(href) {
+    // Strip query + fragment before checking extension
+    const noFragment = href.split(/[?#]/)[0];
+    return ASSET_EXT_RE.test(noFragment);
+}
+export function validateRefs(pages, options) {
+    const basePath = options.basePath.replace(/\/$/, "");
+    const knownAssets = options.knownAssets;
+    // Page slug set — multiple match forms so the validator
+    // accepts the common ways authors write the same slug.
+    // For slug "tutorials/quickstart" any of these match:
+    //   /tutorials/quickstart
+    //   /tutorials/quickstart/
+    //   /tutorials/quickstart.md
+    //   /<basePath>/tutorials/quickstart   (and the above three)
+    const pageSlugs = new Set();
+    for (const p of pages) {
+        pageSlugs.add(p.slug);
+        // Index pages can be referenced as their parent dir
+        if (p.slug === "index")
+            pageSlugs.add("");
+        if (p.slug.endsWith("/index")) {
+            pageSlugs.add(p.slug.slice(0, -"/index".length));
+        }
+    }
+    const result = { linkMisses: [], assetMisses: [] };
+    for (const page of pages) {
+        walkTree(page.tree, (href, kind) => {
+            check(href, kind, page.slug, basePath, pageSlugs, knownAssets, result);
+        });
+    }
+    return result;
+}
+/** Internal walk — invokes `visit` for every href/src in the tree. */
+function walkTree(nodes, visit) {
+    for (const node of nodes) {
+        if (node.inline)
+            walkInline(node.inline, visit);
+        if (node.html)
+            walkHtml(node.html, visit);
+        if (node.props?.href && typeof node.props.href === "string") {
+            visit(node.props.href, "link");
+        }
+        if (node.props?.src && typeof node.props.src === "string") {
+            visit(node.props.src, "image");
+        }
+        if (node.children)
+            walkTree(node.children, visit);
+    }
+}
+function walkInline(nodes, visit) {
+    for (const node of nodes) {
+        if (node.type === "link" && typeof node.href === "string") {
+            visit(node.href, "link");
+            if (node.children)
+                walkInline(node.children, visit);
+        }
+        else if (node.type === "image" && typeof node.src === "string") {
+            visit(node.src, "image");
+        }
+        else if (node.type === "highlight" && node.children) {
+            walkInline(node.children, visit);
+        }
+    }
+}
+function walkHtml(html, visit) {
+    // Pre-rendered HTML in `node.html` (Starlight importer output,
+    // MkDocs raw HTML, etc.) — scan with regexes. Cheap; the audit
+    // doesn't need a full HTML parser for this.
+    const aRe = /<a\b[^>]*\shref="([^"]+)"/gi;
+    let m;
+    while ((m = aRe.exec(html)) !== null)
+        visit(m[1], "link");
+    const imgRe = /<img\b[^>]*\ssrc="([^"]+)"/gi;
+    while ((m = imgRe.exec(html)) !== null)
+        visit(m[1], "image");
+}
+function check(href, kind, pageSlug, basePath, pageSlugs, knownAssets, result) {
+    if (!href || EXTERNAL_RE.test(href))
+        return;
+    if (ANCHOR_ONLY_RE.test(href))
+        return;
+    if (DATA_URI_RE.test(href))
+        return;
+    if (!href.startsWith("/"))
+        return; // relative — out of scope for v1
+    // Strip query and fragment for the comparison.
+    const cleanHref = href.split(/[?#]/)[0];
+    // Strip basePath if the author prefixed it. We compare against
+    // bare slugs, so `/docs/intro` and `/intro` should both match
+    // page slug "intro" (basePath="/docs").
+    let path = cleanHref;
+    if (basePath && (path === basePath || path.startsWith(`${basePath}/`))) {
+        path = path.slice(basePath.length);
+    }
+    // Strip leading / and trailing /, plus the `.md` extension.
+    let trimmed = path.replace(/^\//, "").replace(/\/$/, "");
+    trimmed = trimmed.replace(/\.md$/i, "");
+    // Image src — always asset.
+    // Anchor href — asset if it has an asset extension, else page.
+    const wantsAsset = kind === "image" || looksLikeAsset(cleanHref);
+    if (wantsAsset) {
+        // Asset paths are absolute under the public root.
+        // knownAssets stores entries like "/_assets/foo.png".
+        const lookup = cleanHref.startsWith("/") && basePath && cleanHref.startsWith(`${basePath}/`)
+            ? cleanHref.slice(basePath.length)
+            : cleanHref;
+        if (!knownAssets.has(lookup)) {
+            result.assetMisses.push({
+                pageSlug,
+                href,
+                resolvedAgainst: "asset",
+            });
+        }
+        return;
+    }
+    // Page lookup
+    if (!pageSlugs.has(trimmed)) {
+        result.linkMisses.push({ pageSlug, href, resolvedAgainst: "page" });
+    }
+}

package/dist/audit/rules/seo/sitemap.js

@@ -22,7 +22,7 @@
  * `_resetSitemapCache()`.
  */
 import { existsSync, readFileSync } from "node:fs";
-import { join } from "node:path";
+import { basename, dirname, join } from "node:path";
 const STATE_CACHE = new Map();
 export function _resetSitemapCache() {
     STATE_CACHE.clear();
@@ -50,11 +50,26 @@ function getState(distRoot) {
             // sitemap integration which produces well-formed XML, so
             // the only failure mode is "missing entirely" or
             // "doesn't parse at all."
-            if (/<(urlset|sitemapindex)[\s>]/.test(content)) {
+            const isIndex = /<sitemapindex[\s>]/.test(content);
+            const isUrlset = /<urlset[\s>]/.test(content);
+            if (isIndex || isUrlset) {
                 sitemapValid = true;
-                const matches = content.matchAll(/<loc>([^<]+)<\/loc>/g);
-                for (const m of matches) {
-                    sitemapLocs.add(m[1].trim());
+                if (isIndex) {
+                    // sitemap-index.xml — each <loc> points at a child
+                    // sitemap file (not a page). Read each child and union
+                    // its <loc> entries (those ARE the page URLs). Without
+                    // this recursion, sitemapLocs would carry sitemap file
+                    // URLs only and every page would be flagged "missing
+                    // from sitemap" — the entire audit class is unusable.
+                    collectChildSitemapLocs(content, sitemapPath, sitemapLocs);
+                }
+                else {
+                    // Plain <urlset> — extract <loc>s directly. Same as
+                    // before.
+                    const matches = content.matchAll(/<loc>([^<]+)<\/loc>/g);
+                    for (const m of matches) {
+                        sitemapLocs.add(m[1].trim());
+                    }
                 }
             }
         }
@@ -123,7 +138,7 @@ export const sitemapComplete = {
     severity: "warning",
     description: "Every built HTML page is listed in the sitemap.",
     run(ctx) {
-        const { file, distRoot, allFiles } = ctx;
+        const { file, distRoot, allFiles, config } = ctx;
         const state = getState(distRoot);
         // Emit once per audit run — we have access to allFiles here.
         if (state.emitted.has("seo/sitemap-complete"))
@@ -134,13 +149,33 @@ export const sitemapComplete = {
             // to add here.
             return [];
         }
+        // urlBase comes from the path component of site.url. When the
+        // site is mounted at a subpath (typical GH Pages project deploy:
+        // `site.url: https://user.github.io/repo`), sitemap entries
+        // carry the urlBase prefix (the emitter writes absolute URLs),
+        // but the filesystem-derived urlPath below doesn't — files
+        // live at `dist/<page>/index.html` regardless of urlBase. Pass
+        // urlBase into the comparison so the suffix-strip step
+        // accounts for it. See plans/sitemap-audit-urlbase.md.
+        const urlBase = urlBaseFromSiteUrl(config?.siteUrl);
         // Map each html file to its likely "page URL" form. Astro
         // emits `docs/intro/index.html` for the URL `/docs/intro/`,
         // so we strip `/index.html` and ensure a leading slash.
+        //
+        // Skip pages with `<meta name="robots" content="noindex">`.
+        // The sitemap emitter intentionally excludes those (per the
+        // platform contract documented in reference/frontmatter.md
+        // and `packages/format-astro/src/sitemap.ts` isExcluded), so
+        // flagging them as "missing" would tell users to fix something
+        // that's working as intended. The auto-emitted taxonomy index
+        // pages (/tags/, /by-audience/, …) all ship with noindex.
         const issues = [];
         for (const f of allFiles) {
+            const robots = f.$('meta[name="robots"]').attr("content") ?? "";
+            if (/\bnoindex\b/i.test(robots))
+                continue;
             const expectedPath = htmlPathToUrlPath(f.path);
-            const found = pathListedInSitemap(expectedPath, state.sitemapLocs);
+            const found = pathListedInSitemap(expectedPath, state.sitemapLocs, urlBase);
             if (!found) {
                 issues.push({
                     ruleId: "seo/sitemap-complete",
@@ -194,6 +229,51 @@ export const sitemapRobotsCoherence = {
         return issues;
     },
 };
+/**
+ * Recurse into a `<sitemapindex>` document: for each child
+ * sitemap referenced by `<loc>`, read the file from disk and
+ * union its page-level `<loc>` entries into the supplied set.
+ *
+ * Resolution strategy: take the basename of the child URL and
+ * resolve it against the directory that contains the index file.
+ * That covers both layouts in use today — host-root
+ * (`dist/sitemap-index.xml` + `dist/sitemap-0.xml`) and
+ * per-mount (`dist/<basePath>/sitemap-index.xml` +
+ * `dist/<basePath>/sitemap-0.xml`). Children that don't resolve
+ * to a local file are silently skipped — better to miss a few
+ * URLs than to crash the audit when a deploy ships a partial
+ * sitemap. The page-level rule still surfaces orphan pages, so
+ * an incomplete recursion just downgrades to the previous
+ * misbehaviour (all pages flagged as missing) — never worse.
+ */
+function collectChildSitemapLocs(indexContent, indexPath, out) {
+    const indexDir = dirname(indexPath);
+    for (const m of indexContent.matchAll(/<loc>([^<]+)<\/loc>/g)) {
+        const childUrl = m[1].trim();
+        let childName;
+        try {
+            const u = new URL(childUrl);
+            childName = basename(u.pathname);
+        }
+        catch {
+            childName = basename(childUrl);
+        }
+        if (!childName || childName === "/")
+            continue;
+        const childPath = join(indexDir, childName);
+        if (!existsSync(childPath))
+            continue;
+        try {
+            const childContent = readFileSync(childPath, "utf-8");
+            for (const cm of childContent.matchAll(/<loc>([^<]+)<\/loc>/g)) {
+                out.add(cm[1].trim());
+            }
+        }
+        catch {
+            // Skip unreadable child — see function-level comment.
+        }
+    }
+}
 /**
  * Convert an HTML file path within `dist/` into the public URL
  * path the sitemap is likely to list. Drops `/index.html` and
@@ -216,9 +296,18 @@ function htmlPathToUrlPath(htmlPath) {
  * Sitemap entries can be absolute (`https://example.com/foo/`)
  * or relative (`/foo/`); we match either by suffix.
  *
+ * `urlBase` is the path component of `site.url` — e.g. `/repo` for
+ * a GH Pages project deploy at `https://user.github.io/repo`. When
+ * non-empty, the platform's sitemap emitter writes absolute URLs
+ * including that segment (`https://user.github.io/repo/intro/`),
+ * but the filesystem-derived `urlPath` doesn't have it (files live
+ * at `dist/intro/index.html`). Strip the urlBase off the sitemap
+ * suffix before comparing so the audit doesn't flag every page as
+ * missing on every subpath-mounted deploy.
+ *
  * Trailing slash tolerant: `/foo/` matches `/foo` too.
  */
-function pathListedInSitemap(urlPath, locs) {
+function pathListedInSitemap(urlPath, locs, urlBase) {
     for (const loc of locs) {
         if (loc === urlPath)
             return true;
@@ -230,6 +319,16 @@ function pathListedInSitemap(urlPath, locs) {
             if (afterHost >= 0)
                 suffix = loc.slice(afterHost);
         }
+        // Strip the urlBase prefix when the sitemap entry carries one
+        // and the comparison target doesn't.
+        if (urlBase) {
+            if (suffix === urlBase) {
+                suffix = "/";
+            }
+            else if (suffix.startsWith(`${urlBase}/`)) {
+                suffix = suffix.slice(urlBase.length);
+            }
+        }
         if (suffix === urlPath)
             return true;
         // Trailing-slash flexibility
@@ -238,3 +337,21 @@ function pathListedInSitemap(urlPath, locs) {
     }
     return false;
 }
+/**
+ * Extract the path component of `site.url` for use as the urlBase
+ * stripping prefix. Returns `""` when site.url is missing, has no
+ * path, or doesn't parse — comparison falls back to today's
+ * behaviour.
+ */
+function urlBaseFromSiteUrl(siteUrl) {
+    if (!siteUrl)
+        return "";
+    try {
+        const u = new URL(siteUrl);
+        const path = u.pathname.replace(/\/$/, "");
+        return path === "/" ? "" : path;
+    }
+    catch {
+        return "";
+    }
+}

package/dist/audit/rules/structure/asset-refs.js

@@ -0,0 +1,41 @@
+import { validateRefs } from "../../lib/validate-refs.js";
+export const assetRefs = {
+    id: "structure/asset-refs",
+    category: "structure",
+    stage: "source-corpus",
+    severity: "error",
+    description: "Every absolute `<img src>` and asset-style `<a href>` resolves to a file under the content tree.",
+    run(rawCtx) {
+        const ctx = rawCtx;
+        if (!ctx.pages || ctx.pages.length === 0)
+            return [];
+        if (!ctx.knownAssets) {
+            // No asset set available — typically a unit-test ctx that
+            // didn't bother. No-op rather than false-positive.
+            return [];
+        }
+        const { assetMisses } = validateRefs(ctx.pages, {
+            basePath: ctx.basePath ?? "",
+            knownAssets: ctx.knownAssets,
+        });
+        return assetMisses.map((m) => {
+            // Where the user should drop the file to satisfy THIS ref —
+            // strip optional basePath, prepend `content` for the
+            // suggestion. (We can't know the exact content dir layout
+            // for multi-source sites without more plumbing; keep it
+            // intuitive.)
+            const cleanHref = m.href.split(/[?#]/)[0];
+            const expected = `content${cleanHref}`;
+            return {
+                ruleId: "structure/asset-refs",
+                severity: "error",
+                file: `${m.pageSlug}.md`,
+                message: `Asset reference ${m.href} doesn't match any file in the ` +
+                    `content tree. To fix: correct the path, rename the ` +
+                    `existing file to match, or drop the asset at ` +
+                    `${expected} to satisfy this exact reference.`,
+                context: m.href,
+            };
+        });
+    },
+};

package/dist/audit/rules/structure/index.js

@@ -11,8 +11,12 @@
  *   - structure/duplicate-slugs      (corpus; two sources produced the same URL)
  */
 import { registerRule } from "../../registry.js";
+import { assetRefs } from "./asset-refs.js";
+import { internalLinks } from "./internal-links.js";
 import { localeCoherence } from "./locale-coherence.js";
 import { namespaceCoherence } from "./namespace-coherence.js";
+import { navTargetExists } from "./nav-target-exists.js";
+import { unresolvedDirectives } from "./unresolved-directives.js";
 import { versionCoherence } from "./version-coherence.js";
 let registered = false;
 /**
@@ -26,6 +30,10 @@ export function registerStructureRules() {
     registerRule(namespaceCoherence);
     registerRule(versionCoherence);
     registerRule(localeCoherence);
+    registerRule(navTargetExists);
+    registerRule(internalLinks);
+    registerRule(assetRefs);
+    registerRule(unresolvedDirectives);
 }
 /**
  * Test-only: reset the "registered" flag so unit tests can

package/dist/audit/rules/structure/internal-links.js

@@ -0,0 +1,28 @@
+import { validateRefs } from "../../lib/validate-refs.js";
+export const internalLinks = {
+    id: "structure/internal-links",
+    category: "structure",
+    stage: "source-corpus",
+    severity: "error",
+    description: "Every absolute internal `<a href>` resolves to a built page in the corpus.",
+    run(rawCtx) {
+        const ctx = rawCtx;
+        if (!ctx.pages || ctx.pages.length === 0)
+            return [];
+        const { linkMisses } = validateRefs(ctx.pages, {
+            basePath: ctx.basePath ?? "",
+            knownAssets: ctx.knownAssets ?? new Set(),
+        });
+        return linkMisses.map((m) => ({
+            ruleId: "structure/internal-links",
+            severity: "error",
+            file: `${m.pageSlug}.md`,
+            message: `Internal link to ${m.href} doesn't resolve to any built ` +
+                `page. To fix: correct the path, rename the target page ` +
+                `to match, or remove the link. (Common cause: page renamed ` +
+                `without updating the body text. ` +
+                `Use \`dogsbay site check\` to surface every miss.)`,
+            context: m.href,
+        }));
+    },
+};

package/dist/audit/rules/structure/nav-target-exists.js

@@ -0,0 +1,107 @@
+/**
+ * `structure/nav-target-exists` — every `file:` reference in a
+ * declared nav file resolves to a content file on disk.
+ *
+ * Re-runs the same resolution `loadNavFile` does at build time,
+ * but in `collect` mode: instead of `console.warn` + label-only
+ * fallback (the build-pipeline behaviour), the audit surfaces
+ * every miss as a structured Issue. This is what makes nav-file
+ * typos:
+ *
+ *   - count in the `dogsbay site check` summary
+ *   - bump the exit code (error severity)
+ *   - get categorised under `structure` for selective opt-out
+ *
+ * Without this rule, a misnamed file in nav.yml emitted a
+ * `[dogsbay] Nav file references missing file: ...` line to
+ * stderr during build but never entered the audit catalog, so
+ * CI couldn't catch it. Reported by the docs team against
+ * dogsbay-docs-platform.
+ *
+ * Source content + nav heuristic: matches `buildNavFromDirectory`
+ * — an explicit `source.nav` path wins, otherwise we look for
+ * `nav.yml` / `nav.yaml` / `nav.json` in the resolved content
+ * dir. If no nav file is present, the rule is a no-op (the site
+ * uses directory-scan nav, which has its own validation in the
+ * importer).
+ */
+import { existsSync } from "node:fs";
+import { isAbsolute, join, resolve } from "node:path";
+import { loadNavFile } from "@dogsbay/format-dogsbay-md";
+const HEURISTIC_NAV_FILES = ["nav.yml", "nav.yaml", "nav.json"];
+export const navTargetExists = {
+    id: "structure/nav-target-exists",
+    category: "structure",
+    stage: "source-corpus",
+    severity: "error",
+    description: "Every `file:` reference in nav files (nav.yml / nav.yaml / nav.json) resolves to a content file that exists on disk.",
+    run(rawCtx) {
+        const ctx = rawCtx;
+        const auditSources = ctx.auditSources;
+        const siteRoot = ctx.siteRoot;
+        if (!auditSources || auditSources.length === 0 || !siteRoot) {
+            // No way to find nav files — typically a unit-test run with
+            // pre-imported nav.
+            return [];
+        }
+        const issues = [];
+        for (const src of auditSources) {
+            const { contentDir, navOverride } = src;
+            const navPath = resolveNavFile(navOverride, contentDir, siteRoot);
+            if (!navPath)
+                continue; // directory-scan source → skip
+            // Use `collect` mode so the loader pushes structured
+            // findings to onMissingTarget instead of console.warn'ing.
+            const misses = [];
+            try {
+                loadNavFile(navPath, contentDir, {
+                    missingFile: "collect",
+                    onMissingTarget: (m) => misses.push(m),
+                });
+            }
+            catch (err) {
+                // Nav file itself is unreadable / malformed — emit as a
+                // distinct issue. The build's heuristic-loader warns and
+                // falls back to directory scan; the audit makes the
+                // condition visible.
+                issues.push({
+                    ruleId: "structure/nav-target-exists",
+                    severity: "error",
+                    file: navPath,
+                    message: `Nav file ${navPath} failed to load: ${err.message}`,
+                });
+                continue;
+            }
+            for (const miss of misses) {
+                issues.push({
+                    ruleId: "structure/nav-target-exists",
+                    severity: "error",
+                    file: navPath,
+                    message: `Nav references missing file: ${miss.file} ` +
+                        `(resolved to ${miss.resolvedAbs}). To fix: rename the ` +
+                        `file to match, correct the path in the nav file, or ` +
+                        `remove the entry.`,
+                    context: miss.file,
+                });
+            }
+        }
+        return issues;
+    },
+};
+/**
+ * Mirror the resolution `buildNavFromDirectory` uses:
+ *   1. Explicit `source.nav` wins (relative to siteRoot)
+ *   2. Auto-detect nav.{yml,yaml,json} in the resolved content dir
+ *   3. Otherwise undefined (directory-scan nav — no nav file to audit)
+ */
+function resolveNavFile(navOverride, contentDir, siteRoot) {
+    if (navOverride) {
+        return isAbsolute(navOverride) ? navOverride : resolve(siteRoot, navOverride);
+    }
+    for (const fileName of HEURISTIC_NAV_FILES) {
+        const candidate = join(contentDir, fileName);
+        if (existsSync(candidate))
+            return candidate;
+    }
+    return undefined;
+}