@stackql/docusaurus-plugin-aeo 0.4.2 → 0.5.1

package/src/companionPath.js

@@ -0,0 +1,89 @@
+// The one rule for where a page's `.md` companion lives, shared by the
+// emitter (feature 1), llms.txt (feature 2) and the Ask AI button (feature 3)
+// so the three can never disagree.
+//
+// Docusaurus emits either /foo/index.html (trailingSlash true or undefined)
+// or /foo.html (trailingSlash false); the companion mirrors the HTML:
+//
+//   route            trailingSlash: false   trailingSlash: true / undefined
+//   /                /index.md              /index.md
+//   /foo             /foo.md                /foo/index.md
+//   /foo/            /foo.md                /foo/index.md
+//   /docs/intro      /docs/intro.md         /docs/intro/index.md
+//
+// Plain CommonJS with no dependencies: required by the Node-side features
+// and bundled into the theme component by webpack.
+
+function normalizeRoute(routePath) {
+  const cleaned = String(routePath || '').replace(/\/+$/, '');
+  return cleaned === '' ? '/' : cleaned;
+}
+
+// Every function below takes an optional `baseUrl` (siteConfig.baseUrl).
+// Docusaurus permalinks and location.pathname carry it, and the part of
+// the route that is the base is a root, not a page: on a site built at
+// '/docs/' or in a localized build (baseUrl '/de/') the route '/de/' is the
+// home page, whose companion is '/de/index.md', not '/de.md'. With the
+// default '/' nothing changes.
+function withBase(baseUrl, sitePath) {
+  return `${String(baseUrl || '/').replace(/\/+$/, '')}${sitePath}`;
+}
+
+// Site-relative companion path for a route, e.g. '/foo.md'. Always starts
+// with '/'.
+function companionPath(routePath, trailingSlash, baseUrl) {
+  const route = outputRoute(routePath, baseUrl);
+  const rel = route === '/'
+    ? '/index.md'
+    : trailingSlash === false ? `${route}.md` : `${route}/index.md`;
+  return withBase(baseUrl, rel);
+}
+
+// Absolute companion URL for a route, e.g. 'https://example.com/foo.md'.
+// `siteUrl` is siteConfig.url.
+function companionUrl(siteUrl, routePath, trailingSlash, baseUrl) {
+  return `${String(siteUrl || '').replace(/\/+$/, '')}${companionPath(routePath, trailingSlash, baseUrl)}`;
+}
+
+// Site-relative path of the HTML file Docusaurus writes for a route, the
+// same rule as @docusaurus/core's pathnameToFilename once the route has
+// been normalised: '/foo.html' when trailingSlash is false, '/foo/index.html'
+// otherwise, '/index.html' for the root.
+function htmlPath(routePath, trailingSlash, baseUrl) {
+  const route = outputRoute(routePath, baseUrl);
+  const rel = route === '/'
+    ? '/index.html'
+    : trailingSlash === false ? `${route}.html` : `${route}/index.html`;
+  return withBase(baseUrl, rel);
+}
+
+// Absolute URL of the page itself: canonical, no trailing slash except on
+// the root (the site root, or the localized root '/de/').
+function pageUrl(siteUrl, routePath, baseUrl) {
+  const base = String(siteUrl || '').replace(/\/+$/, '');
+  const route = outputRoute(routePath, baseUrl);
+  return `${base}${withBase(baseUrl, route)}`;
+}
+
+// The route relative to the build output tree. Docusaurus permalinks carry
+// siteConfig.baseUrl, but the output tree is rooted at it: a site with
+// baseUrl '/docs/' writes /docs/intro to build/intro.html, and a localized
+// build (baseUrl '/de/', outDir build/de) writes /de/intro to
+// build/de/intro.html. URLs keep the full route (pass baseUrl to the
+// functions above); file paths under outDir use this route with no base.
+function outputRoute(routePath, baseUrl) {
+  return stripBase(normalizeRoute(routePath), baseUrl);
+}
+
+// A route with siteConfig.baseUrl removed and nothing else changed (a
+// trailing slash stays): '/de/blog/post/' under '/de/' is '/blog/post/', and
+// the base itself is '/'. A route outside the base is returned as is.
+function stripBase(routePath, baseUrl) {
+  const route = String(routePath || '');
+  const base = String(baseUrl || '/').replace(/\/+$/, '');
+  if (base === '') return route;
+  if (route === base) return '/';
+  return route.startsWith(`${base}/`) ? route.slice(base.length) : route;
+}
+
+module.exports = { companionPath, companionUrl, htmlPath, pageUrl, outputRoute, stripBase, normalizeRoute };

package/src/features/aiRoutes.js

@@ -5,26 +5,35 @@
 //   ai/faqs/<slug>.md          -> must declare a `faq` frontmatter object
 //   ai/howto/<slug>.md         -> must declare a `howTo` frontmatter object
 //   ai/howtos/<slug>.md        -> same as above
+//   ai/how-tos/<slug>.md       -> same as above
 //   ai/apps/<slug>.md          -> must declare a `softwareApplication`
 //                                   frontmatter object
 //
 // Anything else under /ai/* is permitted without a payload (consumer can
 // extend this with additional checks if needed).
 
+const { outputRoute } = require('../companionPath');
+
 const KIND_FOR_DIR = {
   faqs: 'faq',
   faq: 'faq',
   howto: 'howTo',
   howtos: 'howTo',
+  'how-tos': 'howTo',
+  'how-to': 'howTo',
   apps: 'softwareApplication',
   app: 'softwareApplication',
   applications: 'softwareApplication',
 };
 
-function classify(permalink) {
-  // permalink like "/ai/faqs/foo"
-  if (!permalink.startsWith('/ai/')) return null;
-  const segments = permalink.split('/').filter(Boolean); // ["ai", "faqs", "foo"]
+// `baseUrl` is siteConfig.baseUrl. The convention is relative to it: on a
+// site built at '/docs/' the routes are '/docs/ai/...', and in a localized
+// build '/de/ai/...'.
+function classify(permalink, baseUrl) {
+  // route like "/ai/faqs/foo"
+  const route = outputRoute(permalink, baseUrl);
+  if (!route.startsWith('/ai/')) return null;
+  const segments = route.split('/').filter(Boolean); // ["ai", "faqs", "foo"]
   if (segments.length < 2) return null;
   const dir = segments[1].toLowerCase();
   return KIND_FOR_DIR[dir] || null;
@@ -42,7 +51,7 @@ function collectDocs(content) {
   return out;
 }
 
-function validateAiRoutes({ loadedContentByPlugin, verbose }) {
+function validateAiRoutes({ loadedContentByPlugin, baseUrl, verbose }) {
   const problems = [];
   for (const [, entry] of loadedContentByPlugin) {
     if (entry.pluginName !== 'docusaurus-plugin-content-docs') continue;
@@ -50,7 +59,7 @@ function validateAiRoutes({ loadedContentByPlugin, verbose }) {
     for (const doc of docs) {
       const permalink = doc.permalink;
       if (!permalink) continue;
-      const expectedKey = classify(permalink);
+      const expectedKey = classify(permalink, baseUrl);
       if (!expectedKey) continue;
       const fm = doc.frontMatter || {};
       const value = fm[expectedKey];

package/src/features/alternateLinks.js

@@ -0,0 +1,133 @@
+// Feature 1b: advertise each page's .md companion from the page itself.
+//
+// After the companions are written, every route that got one has
+//
+//   <link rel="alternate" type="text/markdown" href="https://site/foo.md">
+//
+// inserted into its built HTML before </head>, so an agent that lands on
+// the page can find the markdown without guessing the URL. Optionally the
+// same relation is written as an HTTP header via a `_headers` file
+// (`Link: <url>; rel="alternate"; type="text/markdown"`), for agents that
+// read headers before bodies. Only hosts that read `_headers` use it
+// (Netlify, Cloudflare Pages); elsewhere the file is inert and the <link>
+// tag is what advertises the companion. The file sits at the root of the
+// build directory, shared by every locale built into it.
+//
+// Same approach as @stackql/docusaurus-plugin-structured-data, which also
+// post-processes the built HTML in postBuild: the file is a string, the
+// insertion point is the first </head>, no DOM library.
+
+const path = require('path');
+const fs = require('fs').promises;
+const { htmlPath, companionUrl, outputRoute } = require('../companionPath');
+
+const MARK = 'rel="alternate" type="text/markdown"';
+
+function linkTag(href) {
+  return `<link ${MARK} href="${href}">`;
+}
+
+function headerBlock(route, href) {
+  return `${route}\n  Link: <${href}>; rel="alternate"; type="text/markdown"\n`;
+}
+
+// Where `_headers` goes. The hosts that read the file (Netlify, Cloudflare
+// Pages) document it at the root of the published directory, so that is
+// the only place it can be relied on. A build of several locales puts
+// every locale but the default in a sub-directory of the build directory
+// named after the locale (baseUrl '/de/' or '/docs/de/' -> build/de), so
+// that locale's rules belong one level up, in the file the default
+// locale's build wrote (Docusaurus builds the default locale first). This
+// assumes the build directory is deployed as one site. Every other build
+// is its own root: one locale, a locale built on its own (`--locale de`,
+// baseUrl '/'), a locale given its own baseUrl, or a core that passes no
+// i18n.
+function headersDir({ outDir, siteConfig, i18n }) {
+  const locale = i18n && i18n.currentLocale;
+  if (!locale || locale === i18n.defaultLocale) return outDir;
+  const base = String(siteConfig.baseUrl || '/').replace(/\/+$/, '');
+  const nested = base.endsWith(`/${locale}`) && path.basename(outDir) === locale;
+  return nested ? path.dirname(outDir) : outDir;
+}
+
+async function readIfExists(file) {
+  try {
+    return await fs.readFile(file, 'utf8');
+  } catch (e) {
+    if (e.code === 'ENOENT') return null;
+    throw e;
+  }
+}
+
+module.exports = async function injectAlternateLinks({
+  props,
+  emittedCompanions,
+  linkHeader,
+  verbose,
+}) {
+  const { outDir, siteConfig } = props;
+  const trailingSlash = siteConfig.trailingSlash;
+  const siteUrl = siteConfig.url;
+
+  let injected = 0;
+  let skipped = 0;
+  const headerBlocks = [];
+
+  for (const item of emittedCompanions) {
+    const href = companionUrl(siteUrl, item.permalink, trailingSlash, siteConfig.baseUrl);
+    // The HTML lives under outDir at the route without siteConfig.baseUrl
+    // (see outputRoute); the URL and the _headers route keep the full one.
+    const file = path.join(
+      outDir,
+      htmlPath(outputRoute(item.permalink, siteConfig.baseUrl), trailingSlash),
+    );
+    const html = await readIfExists(file);
+    if (html === null) {
+      skipped++;
+      if (verbose) {
+        console.log(`[plugin-aeo] alternate link: no HTML at ${path.relative(outDir, file)} for ${item.permalink}`);
+      }
+      continue;
+    }
+    if (!html.includes(MARK)) {
+      const at = html.indexOf('</head>');
+      if (at === -1) {
+        skipped++;
+        if (verbose) {
+          console.log(`[plugin-aeo] alternate link: no </head> in ${path.relative(outDir, file)}`);
+        }
+        continue;
+      }
+      await fs.writeFile(file, `${html.slice(0, at)}${linkTag(href)}${html.slice(at)}`, 'utf8');
+      injected++;
+    }
+    if (linkHeader) {
+      // A `_headers` rule matches the path as requested; with
+      // trailingSlash false that is the bare route, otherwise the route
+      // with its slash.
+      const route = htmlPath(item.permalink, trailingSlash, siteConfig.baseUrl).replace(/index\.html$/, '').replace(/\.html$/, '');
+      headerBlocks.push(headerBlock(route === '' ? '/' : route, href));
+    }
+  }
+
+  if (linkHeader && headerBlocks.length > 0) {
+    // Append to a `_headers` the site may already ship from static/, or
+    // that another locale's build has already added its rules to. A rule
+    // already in the file is not written again.
+    const file = path.join(headersDir(props), '_headers');
+    const existing = (await readIfExists(file)) || '';
+    const fresh = headerBlocks.filter((block) => !existing.includes(block));
+    if (fresh.length > 0) {
+      const banner = '# @stackql/docusaurus-plugin-aeo: .md companions as Link headers\n';
+      const sep = existing && !existing.endsWith('\n') ? '\n\n' : existing ? '\n' : '';
+      await fs.writeFile(file, `${existing}${sep}${banner}${fresh.join('')}`, 'utf8');
+    }
+  }
+
+  if (verbose) {
+    console.log(
+      `[plugin-aeo] alternate link: injected into ${injected} page(s)${skipped ? `, skipped ${skipped}` : ''}${linkHeader ? `, ${headerBlocks.length} _headers rule(s)` : ''}`,
+    );
+  }
+  return { injected, skipped, headerRules: headerBlocks.length };
+};

package/src/features/companions.js

@@ -1,79 +1,9 @@
 const path = require('path');
 const fs = require('fs').promises;
 const matter = require('gray-matter');
-
-// Minimal glob matcher for route paths. Patterns like "/blog/tags/**",
-// "/search". Avoids pulling micromatch directly.
-function matchAny(value, patterns) {
-  if (!patterns || patterns.length === 0) return false;
-  for (const p of patterns) {
-    if (toRegExp(p).test(value)) return true;
-  }
-  return false;
-}
-
-function toRegExp(glob) {
-  // Minimal globstar -> regex translation. Supports: **, *, ?, character
-  // classes left untouched. Good enough for route paths like
-  // "/blog/tags/**" and "/search".
-  let re = '';
-  for (let i = 0; i < glob.length; i++) {
-    const c = glob[i];
-    if (c === '*') {
-      if (glob[i + 1] === '*') {
-        re += '.*';
-        i++;
-        if (glob[i + 1] === '/') i++;
-      } else {
-        re += '[^/]*';
-      }
-    } else if (c === '?') {
-      re += '[^/]';
-    } else if ('.+^$()|{}[]\\'.includes(c)) {
-      re += '\\' + c;
-    } else {
-      re += c;
-    }
-  }
-  return new RegExp('^' + re + '$');
-}
-
-function stripMdx(source) {
-  let out = source;
-  // Strip import / export lines (top-of-file MDX).
-  out = out.replace(/^\s*import\s+[^\n]+\n/gm, '');
-  out = out.replace(/^\s*export\s+[^\n]+\n/gm, '');
-  // Strip self-closing JSX tags like <Component prop="x" />.
-  out = out.replace(/<([A-Z][A-Za-z0-9]*)\b[^>]*\/>\s*/g, '');
-  // Strip paired JSX blocks like <Foo>...</Foo> (non-greedy, single component).
-  out = out.replace(
-    /<([A-Z][A-Za-z0-9]*)\b[^>]*>[\s\S]*?<\/\1>\s*/g,
-    '',
-  );
-  // Strip stray opening or closing component tags left behind.
-  out = out.replace(/<\/?[A-Z][A-Za-z0-9]*\b[^>]*>/g, '');
-  // Collapse 3+ blank lines.
-  out = out.replace(/\n{3,}/g, '\n\n');
-  return out;
-}
-
-function routeToCompanionPath(routePath, trailingSlash, outDir) {
-  // Docusaurus emits either /foo/index.html (trailingSlash: true / undefined)
-  // or /foo.html (trailingSlash: false). The companion mirrors the HTML.
-  let rel;
-  if (routePath === '/' || routePath === '') {
-    rel = 'index.md';
-  } else {
-    const cleaned = routePath.replace(/^\/+|\/+$/g, '');
-    if (trailingSlash === false) {
-      rel = `${cleaned}.md`;
-    } else {
-      // true or undefined (Docusaurus default behavior: index.html in folder)
-      rel = `${cleaned}/index.md`;
-    }
-  }
-  return path.join(outDir, rel);
-}
+const { companionPath, pageUrl, outputRoute } = require('../companionPath');
+const { matchAny, matchesRoute } = require('../routeGlob');
+const { toPlainMarkdown, resolveFormat } = require('./plainMarkdown');
 
 function collectFromDocs(content, pluginName, pluginId) {
   // Docs plugin loaded content shape:
@@ -122,6 +52,28 @@ function collectFromBlog(content, pluginName, pluginId) {
   return results;
 }
 
+// Every page that gets a companion: docs and blog pages with a source file,
+// minus `companions.exclude`. Generated category indexes, React pages and
+// blog list/tag/author pages have no source and are not in this list. Used
+// by postBuild to emit the files and by allContentLoaded to tell the Ask AI
+// button which routes have one. `baseUrl` is siteConfig.baseUrl: an exclude
+// pattern matches a route with or without it (see src/routeGlob.js), so the
+// same patterns apply in every locale's build.
+function collectCompanionItems(loadedContentByPlugin, options, baseUrl) {
+  const items = [];
+  for (const [, entry] of loadedContentByPlugin) {
+    const { pluginName, pluginId, content } = entry;
+    if (pluginName === 'docusaurus-plugin-content-docs') {
+      items.push(...collectFromDocs(content, pluginName, pluginId));
+    } else if (pluginName === 'docusaurus-plugin-content-blog') {
+      items.push(...collectFromBlog(content, pluginName, pluginId));
+    }
+    // Custom pages plugin emits React components, not markdown. Skip silently.
+  }
+  const exclude = (options && options.exclude) || [];
+  return items.filter((item) => !matchesRoute(item.permalink, exclude, baseUrl));
+}
+
 function resolveSourcePath(sourceRef, siteDir) {
   if (!sourceRef) return null;
   // Docusaurus content references look like "@site/docs/intro.md".
@@ -136,6 +88,34 @@ async function ensureDir(p) {
   await fs.mkdir(path.dirname(p), { recursive: true });
 }
 
+// Put the page's title, description and canonical URL at the top of a
+// plain companion so an LLM sees them without parsing YAML, and can cite
+// the page:
+//
+//   # Title
+//
+//   > description
+//
+//   Source: https://site/route
+//
+// A body that already opens with an H1 keeps it; the rest goes under it.
+function withTitleBlock(markdown, title, description, sourceUrl) {
+  const lines = markdown.split('\n');
+  const first = lines.findIndex((l) => l.trim() !== '');
+  const hasH1 = first >= 0 && /^#\s+\S/.test(lines[first]);
+  const extras = [];
+  if (description) extras.push(`> ${String(description).replace(/\s+/g, ' ').trim()}`);
+  if (sourceUrl) extras.push(`Source: ${sourceUrl}`);
+  if (hasH1) {
+    if (extras.length === 0) return markdown;
+    return [...lines.slice(0, first + 1), '', extras.join('\n\n'), ...lines.slice(first + 1)].join('\n');
+  }
+  const head = [];
+  if (title) head.push(`# ${title}`);
+  head.push(...extras);
+  return head.length ? `${head.join('\n\n')}\n\n${markdown}` : markdown;
+}
+
 module.exports = async function emitCompanions({
   props,
   options,
@@ -144,30 +124,16 @@ module.exports = async function emitCompanions({
 }) {
   const { outDir, siteConfig, siteDir } = props;
   const trailingSlash = siteConfig.trailingSlash;
-  const excludePatterns = options.exclude || [];
-  const format = options.format;
+  // 'clean' is accepted as a synonym for 'plain' (the name the external
+  // audit of 0.4.2 used for this behaviour).
+  const format = options.format === 'clean' ? 'plain' : options.format;
+  const markdownConfigFormat = siteConfig.markdown && siteConfig.markdown.format;
 
-  // Build the work list from captured loadedContent.
-  const items = [];
-  for (const [, entry] of loadedContentByPlugin) {
-    const { pluginName, pluginId, content } = entry;
-    if (pluginName === 'docusaurus-plugin-content-docs') {
-      items.push(...collectFromDocs(content, pluginName, pluginId));
-    } else if (pluginName === 'docusaurus-plugin-content-blog') {
-      items.push(...collectFromBlog(content, pluginName, pluginId));
-    }
-    // Custom pages plugin emits React components, not markdown. Skip silently.
-  }
+  const items = collectCompanionItems(loadedContentByPlugin, options, siteConfig.baseUrl);
 
   const emitted = [];
+  let fallbacks = 0;
   for (const item of items) {
-    if (matchAny(item.permalink, excludePatterns)) {
-      if (verbose) {
-        console.log(`[plugin-aeo] companions: excluded ${item.permalink}`);
-      }
-      continue;
-    }
-
     const sourcePath = resolveSourcePath(item.sourceRef, siteDir);
     if (!sourcePath) {
       if (verbose) {
@@ -192,20 +158,37 @@ module.exports = async function emitCompanions({
 
     let body = raw;
     if (format === 'plain') {
-      // Strip frontmatter, then strip MDX-specific bits.
       const parsed = matter(raw);
-      body = stripMdx(parsed.content);
-      // Re-prepend a minimal "title + description" block so an LLM can see them
-      // without parsing YAML.
-      const prefixParts = [];
-      if (item.title) prefixParts.push(`# ${item.title}`);
-      if (item.description) prefixParts.push(`> ${item.description}`);
-      if (prefixParts.length > 0) {
-        body = `${prefixParts.join('\n\n')}\n\n${body.trim()}\n`;
+      const sourceFormat = resolveFormat({
+        filePath: sourcePath,
+        frontMatterFormat: parsed.data && parsed.data.format,
+        markdownConfigFormat,
+      });
+      const result = await toPlainMarkdown(parsed.content, {
+        format: sourceFormat,
+        filePath: sourcePath,
+      });
+      if (result.fallback) {
+        fallbacks++;
+        console.warn(
+          `[plugin-aeo] companions: ${path.relative(siteDir, sourcePath)} did not parse as ${sourceFormat} (${result.error && result.error.message}); used the regex stripper for its companion`,
+        );
       }
+      body = withTitleBlock(
+        result.markdown,
+        item.title,
+        item.description,
+        pageUrl(siteConfig.url, item.permalink, siteConfig.baseUrl),
+      );
     }
 
-    const target = routeToCompanionPath(item.permalink, trailingSlash, outDir);
+    // The permalink carries siteConfig.baseUrl (localized to /<locale>/ in a
+    // localized build, whose outDir is build/<locale>); the file goes under
+    // outDir at the route without it.
+    const target = path.join(
+      outDir,
+      companionPath(outputRoute(item.permalink, siteConfig.baseUrl), trailingSlash),
+    );
     await ensureDir(target);
     await fs.writeFile(target, body, 'utf8');
 
@@ -228,7 +211,13 @@ module.exports = async function emitCompanions({
   }
 
   if (verbose) {
-    console.log(`[plugin-aeo] companions: emitted ${emitted.length} file(s)`);
+    console.log(
+      `[plugin-aeo] companions: emitted ${emitted.length} file(s)${fallbacks ? `, ${fallbacks} via the regex fallback` : ''}`,
+    );
   }
   return emitted;
 };
+
+module.exports.collectCompanionItems = collectCompanionItems;
+module.exports.withTitleBlock = withTitleBlock;
+module.exports.matchAny = matchAny;