@stackql/docusaurus-plugin-aeo 0.4.1 → 0.5.0

package/src/features/alternateLinks.js

@@ -0,0 +1,100 @@
+// Feature 1b: advertise each page's .md companion from the page itself.
+//
+// After the companions are written, every route that got one has
+//
+//   <link rel="alternate" type="text/markdown" href="https://site/foo.md">
+//
+// inserted into its built HTML before </head>, so an agent that lands on
+// the page can find the markdown without guessing the URL. Optionally the
+// same relation is written as an HTTP header for Netlify-style hosts via a
+// `_headers` file (`Link: <url>; rel="alternate"; type="text/markdown"`),
+// for agents that read headers before bodies.
+//
+// Same approach as @stackql/docusaurus-plugin-structured-data, which also
+// post-processes the built HTML in postBuild: the file is a string, the
+// insertion point is the first </head>, no DOM library.
+
+const path = require('path');
+const fs = require('fs').promises;
+const { htmlPath, companionUrl } = require('../companionPath');
+
+const MARK = 'rel="alternate" type="text/markdown"';
+
+function linkTag(href) {
+  return `<link ${MARK} href="${href}">`;
+}
+
+function headerBlock(route, href) {
+  return `${route}\n  Link: <${href}>; rel="alternate"; type="text/markdown"\n`;
+}
+
+async function readIfExists(file) {
+  try {
+    return await fs.readFile(file, 'utf8');
+  } catch (e) {
+    if (e.code === 'ENOENT') return null;
+    throw e;
+  }
+}
+
+module.exports = async function injectAlternateLinks({
+  props,
+  emittedCompanions,
+  linkHeader,
+  verbose,
+}) {
+  const { outDir, siteConfig } = props;
+  const trailingSlash = siteConfig.trailingSlash;
+  const siteUrl = siteConfig.url;
+
+  let injected = 0;
+  let skipped = 0;
+  const headerBlocks = [];
+
+  for (const item of emittedCompanions) {
+    const href = companionUrl(siteUrl, item.permalink, trailingSlash);
+    const file = path.join(outDir, htmlPath(item.permalink, trailingSlash));
+    const html = await readIfExists(file);
+    if (html === null) {
+      skipped++;
+      if (verbose) {
+        console.log(`[plugin-aeo] alternate link: no HTML at ${path.relative(outDir, file)} for ${item.permalink}`);
+      }
+      continue;
+    }
+    if (!html.includes(MARK)) {
+      const at = html.indexOf('</head>');
+      if (at === -1) {
+        skipped++;
+        if (verbose) {
+          console.log(`[plugin-aeo] alternate link: no </head> in ${path.relative(outDir, file)}`);
+        }
+        continue;
+      }
+      await fs.writeFile(file, `${html.slice(0, at)}${linkTag(href)}${html.slice(at)}`, 'utf8');
+      injected++;
+    }
+    if (linkHeader) {
+      // Netlify matches the route as requested; with trailingSlash false
+      // that is the bare route, otherwise the route with its slash.
+      const route = htmlPath(item.permalink, trailingSlash).replace(/index\.html$/, '').replace(/\.html$/, '');
+      headerBlocks.push(headerBlock(route === '' ? '/' : route, href));
+    }
+  }
+
+  if (linkHeader && headerBlocks.length > 0) {
+    // Append to a `_headers` the site may already ship from static/.
+    const file = path.join(outDir, '_headers');
+    const existing = (await readIfExists(file)) || '';
+    const banner = '# @stackql/docusaurus-plugin-aeo: .md companions as Link headers\n';
+    const sep = existing && !existing.endsWith('\n') ? '\n\n' : existing ? '\n' : '';
+    await fs.writeFile(file, `${existing}${sep}${banner}${headerBlocks.join('')}`, 'utf8');
+  }
+
+  if (verbose) {
+    console.log(
+      `[plugin-aeo] alternate link: injected into ${injected} page(s)${skipped ? `, skipped ${skipped}` : ''}${linkHeader ? `, ${headerBlocks.length} _headers rule(s)` : ''}`,
+    );
+  }
+  return { injected, skipped, headerRules: headerBlocks.length };
+};

package/src/features/companions.js

@@ -1,6 +1,8 @@
 const path = require('path');
 const fs = require('fs').promises;
 const matter = require('gray-matter');
+const { companionPath, pageUrl } = require('../companionPath');
+const { toPlainMarkdown, resolveFormat } = require('./plainMarkdown');
 
 // Minimal glob matcher for route paths. Patterns like "/blog/tags/**",
 // "/search". Avoids pulling micromatch directly.
@@ -38,43 +40,6 @@ function toRegExp(glob) {
   return new RegExp('^' + re + '$');
 }
 
-function stripMdx(source) {
-  let out = source;
-  // Strip import / export lines (top-of-file MDX).
-  out = out.replace(/^\s*import\s+[^\n]+\n/gm, '');
-  out = out.replace(/^\s*export\s+[^\n]+\n/gm, '');
-  // Strip self-closing JSX tags like <Component prop="x" />.
-  out = out.replace(/<([A-Z][A-Za-z0-9]*)\b[^>]*\/>\s*/g, '');
-  // Strip paired JSX blocks like <Foo>...</Foo> (non-greedy, single component).
-  out = out.replace(
-    /<([A-Z][A-Za-z0-9]*)\b[^>]*>[\s\S]*?<\/\1>\s*/g,
-    '',
-  );
-  // Strip stray opening or closing component tags left behind.
-  out = out.replace(/<\/?[A-Z][A-Za-z0-9]*\b[^>]*>/g, '');
-  // Collapse 3+ blank lines.
-  out = out.replace(/\n{3,}/g, '\n\n');
-  return out;
-}
-
-function routeToCompanionPath(routePath, trailingSlash, outDir) {
-  // Docusaurus emits either /foo/index.html (trailingSlash: true / undefined)
-  // or /foo.html (trailingSlash: false). The companion mirrors the HTML.
-  let rel;
-  if (routePath === '/' || routePath === '') {
-    rel = 'index.md';
-  } else {
-    const cleaned = routePath.replace(/^\/+|\/+$/g, '');
-    if (trailingSlash === false) {
-      rel = `${cleaned}.md`;
-    } else {
-      // true or undefined (Docusaurus default behavior: index.html in folder)
-      rel = `${cleaned}/index.md`;
-    }
-  }
-  return path.join(outDir, rel);
-}
-
 function collectFromDocs(content, pluginName, pluginId) {
   // Docs plugin loaded content shape:
   //   { loadedVersions: [{ docs: [{ id, permalink, source, title, description, frontMatter }] }] }
@@ -122,6 +87,26 @@ function collectFromBlog(content, pluginName, pluginId) {
   return results;
 }
 
+// Every page that gets a companion: docs and blog pages with a source file,
+// minus `companions.exclude`. Generated category indexes, React pages and
+// blog list/tag/author pages have no source and are not in this list. Used
+// by postBuild to emit the files and by allContentLoaded to tell the Ask AI
+// button which routes have one.
+function collectCompanionItems(loadedContentByPlugin, options) {
+  const items = [];
+  for (const [, entry] of loadedContentByPlugin) {
+    const { pluginName, pluginId, content } = entry;
+    if (pluginName === 'docusaurus-plugin-content-docs') {
+      items.push(...collectFromDocs(content, pluginName, pluginId));
+    } else if (pluginName === 'docusaurus-plugin-content-blog') {
+      items.push(...collectFromBlog(content, pluginName, pluginId));
+    }
+    // Custom pages plugin emits React components, not markdown. Skip silently.
+  }
+  const exclude = (options && options.exclude) || [];
+  return items.filter((item) => !matchAny(item.permalink, exclude));
+}
+
 function resolveSourcePath(sourceRef, siteDir) {
   if (!sourceRef) return null;
   // Docusaurus content references look like "@site/docs/intro.md".
@@ -136,6 +121,34 @@ async function ensureDir(p) {
   await fs.mkdir(path.dirname(p), { recursive: true });
 }
 
+// Put the page's title, description and canonical URL at the top of a
+// plain companion so an LLM sees them without parsing YAML, and can cite
+// the page:
+//
+//   # Title
+//
+//   > description
+//
+//   Source: https://site/route
+//
+// A body that already opens with an H1 keeps it; the rest goes under it.
+function withTitleBlock(markdown, title, description, sourceUrl) {
+  const lines = markdown.split('\n');
+  const first = lines.findIndex((l) => l.trim() !== '');
+  const hasH1 = first >= 0 && /^#\s+\S/.test(lines[first]);
+  const extras = [];
+  if (description) extras.push(`> ${String(description).replace(/\s+/g, ' ').trim()}`);
+  if (sourceUrl) extras.push(`Source: ${sourceUrl}`);
+  if (hasH1) {
+    if (extras.length === 0) return markdown;
+    return [...lines.slice(0, first + 1), '', extras.join('\n\n'), ...lines.slice(first + 1)].join('\n');
+  }
+  const head = [];
+  if (title) head.push(`# ${title}`);
+  head.push(...extras);
+  return head.length ? `${head.join('\n\n')}\n\n${markdown}` : markdown;
+}
+
 module.exports = async function emitCompanions({
   props,
   options,
@@ -144,30 +157,16 @@ module.exports = async function emitCompanions({
 }) {
   const { outDir, siteConfig, siteDir } = props;
   const trailingSlash = siteConfig.trailingSlash;
-  const excludePatterns = options.exclude || [];
-  const format = options.format;
+  // 'clean' is accepted as a synonym for 'plain' (the name the external
+  // audit of 0.4.2 used for this behaviour).
+  const format = options.format === 'clean' ? 'plain' : options.format;
+  const markdownConfigFormat = siteConfig.markdown && siteConfig.markdown.format;
 
-  // Build the work list from captured loadedContent.
-  const items = [];
-  for (const [, entry] of loadedContentByPlugin) {
-    const { pluginName, pluginId, content } = entry;
-    if (pluginName === 'docusaurus-plugin-content-docs') {
-      items.push(...collectFromDocs(content, pluginName, pluginId));
-    } else if (pluginName === 'docusaurus-plugin-content-blog') {
-      items.push(...collectFromBlog(content, pluginName, pluginId));
-    }
-    // Custom pages plugin emits React components, not markdown. Skip silently.
-  }
+  const items = collectCompanionItems(loadedContentByPlugin, options);
 
   const emitted = [];
+  let fallbacks = 0;
   for (const item of items) {
-    if (matchAny(item.permalink, excludePatterns)) {
-      if (verbose) {
-        console.log(`[plugin-aeo] companions: excluded ${item.permalink}`);
-      }
-      continue;
-    }
-
     const sourcePath = resolveSourcePath(item.sourceRef, siteDir);
     if (!sourcePath) {
       if (verbose) {
@@ -192,20 +191,31 @@ module.exports = async function emitCompanions({
 
     let body = raw;
     if (format === 'plain') {
-      // Strip frontmatter, then strip MDX-specific bits.
       const parsed = matter(raw);
-      body = stripMdx(parsed.content);
-      // Re-prepend a minimal "title + description" block so an LLM can see them
-      // without parsing YAML.
-      const prefixParts = [];
-      if (item.title) prefixParts.push(`# ${item.title}`);
-      if (item.description) prefixParts.push(`> ${item.description}`);
-      if (prefixParts.length > 0) {
-        body = `${prefixParts.join('\n\n')}\n\n${body.trim()}\n`;
+      const sourceFormat = resolveFormat({
+        filePath: sourcePath,
+        frontMatterFormat: parsed.data && parsed.data.format,
+        markdownConfigFormat,
+      });
+      const result = await toPlainMarkdown(parsed.content, {
+        format: sourceFormat,
+        filePath: sourcePath,
+      });
+      if (result.fallback) {
+        fallbacks++;
+        console.warn(
+          `[plugin-aeo] companions: ${path.relative(siteDir, sourcePath)} did not parse as ${sourceFormat} (${result.error && result.error.message}); used the regex stripper for its companion`,
+        );
       }
+      body = withTitleBlock(
+        result.markdown,
+        item.title,
+        item.description,
+        pageUrl(siteConfig.url, item.permalink),
+      );
     }
 
-    const target = routeToCompanionPath(item.permalink, trailingSlash, outDir);
+    const target = path.join(outDir, companionPath(item.permalink, trailingSlash));
     await ensureDir(target);
     await fs.writeFile(target, body, 'utf8');
 
@@ -228,7 +238,13 @@ module.exports = async function emitCompanions({
   }
 
   if (verbose) {
-    console.log(`[plugin-aeo] companions: emitted ${emitted.length} file(s)`);
+    console.log(
+      `[plugin-aeo] companions: emitted ${emitted.length} file(s)${fallbacks ? `, ${fallbacks} via the regex fallback` : ''}`,
+    );
   }
   return emitted;
 };
+
+module.exports.collectCompanionItems = collectCompanionItems;
+module.exports.withTitleBlock = withTitleBlock;
+module.exports.matchAny = matchAny;

package/src/features/llmsTxt.js

@@ -1,5 +1,6 @@
 const path = require('path');
 const fs = require('fs').promises;
+const { companionUrl, pageUrl } = require('../companionPath');
 
 function matchAny(value, patterns) {
   if (!patterns || patterns.length === 0) return false;
@@ -32,17 +33,6 @@ function toRegExp(glob) {
   return new RegExp('^' + re + '$');
 }
 
-function companionUrl(permalink, trailingSlash) {
-  if (permalink === '/' || permalink === '') {
-    return '/index.md';
-  }
-  const cleaned = permalink.replace(/\/+$/, '');
-  if (trailingSlash === false) {
-    return `${cleaned}.md`;
-  }
-  return `${cleaned}/index.md`;
-}
-
 function groupKind(kind, pluginId) {
   if (kind === 'docs') return 'docs';
   if (kind === 'blog') return 'blog';
@@ -149,30 +139,20 @@ function buildSections(filtered, options, verbose) {
   return sections;
 }
 
-module.exports = async function emitLlmsTxt({
-  props,
-  options,
-  emittedCompanions,
-  companionsEnabled,
-  verbose,
-}) {
-  const { outDir, siteConfig } = props;
-  const trailingSlash = siteConfig.trailingSlash;
-
-  // Filter the companions through include/exclude rules.
-  const filtered = emittedCompanions.filter((item) => {
-    if (options.include && !options.include.some((p) => toRegExp(p).test(item.permalink))) {
-      return false;
-    }
-    if (matchAny(item.permalink, options.exclude)) {
-      return false;
-    }
-    return true;
-  });
-
-  const sections = buildSections(filtered, options, verbose);
+// Normalised view of `llmsTxt.fullTxt`: the boolean form (`true` = every
+// section, no cap) or the object form `{ include, maxBytes }`.
+function fullTxtOptions(fullTxt) {
+  if (!fullTxt) return null;
+  if (fullTxt === true) return { include: null, maxBytes: null };
+  return {
+    include: Array.isArray(fullTxt.include) && fullTxt.include.length > 0 ? fullTxt.include : null,
+    maxBytes: Number.isFinite(fullTxt.maxBytes) && fullTxt.maxBytes > 0 ? fullTxt.maxBytes : null,
+  };
+}
 
-  // Build llms.txt.
+// Render llms.txt from the sections. Pure, so it can be tested without a
+// build: returns the file text.
+function renderLlmsTxt({ siteConfig, siteUrl, trailingSlash, sections, options, companionsEnabled }) {
   const lines = [];
   lines.push(`# ${siteConfig.title}`);
   lines.push('');
@@ -189,9 +169,11 @@ module.exports = async function emitLlmsTxt({
     lines.push(`## ${section.title}`);
     lines.push('');
     for (const item of section.items) {
+      // Absolute, as the llms.txt spec's examples are: the file is read
+      // away from the site, where a relative link has no base.
       const url = companionsEnabled
-        ? companionUrl(item.permalink, trailingSlash)
-        : item.permalink;
+        ? companionUrl(siteUrl, item.permalink, trailingSlash)
+        : pageUrl(siteUrl, item.permalink);
       const desc = item.description || siteConfig.tagline || '';
       const titleText = item.title || item.permalink;
       if (desc) {
@@ -203,8 +185,68 @@ module.exports = async function emitLlmsTxt({
     lines.push('');
   }
 
+  // The llmstxt.org convention for secondary resources: an "Optional"
+  // section a reader can skip. llms-full.txt is the whole corpus in one
+  // file, so it belongs here rather than inline with the page list.
+  if (options.fullTxt && options.linkFullTxt !== false && companionsEnabled) {
+    lines.push('## Optional');
+    lines.push('');
+    lines.push(
+      `- [Full text of every page](${siteUrl}/llms-full.txt): one file with the content of every page listed above`,
+    );
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+// One block of llms-full.txt: the companion body with a `Source:` line (and
+// the title, when the body does not already open with it). A plain
+// companion already carries both, so the header is skipped and the body
+// is emitted as is.
+function fullTxtBlock({ body, item, siteUrl }) {
+  const url = pageUrl(siteUrl, item.permalink);
+  const head = body.trimStart().split('\n', 8);
+  const hasH1 = /^#\s+\S/.test(head[0] || '');
+  const hasSource = head.some((l) => /^Source: \S/.test(l));
+  const header = [
+    ...(hasH1 ? [] : [`# ${item.title || item.permalink}`, '']),
+    ...(hasSource ? [] : [`Source: ${url}`, '']),
+  ];
+  const prefix = header.length ? `${header.join('\n')}\n` : '';
+  return `${prefix}${body.trim()}\n`;
+}
+
+module.exports = async function emitLlmsTxt({
+  props,
+  options,
+  emittedCompanions,
+  companionsEnabled,
+  verbose,
+}) {
+  const { outDir, siteConfig } = props;
+  const trailingSlash = siteConfig.trailingSlash;
+  const siteUrl = (siteConfig.url || '').replace(/\/+$/, '');
+
+  // Filter the companions through include/exclude rules.
+  const filtered = emittedCompanions.filter((item) => {
+    if (options.include && !options.include.some((p) => toRegExp(p).test(item.permalink))) {
+      return false;
+    }
+    if (matchAny(item.permalink, options.exclude)) {
+      return false;
+    }
+    return true;
+  });
+
+  const sections = buildSections(filtered, options, verbose);
+
   const llmsTxtPath = path.join(outDir, 'llms.txt');
-  await fs.writeFile(llmsTxtPath, lines.join('\n'), 'utf8');
+  await fs.writeFile(
+    llmsTxtPath,
+    renderLlmsTxt({ siteConfig, siteUrl, trailingSlash, sections, options, companionsEnabled }),
+    'utf8',
+  );
   if (verbose) {
     console.log(`[plugin-aeo] llmsTxt: wrote ${llmsTxtPath}`);
   }
@@ -212,7 +254,8 @@ module.exports = async function emitLlmsTxt({
   // Build llms-full.txt by concatenating the companion files that we just
   // emitted in feature 1. Requires feature 1 to have been enabled. Section
   // headings mirror llms.txt so an LLM can navigate the corpus by H2.
-  if (options.fullTxt) {
+  const full = fullTxtOptions(options.fullTxt);
+  if (full) {
     if (!companionsEnabled) {
       if (verbose) {
         console.log(
@@ -222,9 +265,26 @@ module.exports = async function emitLlmsTxt({
       return;
     }
 
-    const sectionChunks = [];
-    for (const section of sections) {
-      const blocks = [];
+    // `include` names content instances ("<plugin>@<id>") or, with per-type
+    // grouping, the group keys (docs, blog, pages). Sections keep the order
+    // of llms.txt; `maxBytes` stops appending once the next block would
+    // cross the cap, so the file always ends on a whole page.
+    const wanted = full.include
+      ? sections.filter((s) => full.include.includes(s.key))
+      : sections;
+    if (full.include && wanted.length === 0 && verbose) {
+      console.log(
+        `[plugin-aeo] llmsTxt: fullTxt.include matched no section (have: ${sections.map((s) => s.key).join(', ')})`,
+      );
+    }
+
+    const SEP = '\n---\n\n';
+    let out = '';
+    let size = 0;
+    let truncated = false;
+    let pages = 0;
+    outer: for (const section of wanted) {
+      let sectionOpen = false;
       for (const item of section.items) {
         let body;
         try {
@@ -237,23 +297,31 @@ module.exports = async function emitLlmsTxt({
           }
           continue;
         }
-        const url = `${(siteConfig.url || '').replace(/\/$/, '')}${item.permalink}`;
-        const header = [
-          `# ${item.title || item.permalink}`,
-          '',
-          `Source: ${url}`,
-          '',
-        ].join('\n');
-        blocks.push(`${header}${body.trim()}\n`);
+        const block = fullTxtBlock({ body, item, siteUrl });
+        const chunk = `${sectionOpen ? SEP : `${out ? SEP : ''}## ${section.title}\n\n`}${block}`;
+        const bytes = Buffer.byteLength(chunk, 'utf8');
+        if (full.maxBytes && size + bytes > full.maxBytes) {
+          truncated = true;
+          break outer;
+        }
+        out += chunk;
+        size += bytes;
+        sectionOpen = true;
+        pages++;
       }
-      if (blocks.length === 0) continue;
-      sectionChunks.push(`## ${section.title}\n\n${blocks.join('\n---\n\n')}`);
     }
 
     const fullPath = path.join(outDir, 'llms-full.txt');
-    await fs.writeFile(fullPath, sectionChunks.join('\n---\n\n'), 'utf8');
+    await fs.writeFile(fullPath, out, 'utf8');
     if (verbose) {
-      console.log(`[plugin-aeo] llmsTxt: wrote ${fullPath}`);
+      console.log(
+        `[plugin-aeo] llmsTxt: wrote ${fullPath} (${pages} page(s), ${size} bytes${truncated ? `, stopped at the ${full.maxBytes}-byte cap` : ''})`,
+      );
     }
   }
 };
+
+module.exports.renderLlmsTxt = renderLlmsTxt;
+module.exports.fullTxtBlock = fullTxtBlock;
+module.exports.fullTxtOptions = fullTxtOptions;
+module.exports.buildSections = buildSections;