@stackql/docusaurus-plugin-aeo 0.1.0

package/src/features/companions.js

@@ -0,0 +1,234 @@
+const path = require('path');
+const fs = require('fs').promises;
+const matter = require('gray-matter');
+
+// Minimal glob matcher for route paths. Patterns like "/blog/tags/**",
+// "/search". Avoids pulling micromatch directly.
+function matchAny(value, patterns) {
+  if (!patterns || patterns.length === 0) return false;
+  for (const p of patterns) {
+    if (toRegExp(p).test(value)) return true;
+  }
+  return false;
+}
+
+function toRegExp(glob) {
+  // Minimal globstar -> regex translation. Supports: **, *, ?, character
+  // classes left untouched. Good enough for route paths like
+  // "/blog/tags/**" and "/search".
+  let re = '';
+  for (let i = 0; i < glob.length; i++) {
+    const c = glob[i];
+    if (c === '*') {
+      if (glob[i + 1] === '*') {
+        re += '.*';
+        i++;
+        if (glob[i + 1] === '/') i++;
+      } else {
+        re += '[^/]*';
+      }
+    } else if (c === '?') {
+      re += '[^/]';
+    } else if ('.+^$()|{}[]\\'.includes(c)) {
+      re += '\\' + c;
+    } else {
+      re += c;
+    }
+  }
+  return new RegExp('^' + re + '$');
+}
+
+function stripMdx(source) {
+  let out = source;
+  // Strip import / export lines (top-of-file MDX).
+  out = out.replace(/^\s*import\s+[^\n]+\n/gm, '');
+  out = out.replace(/^\s*export\s+[^\n]+\n/gm, '');
+  // Strip self-closing JSX tags like <Component prop="x" />.
+  out = out.replace(/<([A-Z][A-Za-z0-9]*)\b[^>]*\/>\s*/g, '');
+  // Strip paired JSX blocks like <Foo>...</Foo> (non-greedy, single component).
+  out = out.replace(
+    /<([A-Z][A-Za-z0-9]*)\b[^>]*>[\s\S]*?<\/\1>\s*/g,
+    '',
+  );
+  // Strip stray opening or closing component tags left behind.
+  out = out.replace(/<\/?[A-Z][A-Za-z0-9]*\b[^>]*>/g, '');
+  // Collapse 3+ blank lines.
+  out = out.replace(/\n{3,}/g, '\n\n');
+  return out;
+}
+
+function routeToCompanionPath(routePath, trailingSlash, outDir) {
+  // Docusaurus emits either /foo/index.html (trailingSlash: true / undefined)
+  // or /foo.html (trailingSlash: false). The companion mirrors the HTML.
+  let rel;
+  if (routePath === '/' || routePath === '') {
+    rel = 'index.md';
+  } else {
+    const cleaned = routePath.replace(/^\/+|\/+$/g, '');
+    if (trailingSlash === false) {
+      rel = `${cleaned}.md`;
+    } else {
+      // true or undefined (Docusaurus default behavior: index.html in folder)
+      rel = `${cleaned}/index.md`;
+    }
+  }
+  return path.join(outDir, rel);
+}
+
+function collectFromDocs(content, pluginName, pluginId) {
+  // Docs plugin loaded content shape:
+  //   { loadedVersions: [{ docs: [{ id, permalink, source, title, description, frontMatter }] }] }
+  const results = [];
+  if (!content || !Array.isArray(content.loadedVersions)) return results;
+  for (const version of content.loadedVersions) {
+    if (!Array.isArray(version.docs)) continue;
+    for (const doc of version.docs) {
+      if (!doc.permalink || !doc.source) continue;
+      results.push({
+        permalink: doc.permalink,
+        // doc.source is prefixed with "@site/" - resolve later against siteDir.
+        sourceRef: doc.source,
+        title: doc.title,
+        description: doc.description,
+        frontMatter: doc.frontMatter || {},
+        kind: 'docs',
+        pluginName,
+        pluginId,
+      });
+    }
+  }
+  return results;
+}
+
+function collectFromBlog(content, pluginName, pluginId) {
+  // Blog plugin loaded content shape:
+  //   { blogPosts: [{ id, metadata: { permalink, source, title, description, frontMatter } }] }
+  const results = [];
+  if (!content || !Array.isArray(content.blogPosts)) return results;
+  for (const post of content.blogPosts) {
+    const m = post.metadata || {};
+    if (!m.permalink || !m.source) continue;
+    results.push({
+      permalink: m.permalink,
+      sourceRef: m.source,
+      title: m.title,
+      description: m.description,
+      frontMatter: m.frontMatter || {},
+      kind: 'blog',
+      pluginName,
+      pluginId,
+    });
+  }
+  return results;
+}
+
+function resolveSourcePath(sourceRef, siteDir) {
+  if (!sourceRef) return null;
+  // Docusaurus content references look like "@site/docs/intro.md".
+  if (sourceRef.startsWith('@site/')) {
+    return path.join(siteDir, sourceRef.slice('@site/'.length));
+  }
+  if (path.isAbsolute(sourceRef)) return sourceRef;
+  return path.join(siteDir, sourceRef);
+}
+
+async function ensureDir(p) {
+  await fs.mkdir(path.dirname(p), { recursive: true });
+}
+
+module.exports = async function emitCompanions({
+  props,
+  options,
+  loadedContentByPlugin,
+  verbose,
+}) {
+  const { outDir, siteConfig, siteDir } = props;
+  const trailingSlash = siteConfig.trailingSlash;
+  const excludePatterns = options.exclude || [];
+  const format = options.format;
+
+  // Build the work list from captured loadedContent.
+  const items = [];
+  for (const [, entry] of loadedContentByPlugin) {
+    const { pluginName, pluginId, content } = entry;
+    if (pluginName === 'docusaurus-plugin-content-docs') {
+      items.push(...collectFromDocs(content, pluginName, pluginId));
+    } else if (pluginName === 'docusaurus-plugin-content-blog') {
+      items.push(...collectFromBlog(content, pluginName, pluginId));
+    }
+    // Custom pages plugin emits React components, not markdown. Skip silently.
+  }
+
+  const emitted = [];
+  for (const item of items) {
+    if (matchAny(item.permalink, excludePatterns)) {
+      if (verbose) {
+        console.log(`[plugin-aeo] companions: excluded ${item.permalink}`);
+      }
+      continue;
+    }
+
+    const sourcePath = resolveSourcePath(item.sourceRef, siteDir);
+    if (!sourcePath) {
+      if (verbose) {
+        console.log(
+          `[plugin-aeo] companions: skipping ${item.permalink} (no source)`,
+        );
+      }
+      continue;
+    }
+
+    let raw;
+    try {
+      raw = await fs.readFile(sourcePath, 'utf8');
+    } catch (e) {
+      if (verbose) {
+        console.log(
+          `[plugin-aeo] companions: failed to read source for ${item.permalink}: ${e.message}`,
+        );
+      }
+      continue;
+    }
+
+    let body = raw;
+    if (format === 'plain') {
+      // Strip frontmatter, then strip MDX-specific bits.
+      const parsed = matter(raw);
+      body = stripMdx(parsed.content);
+      // Re-prepend a minimal "title + description" block so an LLM can see them
+      // without parsing YAML.
+      const prefixParts = [];
+      if (item.title) prefixParts.push(`# ${item.title}`);
+      if (item.description) prefixParts.push(`> ${item.description}`);
+      if (prefixParts.length > 0) {
+        body = `${prefixParts.join('\n\n')}\n\n${body.trim()}\n`;
+      }
+    }
+
+    const target = routeToCompanionPath(item.permalink, trailingSlash, outDir);
+    await ensureDir(target);
+    await fs.writeFile(target, body, 'utf8');
+
+    emitted.push({
+      permalink: item.permalink,
+      companionPath: target,
+      title: item.title,
+      description: item.description,
+      kind: item.kind,
+      pluginName: item.pluginName,
+      pluginId: item.pluginId,
+      frontMatter: item.frontMatter,
+    });
+
+    if (verbose) {
+      console.log(
+        `[plugin-aeo] companions: emitted ${path.relative(outDir, target)}`,
+      );
+    }
+  }
+
+  if (verbose) {
+    console.log(`[plugin-aeo] companions: emitted ${emitted.length} file(s)`);
+  }
+  return emitted;
+};

package/src/features/llmsTxt.js

@@ -0,0 +1,175 @@
+const path = require('path');
+const fs = require('fs').promises;
+
+function matchAny(value, patterns) {
+  if (!patterns || patterns.length === 0) return false;
+  for (const p of patterns) {
+    if (toRegExp(p).test(value)) return true;
+  }
+  return false;
+}
+
+function toRegExp(glob) {
+  let re = '';
+  for (let i = 0; i < glob.length; i++) {
+    const c = glob[i];
+    if (c === '*') {
+      if (glob[i + 1] === '*') {
+        re += '.*';
+        i++;
+        if (glob[i + 1] === '/') i++;
+      } else {
+        re += '[^/]*';
+      }
+    } else if (c === '?') {
+      re += '[^/]';
+    } else if ('.+^$()|{}[]\\'.includes(c)) {
+      re += '\\' + c;
+    } else {
+      re += c;
+    }
+  }
+  return new RegExp('^' + re + '$');
+}
+
+function companionUrl(permalink, trailingSlash) {
+  if (permalink === '/' || permalink === '') {
+    return '/index.md';
+  }
+  const cleaned = permalink.replace(/\/+$/, '');
+  if (trailingSlash === false) {
+    return `${cleaned}.md`;
+  }
+  return `${cleaned}/index.md`;
+}
+
+function groupKind(kind, pluginId) {
+  if (kind === 'docs') return 'docs';
+  if (kind === 'blog') return 'blog';
+  if (kind === 'pages') return 'pages';
+  // Treat additional content-docs instances (id !== 'default') as docs too.
+  if (pluginId && pluginId !== 'default') return 'docs';
+  return 'pages';
+}
+
+module.exports = async function emitLlmsTxt({
+  props,
+  options,
+  emittedCompanions,
+  companionsEnabled,
+  verbose,
+}) {
+  const { outDir, siteConfig } = props;
+  const trailingSlash = siteConfig.trailingSlash;
+
+  // Filter the companions through include/exclude rules.
+  const filtered = emittedCompanions.filter((item) => {
+    if (options.include && !options.include.some((p) => toRegExp(p).test(item.permalink))) {
+      return false;
+    }
+    if (matchAny(item.permalink, options.exclude)) {
+      return false;
+    }
+    return true;
+  });
+
+  // Group by kind.
+  const groups = { docs: [], blog: [], pages: [] };
+  for (const item of filtered) {
+    const g = groupKind(item.kind, item.pluginId);
+    groups[g].push(item);
+  }
+
+  // Build llms.txt.
+  const lines = [];
+  lines.push(`# ${siteConfig.title}`);
+  lines.push('');
+  if (siteConfig.tagline) {
+    lines.push(`> ${siteConfig.tagline}`);
+    lines.push('');
+  }
+  if (options.header) {
+    lines.push(options.header.trim());
+    lines.push('');
+  }
+
+  const sectionOrder = [
+    ['docs', options.sections.docs],
+    ['blog', options.sections.blog],
+    ['pages', options.sections.pages],
+  ];
+
+  for (const [key, title] of sectionOrder) {
+    const items = groups[key];
+    if (!items || items.length === 0) continue;
+    lines.push(`## ${title}`);
+    lines.push('');
+    // Stable order: by permalink.
+    items.sort((a, b) => a.permalink.localeCompare(b.permalink));
+    for (const item of items) {
+      const url = companionsEnabled
+        ? companionUrl(item.permalink, trailingSlash)
+        : item.permalink;
+      const desc = item.description || siteConfig.tagline || '';
+      const titleText = item.title || item.permalink;
+      if (desc) {
+        lines.push(`- [${titleText}](${url}): ${desc}`);
+      } else {
+        lines.push(`- [${titleText}](${url})`);
+      }
+    }
+    lines.push('');
+  }
+
+  const llmsTxtPath = path.join(outDir, 'llms.txt');
+  await fs.writeFile(llmsTxtPath, lines.join('\n'), 'utf8');
+  if (verbose) {
+    console.log(`[plugin-aeo] llmsTxt: wrote ${llmsTxtPath}`);
+  }
+
+  // Build llms-full.txt by concatenating the companion files that we just
+  // emitted in feature 1. Requires feature 1 to have been enabled.
+  if (options.fullTxt) {
+    if (!companionsEnabled) {
+      if (verbose) {
+        console.log(
+          '[plugin-aeo] llmsTxt: companions disabled, skipping llms-full.txt',
+        );
+      }
+      return;
+    }
+
+    const blocks = [];
+    for (const [key] of sectionOrder) {
+      const items = groups[key];
+      if (!items) continue;
+      for (const item of items) {
+        let body;
+        try {
+          body = await fs.readFile(item.companionPath, 'utf8');
+        } catch (e) {
+          if (verbose) {
+            console.log(
+              `[plugin-aeo] llmsTxt: cannot read ${item.companionPath}: ${e.message}`,
+            );
+          }
+          continue;
+        }
+        const url = `${(siteConfig.url || '').replace(/\/$/, '')}${item.permalink}`;
+        const header = [
+          `# ${item.title || item.permalink}`,
+          '',
+          `Source: ${url}`,
+          '',
+        ].join('\n');
+        blocks.push(`${header}${body.trim()}\n`);
+      }
+    }
+
+    const fullPath = path.join(outDir, 'llms-full.txt');
+    await fs.writeFile(fullPath, blocks.join('\n---\n\n'), 'utf8');
+    if (verbose) {
+      console.log(`[plugin-aeo] llmsTxt: wrote ${fullPath}`);
+    }
+  }
+};

package/src/helpers.js

@@ -0,0 +1,61 @@
+// Helpers exported as "@stackql/docusaurus-plugin-aeo/helpers".
+//
+// These are pure data helpers - consumers spread them into other plugin
+// configs so that /ai/* routes don't pollute artifacts aimed at humans
+// (sitemaps, breadcrumb graphs, etc.).
+
+// Glob patterns matching the /ai/* convention. Suitable for plugins whose
+// exclude lists accept globs (notably @docusaurus/plugin-sitemap, which
+// uses `ignorePatterns`).
+const AI_ROUTE_PATTERNS = ['/ai', '/ai/**'];
+
+// Sitemap slice for @docusaurus/plugin-sitemap (standalone or via preset).
+// Usage:
+//   [
+//     '@docusaurus/plugin-sitemap',
+//     { ...sitemapExclude, /* your other sitemap options */ },
+//   ]
+const sitemapExclude = {
+  ignorePatterns: [...AI_ROUTE_PATTERNS],
+};
+
+// Predicate consumers can use in custom code or when building an exact
+// excludedRoutes list for plugins that don't accept globs (e.g.
+// @stackql/docusaurus-plugin-structured-data, whose `excludedRoutes`
+// is an exact-match array).
+function isAiRoute(permalink) {
+  return typeof permalink === 'string' && permalink.startsWith('/ai/');
+}
+
+// Build an exact-match exclude list for @stackql/docusaurus-plugin-
+// structured-data. That plugin compares `themeConfig.structuredData
+// .excludedRoutes` with strict equality, so globs don't work there.
+//
+// Pass an array of route permalinks (typically the docs/blog plugin
+// outputs, or your own enumerated list); the function returns the subset
+// that matches /ai/*. Spread the result into your structuredData config:
+//
+//   themeConfig: {
+//     structuredData: {
+//       excludedRoutes: [
+//         ...buildStructuredDataAiExcludes(allRoutePaths),
+//         // ...other exact-match routes you want to skip
+//       ],
+//       // ...rest of structuredData config
+//     },
+//   }
+//
+// If you don't have the route list at config time, list the known
+// /ai/* parents by hand (e.g. ['/ai', '/ai/faqs/install', ...]); the
+// glob-based AI_ROUTE_PATTERNS will NOT work for that plugin.
+function buildStructuredDataAiExcludes(allRoutePaths) {
+  if (!Array.isArray(allRoutePaths)) return [];
+  return allRoutePaths.filter(isAiRoute).concat(['/ai']);
+}
+
+module.exports = {
+  AI_ROUTE_PATTERNS,
+  sitemapExclude,
+  isAiRoute,
+  buildStructuredDataAiExcludes,
+};

package/src/index.js

@@ -0,0 +1,200 @@
+const path = require('path');
+const emitCompanions = require('./features/companions');
+const emitLlmsTxt = require('./features/llmsTxt');
+const { validateAiRoutes } = require('./features/aiRoutes');
+
+const DEFAULT_LLMS_EXCLUDE = [
+  '/search',
+  '/404',
+  '/blog/tags/**',
+  '/blog/page/**',
+  '/blog/archive',
+  '/blog/authors/**',
+];
+
+const DEFAULT_PROVIDER_ORDER = ['claude', 'chatgpt', 'perplexity', 'gemini'];
+const VALID_PROVIDERS = new Set(DEFAULT_PROVIDER_ORDER);
+const DEFAULT_PROMPT_TEMPLATE =
+  'Read {pageUrl}.md and help me with the following question about it: ';
+
+function normalizeOptions(raw) {
+  const opts = raw || {};
+  const companions = opts.companions || {};
+  const llmsTxt = opts.llmsTxt || {};
+  const askAi = opts.askAi || {};
+  const aiRoutes = opts.aiRoutes || {};
+  const llmsSections = llmsTxt.sections || {};
+
+  return {
+    companions: {
+      enabled: companions.enabled !== false,
+      format: companions.format || 'raw',
+      exclude: companions.exclude || [],
+    },
+    llmsTxt: {
+      enabled: llmsTxt.enabled !== false,
+      exclude: llmsTxt.exclude || DEFAULT_LLMS_EXCLUDE,
+      include: llmsTxt.include || null,
+      header: llmsTxt.header || null,
+      sections: {
+        docs: llmsSections.docs || 'Documentation',
+        blog: llmsSections.blog || 'Blog',
+        pages: llmsSections.pages || 'Pages',
+      },
+      fullTxt: llmsTxt.fullTxt !== false,
+    },
+    askAi: {
+      enabled: askAi.enabled !== false,
+      providerOrder: askAi.providerOrder || DEFAULT_PROVIDER_ORDER,
+      promptTemplate: askAi.promptTemplate || DEFAULT_PROMPT_TEMPLATE,
+      placement: askAi.placement || 'doc-footer',
+    },
+    aiRoutes: {
+      validate: aiRoutes.validate === true,
+    },
+    verbose: opts.verbose === true,
+  };
+}
+
+function validateOptions(opts) {
+  const errs = [];
+
+  if (!['raw', 'plain'].includes(opts.companions.format)) {
+    errs.push(
+      `companions.format must be "raw" or "plain", got "${opts.companions.format}"`,
+    );
+  }
+  if (!Array.isArray(opts.companions.exclude)) {
+    errs.push('companions.exclude must be an array of glob patterns');
+  }
+
+  if (!Array.isArray(opts.llmsTxt.exclude)) {
+    errs.push('llmsTxt.exclude must be an array of glob patterns');
+  }
+  if (opts.llmsTxt.include !== null && !Array.isArray(opts.llmsTxt.include)) {
+    errs.push('llmsTxt.include must be an array of glob patterns or null');
+  }
+  if (opts.llmsTxt.header !== null && typeof opts.llmsTxt.header !== 'string') {
+    errs.push('llmsTxt.header must be a string or null');
+  }
+
+  if (!Array.isArray(opts.askAi.providerOrder)) {
+    errs.push('askAi.providerOrder must be an array');
+  } else {
+    for (const p of opts.askAi.providerOrder) {
+      if (!VALID_PROVIDERS.has(p)) {
+        errs.push(
+          `askAi.providerOrder contains unknown provider "${p}". Valid: ${[...VALID_PROVIDERS].join(', ')}`,
+        );
+      }
+    }
+  }
+  if (typeof opts.askAi.promptTemplate !== 'string') {
+    errs.push('askAi.promptTemplate must be a string');
+  }
+  if (!['doc-footer', 'none'].includes(opts.askAi.placement)) {
+    errs.push(
+      `askAi.placement must be "doc-footer" or "none", got "${opts.askAi.placement}"`,
+    );
+  }
+
+  if (errs.length > 0) {
+    throw new Error(
+      `@stackql/docusaurus-plugin-aeo: invalid options:\n  - ${errs.join('\n  - ')}`,
+    );
+  }
+}
+
+module.exports = function pluginAeo(context, rawOptions) {
+  const options = normalizeOptions(rawOptions);
+  validateOptions(options);
+
+  // Captured from contentLoaded for downstream postBuild use.
+  // Map<pluginName, { plugin: { name, id }, content: any }>
+  const loadedContentByPlugin = new Map();
+
+  return {
+    name: '@stackql/docusaurus-plugin-aeo',
+
+    getThemePath() {
+      if (!options.askAi.enabled || options.askAi.placement === 'none') {
+        return undefined;
+      }
+      return path.resolve(__dirname, './theme');
+    },
+
+    getClientModules() {
+      return [];
+    },
+
+    // Surface the askAi config to theme components via a global data
+    // injection. Docusaurus client code can read this through useDocusaurusContext().
+    async contentLoaded({ actions, allContent }) {
+      if (allContent) {
+        for (const [pluginName, byId] of Object.entries(allContent)) {
+          if (!byId) continue;
+          for (const [pluginId, content] of Object.entries(byId)) {
+            loadedContentByPlugin.set(`${pluginName}@${pluginId}`, {
+              pluginName,
+              pluginId,
+              content,
+            });
+          }
+        }
+      }
+
+      await actions.setGlobalData({
+        askAi: {
+          enabled: options.askAi.enabled,
+          providerOrder: options.askAi.providerOrder,
+          promptTemplate: options.askAi.promptTemplate,
+          placement: options.askAi.placement,
+          companionsEnabled: options.companions.enabled,
+        },
+      });
+
+      if (options.aiRoutes.validate) {
+        validateAiRoutes({
+          loadedContentByPlugin,
+          verbose: options.verbose,
+        });
+      }
+    },
+
+    async postBuild(props) {
+      let emittedCompanions = [];
+
+      if (options.companions.enabled) {
+        emittedCompanions = await emitCompanions({
+          props,
+          options: options.companions,
+          loadedContentByPlugin,
+          verbose: options.verbose,
+        });
+      } else if (options.verbose) {
+        console.log('[plugin-aeo] companions disabled, skipping feature 1');
+      }
+
+      if (options.llmsTxt.enabled) {
+        await emitLlmsTxt({
+          props,
+          options: options.llmsTxt,
+          emittedCompanions,
+          companionsEnabled: options.companions.enabled,
+          verbose: options.verbose,
+        });
+      } else if (options.verbose) {
+        console.log('[plugin-aeo] llmsTxt disabled, skipping feature 2');
+      }
+    },
+  };
+};
+
+module.exports.validateOptions = function validateDocusaurusOptions({
+  options,
+  validate: _validate,
+}) {
+  // Docusaurus passes a Joi validator we deliberately don't use - the plugin
+  // entry runs its own handwritten validator at construction time.
+  return options || {};
+};