@redocly/realm-plugin-asciidoc 0.1.0

package/src/get-ai-search-documents.ts

@@ -0,0 +1,193 @@
+import type { SearchFacet } from '@redocly/theme';
+import type { PageRouteDetails } from '@redocly/realm/dist/server/types';
+import type { AsciidocSection, AsciidocFrontmatter } from './types.js';
+
+import { extractDocumentSearchFacets } from './search-facets.js';
+
+type AiDocumentsStore = {
+  getLLMsTxts: () => Promise<
+    {
+      content: string;
+      title: string;
+      description?: string;
+      slug: string;
+      fsPath: string;
+      includeInLLMsTxt: boolean;
+    }[]
+  >;
+  getSearchDocuments: () => Promise<
+    {
+      url: string;
+      fsPath: string;
+      content: string;
+      title: string;
+      locale: string;
+      product?: string;
+      facets?: Record<string, string>;
+    }[]
+  >;
+};
+
+const AI_SEARCH_CHUNK_SIZE = 50;
+const AI_SEARCH_DOCUMENT_CHUNK_SIZE = 250;
+
+export function getAiDocumentsStore(
+  frontmatter: AsciidocFrontmatter,
+  sections: AsciidocSection[],
+  getSearchFacets: () => Map<string, SearchFacet>,
+): (
+  route: PageRouteDetails,
+  staticData: unknown,
+  context: unknown,
+  actions: unknown,
+) => Promise<AiDocumentsStore | undefined> {
+  return async (
+    route: PageRouteDetails,
+    _staticData: unknown,
+    _context: unknown,
+    _actions: unknown,
+  ): Promise<AiDocumentsStore | undefined> => {
+    if (frontmatter.excludeFromSearch) return;
+
+    const pageTitle = frontmatter.seo?.title || frontmatter.title || route.slug;
+    const description = frontmatter.seo?.description || frontmatter.description;
+
+    return {
+      async getLLMsTxts() {
+        const body = sectionsToMarkdown(sections);
+        const content = `# ${pageTitle}\n\n${body}`;
+
+        return [
+          {
+            title: pageTitle,
+            description,
+            slug: route.slug,
+            fsPath: route.fsPath,
+            content,
+            includeInLLMsTxt: true,
+          },
+        ];
+      },
+      async getSearchDocuments() {
+        const metadata = route.metadata || {};
+        const facets = extractDocumentSearchFacets(metadata, getSearchFacets);
+        const chunks = createContentChunks(sections, pageTitle, route.slug);
+        const mergedChunks = mergeChunks(chunks);
+
+        return mergedChunks
+          .filter((chunk) => chunk.content)
+          .map((chunk) => ({
+            title: chunk.title,
+            url: chunk.url,
+            content: chunk.toc
+              ? `${chunk.toc}\n${chunk.content}`
+              : `# ${pageTitle}\n${chunk.content}`,
+            fsPath: route.fsPath,
+            locale: detectLocaleFromFsPath(route.fsPath),
+            product: route.product?.name,
+            facets,
+          }));
+      },
+    };
+  };
+}
+
+type ContentChunk = {
+  title: string;
+  content: string;
+  url: string;
+  level: number;
+  toc: string;
+};
+
+function sectionsToMarkdown(sections: AsciidocSection[], headingLevel = 1): string {
+  const parts: string[] = [];
+
+  for (const section of sections) {
+    const prefix = '#'.repeat(headingLevel + 1);
+    parts.push(`${prefix} ${section.title}`);
+
+    if (section.textContent) {
+      parts.push(section.textContent);
+    }
+
+    if (section.children.length > 0) {
+      parts.push(sectionsToMarkdown(section.children, headingLevel + 1));
+    }
+  }
+
+  return parts.join('\n\n');
+}
+
+function createContentChunks(
+  sections: AsciidocSection[],
+  pageTitle: string,
+  pageUrl: string,
+): ContentChunk[] {
+  const chunks: ContentChunk[] = [];
+  const headingStack: { title: string; level: number }[] = [];
+
+  function processSection(section: AsciidocSection) {
+    // Update heading stack
+    headingStack.splice(section.level - 1);
+    headingStack.push({ title: section.title, level: section.level });
+
+    const sectionUrl = `${pageUrl}#${section.id}`;
+    const toc = headingStack.map((h) => `${'#'.repeat(h.level + 1)} ${h.title}`).join('\n');
+    const chunkTitle =
+      section.level > 1 ? `${pageTitle} → ${section.title}` : section.title || pageTitle;
+
+    chunks.push({
+      title: chunkTitle,
+      content: section.textContent,
+      url: sectionUrl,
+      level: section.level,
+      toc,
+    });
+
+    for (const child of section.children) {
+      processSection(child);
+    }
+  }
+
+  for (const section of sections) {
+    processSection(section);
+  }
+
+  return chunks;
+}
+
+function estimateChunkSize(content: string): number {
+  return content.length / 4;
+}
+
+function mergeChunks(chunks: ContentChunk[]): ContentChunk[] {
+  if (chunks.length === 0) return [];
+  const mergedChunks: ContentChunk[] = [];
+  let currentChunk = chunks[0];
+
+  for (const chunk of chunks.slice(1)) {
+    const currentChunkSize = estimateChunkSize(currentChunk.content);
+    const chunkSize = estimateChunkSize(chunk.content);
+    const isLastChunk = chunks.indexOf(chunk) === chunks.length - 1;
+
+    if (
+      chunkSize < AI_SEARCH_CHUNK_SIZE &&
+      (currentChunkSize < AI_SEARCH_DOCUMENT_CHUNK_SIZE || isLastChunk) &&
+      chunk.level > currentChunk.level
+    ) {
+      currentChunk.content += '\n' + chunk.content;
+    } else {
+      mergedChunks.push(currentChunk);
+      currentChunk = chunk;
+    }
+  }
+
+  mergedChunks.push(currentChunk);
+  return mergedChunks;
+}
+
+function detectLocaleFromFsPath(relativePath: string): string {
+  const localeCandidate = relativePath.split('/')[0];
+  return /^[a-z]{2}-[A-Z]{2}$/.test(localeCandidate) ? localeCandidate : '';
+}

package/src/plugin.ts

@@ -0,0 +1,483 @@
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, copyFileSync, readFileSync } from 'node:fs';
+import { fileURLToPath } from 'url';
+
+import type { ExternalPlugin, GetStaticDataContext } from '@redocly/realm/dist/server/types';
+import type { ParsedAsciidoc } from './asciidoc-parser.js';
+import type { ContentNode } from './types.js';
+
+import { getPathPrefix, withPathPrefix } from '@redocly/theme/core/utils';
+
+import { searchResolver } from './search-resolver.js';
+import { getAiDocumentsStore } from './get-ai-search-documents.js';
+import { parseAsciidoc } from './asciidoc-parser.js';
+
+const ASCIIDOC_TEMPLATE_ID = 'asciidoc-docs';
+
+export default function asciidocPlugin(): ExternalPlugin {
+  return {
+    id: 'asciidoc',
+    loaders: {
+      asciidoc: async (relativePath, context) => {
+        const content = await context.fs.read(relativePath);
+        return parseAsciidoc(content, relativePath, context.fs.cwd);
+      },
+    },
+    processContent: async (actions, context) => {
+      const asciidocTemplateId = actions.createTemplate(
+        ASCIIDOC_TEMPLATE_ID,
+        fromCurrentDir(import.meta.url, './template.js'),
+      );
+
+      for (const record of context.fs.scan(/\.adoc$/)) {
+        if (await context.isPathIgnored(record.relativePath)) {
+          continue;
+        }
+
+        const { relativePath } = record;
+
+        try {
+          const { data: parsed } = await context.cache.load<ParsedAsciidoc>(
+            relativePath,
+            'asciidoc',
+          );
+
+          if (!parsed) continue;
+
+          const { title, attributes, sections, contentNodes, links } = parsed;
+
+          // Resolve image src attributes in content nodes to hashed asset URLs
+          const resolvedContentNodes = resolveContentNodeImages(
+            contentNodes,
+            actions.contentDir,
+            actions.outdir,
+          );
+
+          const normalizedAttributes = normalizeAsciidocAttributes(attributes);
+          const fallbackTitle = path.basename(relativePath, '.adoc');
+          const pageTitle = title || fallbackTitle;
+          const metadata = {
+            type: 'asciidoc',
+            title: pageTitle,
+            ...normalizedAttributes.metadata,
+          };
+
+          const frontmatter = {
+            title: pageTitle,
+            description: normalizedAttributes.description,
+            keywords: normalizedAttributes.keywords,
+            excludeFromSearch: normalizedAttributes.excludeFromSearch,
+            seo: {
+              title: normalizedAttributes.doctitle || pageTitle,
+              description: normalizedAttributes.description || '',
+            },
+          };
+
+          actions.addRoute({
+            excludeFromSearch: normalizedAttributes.excludeFromSearch,
+            fsPath: relativePath,
+            templateId: asciidocTemplateId,
+            metadata,
+            getNavText: async () => pageTitle,
+            getStaticData: async (_route, dataContext) => {
+              // Resolve cross-page links using Realm's route registry
+              const linkedContentNodes = resolveContentNodeLinks(
+                resolvedContentNodes,
+                relativePath,
+                dataContext,
+              );
+
+              return {
+                props: {
+                  title: pageTitle,
+                  contentNodes: linkedContentNodes,
+                  sections,
+                  links,
+                  frontmatter,
+                },
+              };
+            },
+            getSearchDocuments: searchResolver(
+              frontmatter,
+              relativePath,
+              sections,
+              actions.getSearchFacets,
+              actions.setSearchFacets,
+            ),
+            getAiDocumentsStore: getAiDocumentsStore(
+              frontmatter,
+              sections,
+              actions.getSearchFacets,
+            ),
+          });
+        } catch (e) {
+          console.error(`Failed to process AsciiDoc file: ${relativePath}`, e);
+        }
+      }
+    },
+  };
+}
+
+type NormalizedAsciidocAttributes = {
+  doctitle?: string;
+  description?: string;
+  keywords?: string[];
+  excludeFromSearch: boolean;
+  metadata: Record<string, string>;
+};
+
+function normalizeAsciidocAttributes(
+  attributes: ParsedAsciidoc['attributes'],
+): NormalizedAsciidocAttributes {
+  const doctitle = toStringValue(attributes.doctitle);
+  const description = toStringValue(attributes.description);
+  const rawKeywords = toStringValue(attributes.keywords);
+  const keywords = rawKeywords
+    ? rawKeywords
+        .split(',')
+        .map((keyword) => keyword.trim())
+        .filter(Boolean)
+    : undefined;
+
+  const metadata = Object.entries(attributes).reduce<Record<string, string>>(
+    (acc, [key, value]) => {
+      if (!key.startsWith('metadata-')) return acc;
+      const metadataKey = key.slice('metadata-'.length);
+      if (!metadataKey) return acc;
+
+      acc[metadataKey] = String(value);
+      return acc;
+    },
+    {},
+  );
+
+  const excludeFromSearch = toBooleanValue(attributes['exclude-from-search'], false);
+
+  return {
+    doctitle,
+    description,
+    keywords,
+    excludeFromSearch,
+    metadata,
+  };
+}
+
+function toStringValue(value: string | number | boolean | undefined): string | undefined {
+  if (typeof value === 'string') return value;
+  if (typeof value === 'number' || typeof value === 'boolean') return String(value);
+  return undefined;
+}
+
+function toBooleanValue(
+  value: string | number | boolean | undefined,
+  defaultValue: boolean,
+): boolean {
+  if (typeof value === 'boolean') return value;
+  if (typeof value === 'number') return value !== 0;
+  if (typeof value === 'string') return ['1', 'true', 'yes', 'on'].includes(value.toLowerCase());
+  return defaultValue;
+}
+
+function __dirname(url: string): string {
+  const __filename = fileURLToPath(url);
+  return path.dirname(__filename);
+}
+
+function fromCurrentDir(moduleUrl: string, filePath: string): string {
+  return path.resolve(__dirname(moduleUrl), filePath);
+}
+
+// ─── Image asset resolution ─────────────────────────────────────────────────
+
+const ASSETS_URL_PREFIX = '/assets';
+const ASSETS_DIRNAME = 'assets';
+
+/**
+ * Recursively walks the content node tree and resolves image `src` values
+ * to hashed asset URLs, copying each file to the output assets directory.
+ */
+function resolveContentNodeImages(
+  nodes: ContentNode[],
+  contentDir: string,
+  outdir: string,
+): ContentNode[] {
+  return nodes.map((node) => resolveNodeImages(node, contentDir, outdir));
+}
+
+function resolveNodeImages(node: ContentNode, contentDir: string, outdir: string): ContentNode {
+  switch (node.type) {
+    case 'image': {
+      const src = node.src;
+      if (src.startsWith('/') && !src.startsWith('//') && !src.startsWith('/assets/')) {
+        try {
+          const assetUrl = copyImageToAssets(src.slice(1), contentDir, outdir);
+          return { ...node, src: assetUrl };
+        } catch {
+          return node;
+        }
+      }
+      return node;
+    }
+    case 'section':
+    case 'admonition':
+    case 'quote':
+    case 'sidebar':
+    case 'example':
+    case 'collapsible':
+      return { ...node, children: resolveContentNodeImages(node.children, contentDir, outdir) };
+    case 'list':
+      return {
+        ...node,
+        items: node.items.map((item) => ({
+          ...item,
+          children: resolveContentNodeImages(item.children, contentDir, outdir),
+        })),
+      };
+    case 'descriptionList':
+      return {
+        ...node,
+        items: node.items.map((item) => ({
+          ...item,
+          children: resolveContentNodeImages(item.children, contentDir, outdir),
+        })),
+      };
+    default:
+      return node;
+  }
+}
+
+function copyImageToAssets(fileRelativePath: string, contentDir: string, outdir: string): string {
+  const absolutePath = path.resolve(contentDir, fileRelativePath);
+  if (!existsSync(absolutePath)) {
+    throw new Error(`Image not found: ${absolutePath}`);
+  }
+
+  const assetsDir = path.join(outdir, ASSETS_DIRNAME);
+  if (!existsSync(assetsDir)) {
+    mkdirSync(assetsDir, { recursive: true });
+  }
+
+  const ext = path.extname(absolutePath);
+  const baseName = path.basename(absolutePath, ext);
+  const fileContent = readFileSync(absolutePath);
+  const hash = createHash('sha256').update(fileContent).digest('hex').slice(0, 16);
+  const hashedName = `${slugifyFileName(baseName)}.${hash}${ext}`;
+
+  const destPath = path.join(assetsDir, hashedName);
+  copyFileSync(absolutePath, destPath);
+
+  return `${ASSETS_URL_PREFIX}/${hashedName}`;
+}
+
+function slugifyFileName(name: string): string {
+  return name
+    .toLowerCase()
+    .replace(/[^a-z0-9.-]/g, '-')
+    .replace(/-+/g, '-')
+    .replace(/^-|-$/g, '');
+}
+
+// Link resolution logic adapted from packages/portal/src/server/plugins/markdown/attribute-resolvers/resolve-link.ts
+
+const DOC_EXTENSIONS = new Set(['.adoc', '.asciidoc', '.html', '.md']);
+
+function isLocalLink(href: string): boolean {
+  return (
+    href != null &&
+    !href.match(/^[a-z]+:\/\//) &&
+    !href.startsWith('//') &&
+    !href.startsWith('mailto:')
+  );
+}
+
+function stripKnownExtensions(filePath: string): string {
+  for (const ext of DOC_EXTENSIONS) {
+    if (filePath.endsWith(ext)) return filePath.slice(0, -ext.length);
+  }
+  return filePath;
+}
+
+function resolveAsciidocLink(
+  href: string,
+  fromPage: string,
+  dataContext: GetStaticDataContext,
+): string {
+  if (!href || href.startsWith('#') || !isLocalLink(href)) {
+    return href;
+  }
+
+  const hashIdx = href.indexOf('#');
+  const pathAndQuery = hashIdx >= 0 ? href.slice(0, hashIdx) : href;
+  const hashPart = hashIdx >= 0 ? href.slice(hashIdx) : '';
+
+  const qIdx = pathAndQuery.indexOf('?');
+  const rawPath = qIdx >= 0 ? pathAndQuery.slice(0, qIdx) : pathAndQuery;
+  const queryPart = qIdx >= 0 ? pathAndQuery.slice(qIdx) : '';
+
+  if (!rawPath) return href;
+
+  const ext = path.posix.extname(rawPath);
+
+  if (DOC_EXTENSIONS.has(ext)) {
+    const fsRelativePath = rawPath.startsWith('/')
+      ? rawPath.slice(1)
+      : path.posix.normalize(path.posix.join(path.posix.dirname(fromPage), rawPath));
+
+    const route = dataContext.getRouteByFsPath(fsRelativePath);
+    if (route) {
+      return maybeAddPathPrefix(route.slug + queryPart + hashPart);
+    }
+
+    const slug = '/' + stripKnownExtensions(fsRelativePath).replace(/\/index$/, '');
+    const routeBySlug = dataContext.getRouteBySlug(slug, { followRedirect: true });
+    if (routeBySlug) {
+      return maybeAddPathPrefix(routeBySlug.slug + queryPart + hashPart);
+    }
+
+    return maybeAddPathPrefix(slug + queryPart + hashPart);
+  }
+
+  if (ext === '') {
+    if (rawPath.startsWith('/')) {
+      const route = dataContext.getRouteBySlug(rawPath, { followRedirect: true });
+      if (route) {
+        return maybeAddPathPrefix(route.slug + queryPart + hashPart);
+      }
+      return maybeAddPathPrefix(rawPath + queryPart + hashPart);
+    }
+
+    const currentRoute = dataContext.getRouteByFsPath(fromPage);
+    if (currentRoute) {
+      const isIndexFile =
+        fromPage.endsWith('/index.adoc') ||
+        fromPage.endsWith('/index.md') ||
+        fromPage.endsWith('/index.html');
+      const base = isIndexFile ? currentRoute.slug : path.posix.dirname(currentRoute.slug);
+      const resolved = path.posix.normalize(path.posix.join(base, rawPath));
+      const normalizedSlug = '/' + resolved.replace(/^\/+/, '');
+
+      const route = dataContext.getRouteBySlug(normalizedSlug, { followRedirect: true });
+      if (route) {
+        return maybeAddPathPrefix(route.slug + queryPart + hashPart);
+      }
+      return maybeAddPathPrefix(normalizedSlug + queryPart + hashPart);
+    }
+
+    const baseDir = path.posix.dirname(fromPage);
+    const joined = path.posix.normalize(path.posix.join(baseDir, rawPath));
+    const slug = '/' + joined.replace(/^\/+/, '');
+    return maybeAddPathPrefix(slug + queryPart + hashPart);
+  }
+
+  return href;
+}
+
+function maybeAddPathPrefix(url: string): string {
+  const prefix = getPathPrefix();
+  if (!prefix || url.startsWith(prefix)) return url;
+  return withPathPrefix(url);
+}
+
+function resolveHtmlLinks(
+  html: string,
+  fromPage: string,
+  dataContext: GetStaticDataContext,
+): string {
+  return html.replace(
+    /(<a\s[^>]*?href=")([^"]*?)("[^>]*?>)/gi,
+    (_match, before: string, href: string, after: string) => {
+      return before + resolveAsciidocLink(href, fromPage, dataContext) + after;
+    },
+  );
+}
+
+function resolveContentNodeLinks(
+  nodes: ContentNode[],
+  fromPage: string,
+  dataContext: GetStaticDataContext,
+): ContentNode[] {
+  return nodes.map((node) => resolveNodeLinks(node, fromPage, dataContext));
+}
+
+function resolveNodeLinks(
+  node: ContentNode,
+  fromPage: string,
+  ctx: GetStaticDataContext,
+): ContentNode {
+  switch (node.type) {
+    case 'paragraph':
+      return {
+        ...node,
+        contentHtml: resolveHtmlLinks(node.contentHtml, fromPage, ctx),
+      };
+    case 'heading':
+      return {
+        ...node,
+        contentHtml: resolveHtmlLinks(node.contentHtml, fromPage, ctx),
+      };
+    case 'section':
+      return {
+        ...node,
+        titleHtml: resolveHtmlLinks(node.titleHtml, fromPage, ctx),
+        children: resolveContentNodeLinks(node.children, fromPage, ctx),
+      };
+    case 'admonition':
+    case 'quote':
+    case 'sidebar':
+    case 'example':
+    case 'collapsible':
+      return {
+        ...node,
+        children: resolveContentNodeLinks(node.children, fromPage, ctx),
+      };
+    case 'list':
+      return {
+        ...node,
+        items: node.items.map((item) => ({
+          ...item,
+          contentHtml: resolveHtmlLinks(item.contentHtml, fromPage, ctx),
+          children: resolveContentNodeLinks(item.children, fromPage, ctx),
+        })),
+      };
+    case 'descriptionList':
+      return {
+        ...node,
+        items: node.items.map((item) => ({
+          ...item,
+          termsHtml: item.termsHtml.map((h) => resolveHtmlLinks(h, fromPage, ctx)),
+          descriptionHtml: resolveHtmlLinks(item.descriptionHtml, fromPage, ctx),
+          children: resolveContentNodeLinks(item.children, fromPage, ctx),
+        })),
+      };
+    case 'table':
+      return {
+        ...node,
+        head: node.head.map((row) =>
+          row.map((cell) => ({
+            ...cell,
+            contentHtml: resolveHtmlLinks(cell.contentHtml, fromPage, ctx),
+          })),
+        ),
+        body: node.body.map((row) =>
+          row.map((cell) => ({
+            ...cell,
+            contentHtml: resolveHtmlLinks(cell.contentHtml, fromPage, ctx),
+          })),
+        ),
+        foot: node.foot.map((row) =>
+          row.map((cell) => ({
+            ...cell,
+            contentHtml: resolveHtmlLinks(cell.contentHtml, fromPage, ctx),
+          })),
+        ),
+      };
+    case 'htmlPassthrough':
+      return {
+        ...node,
+        html: resolveHtmlLinks(node.html, fromPage, ctx),
+      };
+    default:
+      return node;
+  }
+}