@redpanda-data/docs-extensions-and-macros 5.13.0 → 5.13.1

package/tools/redpanda-connect/helpers/renderConnectDescription.js

@@ -0,0 +1,125 @@
+'use strict';
+
+// One annotator for every scanner over connector prose, including the ones in
+// metadata-utils that run before these. See annotateVerbatimLines there for
+// why there is exactly one.
+const { descriptionWithMetadataInclude, annotateVerbatimLines } = require('../metadata-utils.js');
+
+// Length (characters) above which a heading-less description is reported as a
+// candidate for upstream structure. Roughly a screen of prose.
+const LONG_HEADINGLESS_THRESHOLD = 1200;
+
+/**
+ * True when the description contains at least one structural heading that sits
+ * outside a `----` listing block or a markdown ```/~~~ fence.
+ *
+ * Used to report long heading-less descriptions during generation. Embedded
+ * headings are otherwise passed through unchanged: they render as top-level
+ * page sections, which matches every published connector page today. Demoting
+ * them is not an option because the description renders before the page's
+ * first `==` section, so demoted headings produce "section title out of
+ * sequence" errors. Collapsible wrapping was evaluated and rejected: it hides
+ * primary content behind unreliable cross-browser find and deep-link behavior,
+ * and bodies containing their own `====` delimiters cannot be wrapped at all.
+ */
+function hasStructuralHeadings (body) {
+  // AsciiDoc headings only. Markdown-style ## headings deliberately do NOT
+  // count: the descriptions that render worst are exactly the ones whose
+  // only structure is markdown headings, and counting them here exempted
+  // those from the long-description report (they get their own report).
+  return annotateVerbatimLines(body)
+    .some(({ line, verbatim }) => !verbatim && /^={2,}\s+\S/.test(line));
+}
+
+/**
+ * True when the description contains a markdown-style heading (`##`) outside
+ * listing blocks and fences. These are reported for upstream conversion to
+ * `==` (a `#` line inside a fence is a comment in an example, not a heading).
+ */
+function hasMarkdownHeadings (body) {
+  return annotateVerbatimLines(body)
+    .some(({ line, verbatim }) => !verbatim && /^#{2,}\s+\S/.test(line));
+}
+
+/**
+ * Handlebars helper: render a connector's description for the regenerated
+ * description partial.
+ *
+ * The `== Metadata` block is replaced by an include of the metadata partial
+ * (de-duplicating it, since metadata is emitted as its own partial).
+ * Everything else passes through unchanged: the description is the page's
+ * primary content, and structure belongs upstream in the connector source.
+ * The generator reports long heading-less descriptions so they get headings
+ * added there instead of being hidden by the docs build.
+ *
+ * @param {object} item connector data (needs `description`, `type`/`typeDir`, `name`)
+ * @returns {string}
+ */
+module.exports = function renderConnectDescription (item) {
+  const body = descriptionWithMetadataInclude(item);
+  if (!body || !body.trim()) return '';
+  return escapePlaceholderBraces(ensureHeadingSeparation(body.trim()));
+};
+
+/**
+ * Insert the blank line Asciidoctor requires before a section title. A
+ * heading glued to the paragraph above it renders as literal text and
+ * orphans its subsections into "section title out of sequence" warnings
+ * (live case: the protobuf processor's "== Operators"). Listing blocks and
+ * fences are untouched — a glued `#` comment inside a fenced example must
+ * not have a blank line pushed into the example.
+ */
+function ensureHeadingSeparation (body) {
+  const out = [];
+  for (const { line, verbatim } of annotateVerbatimLines(body)) {
+    if (
+      !verbatim &&
+      /^(?:={2,6}|#{2,6})\s+\S/.test(line) &&
+      out.length > 0 &&
+      out[out.length - 1].trim() !== ''
+    ) {
+      out.push('');
+    }
+    out.push(line);
+  }
+  return out.join('\n');
+}
+
+/**
+ * Escape template placeholders like {endpoint} so Asciidoctor keeps them
+ * instead of consuming them as attribute references (which substitutes to
+ * nothing and logs "skipping reference to missing attribute" on the
+ * published page — live today on salesforce_graphql and otlp_http).
+ * Single-backtick spans still apply attribute substitution, so spans are
+ * escaped too; `----` listing blocks and ```/~~~ fences are verbatim and stay
+ * untouched (a `\{` inside a fence would render as a literal backslash).
+ */
+function escapePlaceholderBraces (body) {
+  return annotateVerbatimLines(body).map(({ line, verbatim }) => (
+    verbatim ? line : line.replace(/(?<![\\{])\{([a-z][\w.-]{1,30})\}/g, '\\{$1}')
+  )).join('\n');
+}
+
+/**
+ * Depth of the first structural heading (AsciiDoc `=` or markdown-compat
+ * `#`) outside listing blocks and fences, or null when the body has none.
+ * Descriptions whose first heading is deeper than level one (`===`, `###`)
+ * render "section title out of sequence" on the page, so the generator
+ * reports them as upstream fixes (seen on aws_dynamodb_cdc, iceberg,
+ * protobuf).
+ */
+function firstHeadingDepth (body) {
+  for (const { line, verbatim } of annotateVerbatimLines(body)) {
+    if (verbatim) continue;
+    const m = line.match(/^(={2,6}|#{2,6})\s+\S/);
+    if (m) return m[1].length;
+  }
+  return null;
+}
+
+module.exports.hasStructuralHeadings = hasStructuralHeadings;
+module.exports.hasMarkdownHeadings = hasMarkdownHeadings;
+module.exports.escapePlaceholderBraces = escapePlaceholderBraces;
+module.exports.ensureHeadingSeparation = ensureHeadingSeparation;
+module.exports.firstHeadingDepth = firstHeadingDepth;
+module.exports.LONG_HEADINGLESS_THRESHOLD = LONG_HEADINGLESS_THRESHOLD;

package/tools/redpanda-connect/helpers/summaryAttribute.js

@@ -0,0 +1,22 @@
+'use strict';
+
+const { flattenToAttributeValue } = require('../metadata-utils.js');
+
+/**
+ * Handlebars helper: flatten a connector summary into a single-line, plain-text
+ * value suitable for the `:description:` page attribute (and therefore for the
+ * page's `<meta name="description">`).
+ *
+ * Delegates to the shared flattener so the description partial's attrs region
+ * and `backfillPageDescriptions` publish the same text for the same summary.
+ * Collapsing whitespace alone is not enough: summaries carry xrefs, link
+ * macros and inline code, and an unflattened value ships raw
+ * `xref:guides:bloblang/about.adoc[Bloblang mapping]` markup into search
+ * snippets and link previews (verified in a real Antora build).
+ *
+ * @param {string} summary
+ * @returns {string}
+ */
+module.exports = function summaryAttribute (summary) {
+  return flattenToAttributeValue(summary);
+};

package/tools/redpanda-connect/metadata-utils.js

@@ -18,8 +18,18 @@ const path = require('path');
 
 const METADATA_HEADING = /^==\s+Metadata\s*$/;
 // AsciiDoc listing/literal block delimiter (`----`, possibly longer). Lines
-// inside such blocks must not be treated as headings.
+// inside such blocks must not be treated as headings. Asciidoctor only reads a
+// delimiter at column 0 -- an indented `----` is a literal paragraph -- so the
+// test is against the raw line, never a trimmed one.
 const BLOCK_DELIMITER = /^-{4,}$/;
+// AsciiDoc literal block delimiter (`....`, possibly longer) -- as
+// substitution-free as a ---- listing block, and just as liable to contain a
+// brace or heading-shaped line that must not be touched.
+const LITERAL_DELIMITER = /^\.{4,}$/;
+// Markdown-style fence delimiter (``` or ~~~, possibly with a language tag).
+// Descriptions and metadata blocks carry fenced examples alongside AsciiDoc
+// ---- blocks.
+const FENCE_DELIMITER = /^(`{3,}|~{3,})/;
 // The metadata block ends at the next structural element. Besides the next
 // level-2 heading, this also covers page-level constructs that follow the
 // section when locateMetadata runs against a full reference page (not just a
@@ -29,6 +39,54 @@ const BLOCK_DELIMITER = /^-{4,}$/;
 // swallow the trailing `include::...partial$fields[]` and `// end::single-source[]`.
 const SECTION_END = /^(?:==\s+\S|include::|\/\/\s*(?:tag|end)::)/;
 
+/**
+ * Split `text` into lines annotated with whether each line sits inside (or
+ * delimits) a verbatim region: an AsciiDoc `----` listing block or a markdown
+ * ```/~~~ fence. Every scanner over connector prose walks the text through
+ * this, so fence interiors are treated exactly like listing-block interiors:
+ * content, never headings, never escapable prose, never section terminators.
+ *
+ * Layered state: while one delimiter kind is open, only its own closer
+ * matters. A `----` line inside a fence (or a fence line inside a `----`
+ * block) is content -- treating it as a delimiter leaks the state and
+ * swallows everything after it.
+ *
+ * This is the only such state machine in the tree on purpose. There used to
+ * be three, and they had already drifted: two tested the trimmed line and one
+ * the raw line, so an indented `----` opened a verbatim region for one
+ * scanner and was plain content for another, on the same description in the
+ * same pipeline. Asciidoctor itself only honours a delimiter at column 0, so
+ * the raw-line test is the correct one and the only one here.
+ *
+ * @param {string} text
+ * @returns {Array<{line: string, verbatim: boolean}>}
+ */
+function annotateVerbatimLines (text) {
+  let inBlock = false;
+  let inLiteral = false;
+  let fence = null;
+  return String(text == null ? '' : text).split('\n').map((line) => {
+    if (inBlock) {
+      if (BLOCK_DELIMITER.test(line)) inBlock = false;
+      return { line, verbatim: true };
+    }
+    if (inLiteral) {
+      if (LITERAL_DELIMITER.test(line)) inLiteral = false;
+      return { line, verbatim: true };
+    }
+    if (fence) {
+      const closer = line.match(FENCE_DELIMITER);
+      if (closer && closer[1][0] === fence) fence = null;
+      return { line, verbatim: true };
+    }
+    const fenceMatch = line.match(FENCE_DELIMITER);
+    if (fenceMatch) { fence = fenceMatch[1][0]; return { line, verbatim: true }; }
+    if (BLOCK_DELIMITER.test(line)) { inBlock = true; return { line, verbatim: true }; }
+    if (LITERAL_DELIMITER.test(line)) { inLiteral = true; return { line, verbatim: true }; }
+    return { line, verbatim: false };
+  });
+}
+
 /**
  * Locate the `== Metadata` section within a description.
  * @param {string} description
@@ -39,22 +97,19 @@ const SECTION_END = /^(?:==\s+\S|include::|\/\/\s*(?:tag|end)::)/;
 function locateMetadata (description) {
   if (!description || typeof description !== 'string') return null;
 
-  const lines = description.split('\n');
+  const annotated = annotateVerbatimLines(description);
+  const lines = annotated.map((a) => a.line);
   let headingLine = -1;
-  let inBlock = false;
-  for (let i = 0; i < lines.length; i++) {
-    if (BLOCK_DELIMITER.test(lines[i])) { inBlock = !inBlock; continue; }
-    if (!inBlock && METADATA_HEADING.test(lines[i])) { headingLine = i; break; }
+  for (let i = 0; i < annotated.length; i++) {
+    if (!annotated[i].verbatim && METADATA_HEADING.test(lines[i])) { headingLine = i; break; }
   }
   if (headingLine === -1) return null;
 
   // Find the terminating element after the metadata heading: the next level-2
   // heading, an Antora include directive, or a single-source tag comment.
   let endLine = lines.length;
-  inBlock = false;
-  for (let i = headingLine + 1; i < lines.length; i++) {
-    if (BLOCK_DELIMITER.test(lines[i])) { inBlock = !inBlock; continue; }
-    if (!inBlock && SECTION_END.test(lines[i])) { endLine = i; break; }
+  for (let i = headingLine + 1; i < annotated.length; i++) {
+    if (!annotated[i].verbatim && SECTION_END.test(lines[i])) { endLine = i; break; }
   }
 
   // Trim trailing blank lines inside the section so the block ends cleanly.
@@ -78,16 +133,64 @@ function extractMetadata (description) {
   return found ? found.block : '';
 }
 
+// Rate limits are the one component family the upstream data and the docs
+// repo spell differently: the dataset key is `rate-limits`, `item.type` is
+// `rate_limit`, and the pages directory is `rate_limits`. Every spelling has
+// to collapse to one partial directory or the write path and the include path
+// disagree and the include does not resolve.
+const TYPE_DIR_ALIASES = new Map([
+  ['rate-limit', 'rate_limits'],
+  ['rate-limits', 'rate_limits'],
+  ['rate_limit', 'rate_limits'],
+]);
+
+/**
+ * The component families that have their own reference pages under
+ * `modules/components/pages/<typeDir>/`, in canonical typeDir spelling.
+ *
+ * This list existed in eight places and the copies disagreed three ways:
+ * `config` was in two of them, `rate_limits` was missing from one, and two
+ * carried both rate-limit spellings. A component family added upstream that
+ * is missed by one copy silently gets no partial, no page backfill or no
+ * cloud-docs check, depending on which copy forgot it.
+ * @type {string[]}
+ */
+const CONNECTOR_TYPE_DIRS = [
+  'inputs', 'outputs', 'processors', 'caches', 'rate_limits',
+  'buffers', 'metrics', 'scanners', 'tracers',
+];
+
+/**
+ * Every top-level key of the connector dataset that holds component objects:
+ * the page-backed families plus `config`, which has entries but no per-entry
+ * page. Callers that walk the dataset (augmenting, diffing, stripping) want
+ * this; callers that write per-page artifacts want CONNECTOR_TYPE_DIRS.
+ * @type {string[]}
+ */
+const CONNECTOR_DATA_KEYS = [...CONNECTOR_TYPE_DIRS, 'config'];
+
+/**
+ * Canonical partial directory for a plural type directory. Applied to every
+ * derivation of a type directory so `typeDirFor` is the single source of truth
+ * for both the file the generator writes and the include line a page carries.
+ * @param {string} typeDir
+ * @returns {string}
+ */
+function normalizeTypeDir (typeDir) {
+  return TYPE_DIR_ALIASES.get(typeDir) || typeDir;
+}
+
 /**
- * Derive the plural type directory (for example `input` -> `inputs`).
+ * Derive the plural, canonical type directory (for example `input` ->
+ * `inputs`, `rate-limits` -> `rate_limits`).
  * @param {object} item connector data with `type` and/or `typeDir`
  * @returns {string}
  */
 function typeDirFor (item) {
-  if (item && item.typeDir) return item.typeDir;
+  if (item && item.typeDir) return normalizeTypeDir(item.typeDir);
   const type = item && item.type;
   if (!type) return '';
-  return type.endsWith('s') ? type : `${type}s`;
+  return normalizeTypeDir(type.endsWith('s') ? type : `${type}s`);
 }
 
 /**
@@ -139,9 +242,22 @@ function descriptionWithMetadataInclude (item) {
   return description.slice(0, found.start) + metadataIncludeLine(item) + description.slice(found.end);
 }
 
-// Markdown-style fence delimiter (``` or ~~~, possibly with a language tag).
-// Metadata blocks can carry fenced examples alongside AsciiDoc ---- blocks.
-const FENCE_DELIMITER = /^(`{3,}|~{3,})/;
+/**
+ * Build the Antora include directive for a connector's description partial.
+ * Mirrors {@link metadataIncludeLine} so a drafted page can pull its
+ * auto-generated summary and description from a regenerated partial instead of
+ * freezing them into the page body at first draft.
+ *
+ * The partial defines two tag regions, so pages address it twice: `attrs` in
+ * the page header (sets `:description:`) and `body` where the prose renders.
+ * @param {object} item connector data with `type`/`typeDir` and `name`
+ * @param {string} [tag] tag region to include (`attrs` or `body`); omit for the whole file
+ * @returns {string}
+ */
+function descriptionIncludeLine (item, tag) {
+  const attrs = tag ? `tag=${tag}` : '';
+  return `include::connect:components:partial$descriptions/${typeDirFor(item)}/${item.name}.adoc[${attrs}]`;
+}
 
 /**
  * Collect the section heading titles in an AsciiDoc block, ignoring lines
@@ -155,25 +271,8 @@ const FENCE_DELIMITER = /^(`{3,}|~{3,})/;
 function sectionHeadings (text) {
   if (!text || typeof text !== 'string') return [];
   const headings = [];
-  let inBlock = false;
-  let fence = null;
-  for (const line of text.split('\n')) {
-    // Layered state: while inside one delimiter kind, the only thing that
-    // matters is its own closer. A fence-like line inside a ---- literal
-    // block (or a ---- line inside a fence) is content, not a delimiter —
-    // treating it as one leaks the state and swallows every later heading.
-    if (inBlock) {
-      if (BLOCK_DELIMITER.test(line)) inBlock = false;
-      continue;
-    }
-    if (fence) {
-      const closer = line.match(FENCE_DELIMITER);
-      if (closer && closer[1][0] === fence) fence = null;
-      continue;
-    }
-    const fenceMatch = line.match(FENCE_DELIMITER);
-    if (fenceMatch) { fence = fenceMatch[1][0]; continue; }
-    if (BLOCK_DELIMITER.test(line)) { inBlock = true; continue; }
+  for (const { line, verbatim } of annotateVerbatimLines(text)) {
+    if (verbatim) continue;
     const m = line.match(/^=+\s+(\S.*)$/);
     if (m) headings.push(m[1].trim());
   }
@@ -196,13 +295,62 @@ function lostMetadataSections (oldContent, newContent) {
   return sectionHeadings(oldContent).filter((h) => !newHeadings.has(h));
 }
 
+/**
+ * Flatten AsciiDoc prose into a value usable as a `:description:` attribute
+ * and, downstream, as the page's `<meta name="description">`.
+ *
+ * Meta descriptions are read as plain text by search results, link previews
+ * and assistants, so markup has to come out mechanically: xref and link
+ * macros become their labels, bare URL macros become their labels, `<<>>`
+ * shorthand becomes its text, the "open in new tab" caret goes, inline code
+ * loses its backticks, and whitespace (including hard line breaks, which an
+ * attribute value cannot carry) collapses to single spaces. No prose is
+ * edited.
+ *
+ * This is the one implementation: `summaryAttribute` (the description
+ * partial's attrs region) and `backfillPageDescriptions` (the page-header
+ * self-heal) both call it, so the two paths cannot drift into publishing
+ * different meta descriptions for the same summary.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+function flattenToAttributeValue (text) {
+  if (!text) return '';
+  return String(text)
+    .replace(/(?:xref|link):[^\[\]]+\[([^\]]*)\]/g, '$1')
+    // Bare URL macros (no xref:/link: prefix) and internal xref shorthand
+    // also appear in source summaries and render literally if left in place.
+    .replace(/https?:\/\/[^\s\[\]]+\[([^\]]*)\]/g, '$1')
+    // image:: and inline image: macros have no textual value in a meta
+    // description; drop them rather than leaking the target path.
+    .replace(/image:{1,2}[^\[\]\s]*\[[^\]]*\]/g, '')
+    .replace(/<<[^,>]+,([^>]+)>>/g, '$1')
+    .replace(/<<([^>]+)>>/g, '$1')
+    // Strip the "open in new tab" caret that AsciiDoc URL macros carry
+    // inside the link text, e.g. [Open Telemetry collector^].
+    .replace(/\^/g, '')
+    .replace(/`([^`]*)`/g, '$1')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
 module.exports = {
   locateMetadata,
+  flattenToAttributeValue,
+  CONNECTOR_TYPE_DIRS,
+  CONNECTOR_DATA_KEYS,
+  normalizeTypeDir,
   resolvePageTypeDir,
   extractMetadata,
   typeDirFor,
   metadataIncludeLine,
   descriptionWithMetadataInclude,
+  descriptionIncludeLine,
   sectionHeadings,
   lostMetadataSections,
+  annotateVerbatimLines,
+  BLOCK_DELIMITER,
+  LITERAL_DELIMITER,
+  FENCE_DELIMITER,
 };