@redpanda-data/docs-extensions-and-macros 5.49.0 → 5.51.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -37,9 +37,14 @@ function escapeRegExp(str) {
37
37
  * Regex pattern to match and strip the llms directive from markdown content.
38
38
  * Dynamically derived from LLMS_DIRECTIVE_BASE to stay in sync.
39
39
  * Matches the blockquote format with optional component-specific suffix.
40
+ * The directive is written with a root-relative link, but links are made
41
+ * absolute before the exports strip it, so any prefix before /llms.txt matches.
40
42
  */
41
43
  const LLMS_DIRECTIVE_REGEX = new RegExp(
42
- `^> ${escapeRegExp(LLMS_DIRECTIVE_BASE)}.*$`,
44
+ `^> ${escapeRegExp(LLMS_DIRECTIVE_BASE).replace(
45
+ escapeRegExp('(/llms.txt)'),
46
+ '\\([^)\\s]*/llms\\.txt\\)'
47
+ )}.*$`,
43
48
  'gm'
44
49
  );
45
50
 
@@ -96,11 +101,343 @@ function componentsWithExports(components, pages) {
96
101
  );
97
102
  }
98
103
 
104
+ /**
105
+ * Maximum characters per page index file. The Agent-Friendly Documentation
106
+ * Spec passes an llms.txt index at 50,000 characters or less and recommends
107
+ * splitting a large index into section-level files that each stay under that
108
+ * limit. Leave a buffer below 50K.
109
+ */
110
+ const MAX_INDEX_CHARS = 45000;
111
+
112
+ /**
113
+ * Descriptions longer than this are cut at a word boundary so that one verbose
114
+ * page doesn't force an extra split.
115
+ */
116
+ const MAX_INDEX_DESCRIPTION_CHARS = 200;
117
+
118
+ function oneLine(text) {
119
+ return String(text || '')
120
+ .replace(/<[^>]+>/g, '')
121
+ .replace(/\s+/g, ' ')
122
+ .trim();
123
+ }
124
+
125
+ function truncateDescription(text) {
126
+ const clean = oneLine(text);
127
+ if (clean.length <= MAX_INDEX_DESCRIPTION_CHARS) return clean;
128
+ const cut = clean.slice(0, MAX_INDEX_DESCRIPTION_CHARS);
129
+ const lastSpace = cut.lastIndexOf(' ');
130
+ return `${(lastSpace > 0 ? cut.slice(0, lastSpace) : cut).replace(/[\s.,;:]+$/, '')}...`;
131
+ }
132
+
133
+ // Square brackets in link text end the link early in markdown.
134
+ function escapeLinkText(text) {
135
+ return oneLine(text).replace(/([[\]])/g, '\\$1');
136
+ }
137
+
138
+ /**
139
+ * A URL made safe as a Markdown link destination. An unmatched `)` ends the
140
+ * destination early, and a space or `<` ends it outright, so those are
141
+ * percent-encoded. Encoding keeps the plain `[text](url)` shape llms.txt
142
+ * readers expect, where an angle-bracket destination would not.
143
+ */
144
+ function linkDestination(url) {
145
+ return String(url).replace(/[()<> ]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
146
+ }
147
+
148
+ /**
149
+ * Longest shared directory of a set of URL paths, always ending in `/`.
150
+ * `/streaming/26.1/manage/a/` and `/streaming/26.1/deploy/b/` share `/streaming/26.1/`.
151
+ */
152
+ function commonDirectory(urls) {
153
+ const dirs = urls.map((url) => url.replace(/[^/]*\/?$/, '').split('/').filter(Boolean));
154
+ if (!dirs.length) return '/';
155
+ const shared = [];
156
+ for (let i = 0; i < dirs[0].length; i++) {
157
+ const segment = dirs[0][i];
158
+ if (!dirs.every((d) => d[i] === segment)) break;
159
+ shared.push(segment);
160
+ }
161
+ return shared.length ? `/${shared.join('/')}/` : '/';
162
+ }
163
+
164
+ /**
165
+ * URL of the component version's root, derived from how Antora built the page
166
+ * URL: the version root, then the module name (omitted for ROOT), then the
167
+ * page's relative path (index pages drop their own segment). Works for both
168
+ * indexified (`/a/b/`) and `.html` URLs, since the page is one segment either
169
+ * way, except an index page, whose `.html` URL has an `index.html` segment
170
+ * that its indexified URL does not.
171
+ * Returns null when the page lacks the source fields needed.
172
+ */
173
+ function versionRootOf(page) {
174
+ const { module: moduleName, relative } = page.src || {};
175
+ if (!relative || !moduleName) return null;
176
+ const relSegments = relative.replace(/\.adoc$/, '').split('/').filter(Boolean);
177
+ const urlSegments = page.pub.url.split('/').filter(Boolean);
178
+ if (relSegments[relSegments.length - 1] === 'index') {
179
+ relSegments.pop();
180
+ // An index page keeps an `index.html` segment of its own when URLs are
181
+ // not indexified (`/streaming/26.1/index.html`), and has none when they
182
+ // are (`/streaming/26.1/`).
183
+ if (urlSegments[urlSegments.length - 1] === 'index.html') urlSegments.pop();
184
+ }
185
+ const tailCount = relSegments.length + (moduleName === 'ROOT' ? 0 : 1);
186
+ if (tailCount > urlSegments.length) return null;
187
+ const rootSegments = urlSegments.slice(0, urlSegments.length - tailCount);
188
+ return rootSegments.length ? `/${rootSegments.join('/')}/` : '/';
189
+ }
190
+
191
+ function mostCommon(values) {
192
+ const counts = new Map();
193
+ values.forEach((v) => counts.set(v, (counts.get(v) || 0) + 1));
194
+ let best = null;
195
+ counts.forEach((count, value) => {
196
+ if (best === null || count > counts.get(best)) best = value;
197
+ });
198
+ return best;
199
+ }
200
+
201
+ /**
202
+ * The directory segment directly under `dir` that contains `url`, or null when
203
+ * the page sits in `dir` itself. With indexified URLs a page `/a/b/` lives in
204
+ * `/a/`, so the last segment is the page, not a directory.
205
+ */
206
+ function childSegment(url, dir) {
207
+ const rest = url.slice(dir.length).replace(/\/$/, '').replace(/\/index\.html$/, '');
208
+ const segments = rest.split('/').filter(Boolean);
209
+ return segments.length > 1 ? segments[0] : null;
210
+ }
211
+
212
+ function latestDate(dates) {
213
+ const times = dates.filter(Boolean).map((d) => new Date(d)).filter((d) => !isNaN(d));
214
+ return times.length ? new Date(Math.max(...times)).toISOString() : null;
215
+ }
216
+
217
+ function renderIndexFile({ heading, intro, entries }) {
218
+ let out = `# ${heading}\n\n`;
219
+ intro.forEach((line) => {
220
+ out += `> ${line}\n`;
221
+ });
222
+ out += `\n## Pages\n\n`;
223
+ entries.forEach((entry) => {
224
+ out += `${entry.line}\n`;
225
+ });
226
+ return out;
227
+ }
228
+
229
+ /**
230
+ * Pack entries into chunks whose rendered size stays under the limit.
231
+ * Used only when a single directory holds more pages than one file allows.
232
+ */
233
+ function chunkEntries(entries, overhead, maxChars) {
234
+ const chunks = [];
235
+ let current = [];
236
+ let size = overhead;
237
+ entries.forEach((entry) => {
238
+ const lineSize = entry.line.length + 1;
239
+ if (current.length && size + lineSize > maxChars) {
240
+ chunks.push(current);
241
+ current = [];
242
+ size = overhead;
243
+ }
244
+ current.push(entry);
245
+ size += lineSize;
246
+ });
247
+ if (current.length) chunks.push(current);
248
+ return chunks;
249
+ }
250
+
251
+ /**
252
+ * Build page index files for every published component version.
253
+ *
254
+ * Each component version gets an index at `<version root>/llms.txt` listing
255
+ * every page as a markdown link to its .md URL. When a version's index would
256
+ * exceed `maxChars`, it is split by URL directory, recursively, so each file
257
+ * stays under the limit. The root llms.txt links to every file this returns,
258
+ * which keeps every page one hop from the root index.
259
+ *
260
+ * @param {Object} options
261
+ * @param {Array} options.pages - Antora pages that have markdown (page.markdownContents)
262
+ * @param {Array} options.components - Antora components
263
+ * @param {string} options.siteUrl - Absolute site URL without a trailing slash
264
+ * @param {Function} options.toMarkdownUrl - Maps a page's HTML URL to its .md URL
265
+ * @param {number} [options.maxChars] - Size limit per file
266
+ * @returns {Array<{path: string, url: string, title: string, component: string,
267
+ * version: string, isLatest: boolean, pageCount: number, contents: string}>}
268
+ */
269
+ function buildPageIndexes({ pages, components, siteUrl, toMarkdownUrl, maxChars = MAX_INDEX_CHARS }) {
270
+ const componentsByName = new Map((components || []).map((c) => [c.name, c]));
271
+ const groups = new Map();
272
+
273
+ (pages || []).forEach((page) => {
274
+ const url = page.pub && page.pub.url;
275
+ if (!url || !page.out || !page.src) return;
276
+ const key = `${page.src.version}@${page.src.component}`;
277
+ if (!groups.has(key)) groups.set(key, []);
278
+ groups.get(key).push(page);
279
+ });
280
+
281
+ const indexes = [];
282
+
283
+ groups.forEach((groupPages) => {
284
+ const { component: componentName, version } = groupPages[0].src;
285
+ const component = componentsByName.get(componentName);
286
+ if (!component) return;
287
+ const componentVersion = (component.versions || []).find((v) => v.version === version) || {};
288
+ const latest = component.latest || (component.versions || [])[0] || {};
289
+ const isLatest = latest.version === version;
290
+ const versioned = (component.versions || []).length > 1;
291
+ const displayVersion = componentVersion.displayVersion || version;
292
+ const baseTitle = versioned
293
+ ? `${component.title} ${displayVersion}`
294
+ : component.title;
295
+
296
+ const entries = groupPages
297
+ .map((page) => {
298
+ const title = page.asciidoc && (page.asciidoc.doctitle || page.asciidoc.navtitle);
299
+ const description = page.asciidoc && page.asciidoc.attributes && page.asciidoc.attributes.description;
300
+ const mdUrl = `${siteUrl}${toMarkdownUrl(page.pub.url)}`;
301
+ let line = `- [${escapeLinkText(title || page.src.stem)}](${linkDestination(mdUrl)})`;
302
+ if (description) line += `: ${truncateDescription(description)}`;
303
+ const modified = page.asciidoc && page.asciidoc.attributes && page.asciidoc.attributes['page-git-modified-date'];
304
+ return { url: page.pub.url, line, modified };
305
+ })
306
+ .sort((a, b) => a.url.localeCompare(b.url));
307
+
308
+ const roots = groupPages.map(versionRootOf).filter(Boolean);
309
+ const versionRoot = roots.length ? mostCommon(roots) : commonDirectory(entries.map((e) => e.url));
310
+
311
+ const intro = (scope) => {
312
+ const lines = [
313
+ `Index of ${scope} documentation pages. Each link points to the page's markdown version.`,
314
+ ];
315
+ if (versioned && !isLatest) {
316
+ lines.push(`This is version ${displayVersion}. The latest version is ${latest.displayVersion || latest.version}.`);
317
+ }
318
+ lines.push(`For the full documentation index, see ${siteUrl}/llms.txt`);
319
+ return lines;
320
+ };
321
+
322
+ const makeIndex = (dir, dirEntries, label, part) => {
323
+ const heading = label ? `${baseTitle}: ${label}` : baseTitle;
324
+ const title = part > 1 ? `${heading} (part ${part})` : heading;
325
+ const path = `${dir.replace(/^\//, '')}${part > 1 ? `llms-${part}.txt` : 'llms.txt'}`;
326
+ return {
327
+ path,
328
+ url: `${siteUrl}/${path}`,
329
+ title,
330
+ component: componentName,
331
+ version,
332
+ isLatest,
333
+ pageCount: dirEntries.length,
334
+ lastModified: latestDate(dirEntries.map((e) => e.modified)),
335
+ contents: renderIndexFile({
336
+ heading: title,
337
+ intro: intro(label ? `${baseTitle} ${label}` : baseTitle),
338
+ entries: dirEntries,
339
+ }),
340
+ };
341
+ };
342
+
343
+ const split = (dir, dirEntries) => {
344
+ const label = dir.slice(versionRoot.length).replace(/\/$/, '');
345
+ const whole = makeIndex(dir, dirEntries, label, 1);
346
+ if (whole.contents.length <= maxChars) {
347
+ indexes.push(whole);
348
+ return;
349
+ }
350
+
351
+ const children = new Map();
352
+ dirEntries.forEach((entry) => {
353
+ const child = childSegment(entry.url, dir);
354
+ if (child === null) return;
355
+ if (!children.has(child)) children.set(child, []);
356
+ children.get(child).push(entry);
357
+ });
358
+
359
+ // Peel off the largest subdirectories into their own indexes until the
360
+ // rest fits. Small subdirectories stay in this directory's index, so the
361
+ // split produces as few files as the limit allows.
362
+ const fits = (list) => makeIndex(dir, list, label, 1).contents.length <= maxChars;
363
+ const bySize = [...children.entries()].sort((a, b) => b[1].length - a[1].length);
364
+ const kept = new Set(bySize.map(([child]) => child));
365
+ const remaining = () => dirEntries.filter((entry) => {
366
+ const child = childSegment(entry.url, dir);
367
+ return child === null || kept.has(child);
368
+ });
369
+ for (const [child, childEntries] of bySize) {
370
+ if (fits(remaining())) break;
371
+ kept.delete(child);
372
+ split(`${dir}${child}/`, childEntries);
373
+ }
374
+
375
+ const rest = remaining();
376
+ if (!rest.length) return;
377
+ // A directory whose own pages still exceed the limit is split into numbered parts.
378
+ // Header size of the largest possible part, so no chunk overflows once rendered.
379
+ const overhead = makeIndex(dir, [], label, 99).contents.length;
380
+ chunkEntries(rest, overhead, maxChars).forEach((chunk, i) => indexes.push(makeIndex(dir, chunk, label, i + 1)));
381
+ };
382
+
383
+ split(versionRoot, entries);
384
+ });
385
+
386
+ // Latest versions first, then older versions newest first, each in URL order.
387
+ return indexes.sort((a, b) => {
388
+ if (a.component !== b.component) return a.component.localeCompare(b.component);
389
+ if (a.isLatest !== b.isLatest) return a.isLatest ? -1 : 1;
390
+ if (a.version !== b.version) return b.version.localeCompare(a.version, undefined, { numeric: true });
391
+ return a.path.localeCompare(b.path);
392
+ });
393
+ }
394
+
395
+ /**
396
+ * Replace markdown links to the root llms.txt with their plain link text.
397
+ * A root index that links to itself is read as a nested index by tools that
398
+ * walk llms.txt, which pushes every page index one level deeper than intended.
399
+ */
400
+ function unlinkSelfReferences(content, siteUrl) {
401
+ const self = new RegExp(`\\[([^\\]]*)\\]\\((?:${escapeRegExp(siteUrl)})?/llms\\.txt\\)`, 'g');
402
+ return content.replace(self, '$1');
403
+ }
404
+
405
+ /**
406
+ * Render the root llms.txt section that links to every page index file.
407
+ * Every index must be linked from the root so agents, and tools that follow
408
+ * one level of nested indexes, can reach every page.
409
+ */
410
+ function renderPageIndexSection(indexes, components) {
411
+ const titles = new Map((components || []).map((c) => [c.name, c.title]));
412
+ let out = `## Page indexes\n\n`;
413
+ out += `Each file below lists pages as links to their markdown versions. Start with the latest version of a product unless you need a specific older version.\n\n`;
414
+ let currentComponent = null;
415
+ let olderHeadingShown = false;
416
+ indexes.forEach((index) => {
417
+ if (index.component !== currentComponent) {
418
+ currentComponent = index.component;
419
+ olderHeadingShown = false;
420
+ out += `${currentComponent === indexes[0].component ? '' : '\n'}### ${titles.get(index.component) || index.component}\n\n`;
421
+ }
422
+ if (!index.isLatest && !olderHeadingShown) {
423
+ olderHeadingShown = true;
424
+ out += `\nOlder versions:\n\n`;
425
+ }
426
+ out += `- [${escapeLinkText(index.title)}](${linkDestination(index.url)}): ${index.pageCount} ${index.pageCount === 1 ? 'page' : 'pages'}\n`;
427
+ });
428
+ return out;
429
+ }
430
+
99
431
  module.exports = {
100
432
  LLMS_DIRECTIVE_BASE,
101
433
  LLMS_DIRECTIVE_REGEX,
102
434
  SOURCE_COMMENT_REGEX,
435
+ MAX_INDEX_CHARS,
103
436
  formatLlmsDirective,
437
+ linkDestination,
104
438
  stripMarkdownMetadata,
105
439
  componentsWithExports,
440
+ buildPageIndexes,
441
+ renderPageIndexSection,
442
+ unlinkSelfReferences,
106
443
  };
@@ -1267,6 +1267,7 @@ The `convert-llms-to-txt` extension generates AI-optimized documentation exports
1267
1267
  - Generates `llms.txt` from a curated `llms.adoc` page in the home component
1268
1268
  - Creates `llms-full.txt` with all documentation pages
1269
1269
  - Generates component-specific exports (e.g., `ROOT-full.txt`, `cloud-full.txt`)
1270
+ - Generates a page index (`llms.txt`) for every published component version, linked from the root `llms.txt`
1270
1271
  - Unpublishes the HTML version of llms.adoc
1271
1272
  - Replaces site-url attribute dynamically for preview builds
1272
1273
 
@@ -1290,7 +1291,8 @@ Create `modules/home/pages/llms.adoc` in your docs repository with curated docum
1290
1291
  2. Place `llms.txt` at site root
1291
1292
  3. Generate `llms-full.txt` with all pages
1292
1293
  4. Generate component-specific full.txt files
1293
- 5. Unpublish the HTML version
1294
+ 5. Generate a page index for every published component version and link each one from `llms.txt`
1295
+ 6. Unpublish the HTML version
1294
1296
 
1295
1297
  === Generated files
1296
1298
 
@@ -1299,6 +1301,16 @@ Create `modules/home/pages/llms.adoc` in your docs repository with curated docum
1299
1301
  - `{site-url}/ROOT-full.txt` - Self-Managed docs only
1300
1302
  - `{site-url}/cloud-full.txt` - Cloud docs only
1301
1303
  - `{site-url}/redpanda-connect-full.txt` - Connect docs only
1304
+ - `{site-url}/<component>/<version>/llms.txt` - Page index for one component version
1305
+ - `{site-url}/<component>/llms.txt` - Page index for a component without versions, such as `connect/llms.txt`
1306
+
1307
+ === Page indexes
1308
+
1309
+ Each page index lists every published page in one component version as a markdown link to the page's `.md` URL, followed by the page description when it has one. Older versions get their own indexes, which say which version they cover and name the latest version.
1310
+
1311
+ An index that would exceed 45,000 characters is split by URL directory, for example `streaming/current/manage/llms.txt`. A directory that is still too large on its own is split into numbered parts (`llms.txt`, `llms-2.txt`, and so on). The Agent-Friendly Documentation Spec passes an `llms.txt` at 50,000 characters or less and recommends this kind of split for large sites.
1312
+
1313
+ The root `llms.txt` links to every page index in a *Page indexes* section. The index list is never truncated, because an index missing from the root leaves its pages unreachable from `llms.txt`. Page indexes do not link to other page indexes, so every page is one hop from the root.
1302
1314
 
1303
1315
  === Attributes
1304
1316
 
@@ -3,7 +3,13 @@
3
3
  const { raiseListenerLimit } = require('./util/raise-listener-limit')
4
4
 
5
5
  const { toMarkdownUrl } = require('../extension-utils/url-utils');
6
- const { stripMarkdownMetadata, componentsWithExports } = require('../extension-utils/llms-utils');
6
+ const {
7
+ stripMarkdownMetadata,
8
+ componentsWithExports,
9
+ buildPageIndexes,
10
+ renderPageIndexSection,
11
+ unlinkSelfReferences,
12
+ } = require('../extension-utils/llms-utils');
7
13
 
8
14
  /**
9
15
  * Extracts markdown from llms.adoc page and generates AI-friendly documentation exports.
@@ -18,6 +24,9 @@ const { stripMarkdownMetadata, componentsWithExports } = require('../extension-u
18
24
  * 5. Places llms.txt (markdown) at site root
19
25
  * 6. Generates llms-full.txt with markdown from latest versions of all components
20
26
  * 7. Generates component-specific full.txt files (e.g., redpanda-full.txt, cloud-full.txt)
27
+ * 8. Generates page index files (<version root>/llms.txt) for every published component
28
+ * version, split by URL directory to stay under the size limit, and links each
29
+ * one from the root llms.txt so every page is one hop from the root index
21
30
  *
22
31
  * Must run after convert-to-markdown extension to access page.markdownContents.
23
32
  */
@@ -110,6 +119,8 @@ module.exports.register = function () {
110
119
  logger.info('Unpublished llms HTML page');
111
120
  }
112
121
 
122
+ content = unlinkSelfReferences(content, siteUrl);
123
+
113
124
  // Store cleaned markdown content for adding after llms-full.txt
114
125
  llmsPage.llmsTxtContent = content;
115
126
 
@@ -142,145 +153,168 @@ module.exports.register = function () {
142
153
  return latestVersions.has(pageKey);
143
154
  });
144
155
 
145
- if (!pages.length) {
146
- logger.warn('No pages with markdown content found in latest versions, skipping llms-full.txt generation');
156
+ if (!allPages.length) {
157
+ logger.warn('No pages with markdown content found, skipping llms.txt generation');
147
158
  return;
148
159
  }
149
160
 
150
- logger.info(`Filtered to ${pages.length} pages from ${latestVersions.size} latest component versions (from ${allPages.length} total pages)`);
151
-
152
- let fullContent = `# Redpanda Documentation - Full Markdown Export\n\n`;
153
- fullContent += `> This file contains all documentation pages in markdown format for AI agent consumption.\n`;
154
- fullContent += `> Generated from ${pages.length} pages on ${new Date().toISOString()}\n`;
155
- fullContent += `> Site: ${siteUrl}\n\n`;
156
- fullContent += `## About This Export\n\n`;
157
- fullContent += `This export includes only the **latest version** of each component's documentation:\n`;
158
- components.forEach(component => {
159
- const latest = component.latest || component.versions[0];
160
- if (latest) {
161
- fullContent += `- **${component.title}**: version ${latest.version}\n`;
162
- }
163
- });
164
- fullContent += `\n`;
165
- fullContent += `### AI-Friendly Documentation Formats\n\n`;
166
- fullContent += `We provide multiple formats optimized for AI consumption:\n\n`;
167
- fullContent += `- **${siteUrl}/llms.txt**: Curated overview following the llms.txt standard - start here for a quick introduction\n`;
168
- fullContent += `- **${siteUrl}/llms-full.txt**: Complete documentation export (this file) - comprehensive reference with all pages\n`;
169
- fullContent += `- **Component-specific exports**: Focused documentation for individual products:\n`;
170
- // Only advertise exports for components that actually have pages — a
171
- // `<name>-full.txt` is generated per component below, so listing corpus-less
172
- // landing/utility components (e.g. data-platform, self-managed, search) would
173
- // point the AI index at a 404.
174
- componentsWithExports(components, pages).forEach(component => {
175
- fullContent += ` - \`${siteUrl}/${component.name}-full.txt\`: ${component.title}\n`;
176
- });
177
- fullContent += `- **Individual markdown pages**: Each HTML page has a corresponding .md file (e.g., \`/docs/page.html\` → \`/docs/page.md\`)\n\n`;
178
- fullContent += `### Accessing Versioned Content\n\n`;
179
- fullContent += `For components with versioned documentation (like Redpanda Self-Managed), older versions can be accessed by replacing the version segment in the URL:\n`;
180
- fullContent += `- Latest: \`${siteUrl}/current/page-path\`\n`;
181
- fullContent += `- Specific version: \`${siteUrl}/24.3/page-path\`, \`${siteUrl}/25.1/page-path\`, etc.\n\n`;
182
- fullContent += `Available versioned components: ${components.filter(c => c.versions.length > 1).map(c => c.name).join(', ')}\n\n`;
183
- fullContent += `---\n\n`;
184
-
185
- // Sort pages by URL for consistent ordering
186
- pages.sort((a, b) => {
187
- const urlA = a.pub?.url || '';
188
- const urlB = b.pub?.url || '';
189
- return urlA.localeCompare(urlB);
190
- });
191
-
192
- pages.forEach((page, index) => {
193
- const mdUrl = page.pub?.url ? toMarkdownUrl(page.pub.url) : '';
194
- const pageUrl = mdUrl ? `${siteUrl}${mdUrl}` : 'unknown';
195
- const pageTitle = page.asciidoc?.doctitle || page.src?.stem || 'Untitled';
196
-
197
- fullContent += `# Page ${index + 1}: ${pageTitle}\n\n`;
198
- fullContent += `**URL**: ${pageUrl}\n\n`;
199
- fullContent += `---\n\n`;
200
- // Strip metadata (directive, source comments) from page content
201
- fullContent += stripMarkdownMetadata(page.markdownContents);
202
- fullContent += `\n\n---\n\n`;
203
- });
204
-
205
- // Add llms-full.txt to site root
206
- siteCatalog.addFile({
207
- contents: Buffer.from(fullContent, 'utf8'),
208
- out: { path: 'llms-full.txt' },
209
- });
210
- logger.info(`Generated llms-full.txt with ${pages.length} pages`);
211
-
212
- // Generate component-specific full.txt files
213
- logger.info('Generating component-specific full.txt files...');
161
+ // The full exports cover the latest versions only, but page indexes cover
162
+ // every published version, so an empty latest set skips only the exports.
214
163
  const componentGroups = new Map();
215
-
216
- // Group pages by component
217
- pages.forEach(page => {
218
- const componentName = page.src.component;
219
- if (!componentGroups.has(componentName)) {
220
- componentGroups.set(componentName, []);
221
- }
222
- componentGroups.get(componentName).push(page);
223
- });
224
-
225
- // Generate a full.txt file for each component
226
- componentGroups.forEach((componentPages, componentName) => {
227
- const component = components.find(c => c.name === componentName);
228
- if (!component) return;
229
-
230
- const latest = component.latest || component.versions[0];
231
- if (!latest) return;
164
+ if (!pages.length) {
165
+ logger.warn('No pages with markdown content found in latest versions, skipping llms-full.txt generation');
166
+ } else {
167
+ logger.info(`Filtered to ${pages.length} pages from ${latestVersions.size} latest component versions (from ${allPages.length} total pages)`);
168
+
169
+ let fullContent = `# Redpanda Documentation - Full Markdown Export\n\n`;
170
+ fullContent += `> This file contains all documentation pages in markdown format for AI agent consumption.\n`;
171
+ fullContent += `> Generated from ${pages.length} pages on ${new Date().toISOString()}\n`;
172
+ fullContent += `> Site: ${siteUrl}\n\n`;
173
+ fullContent += `## About This Export\n\n`;
174
+ fullContent += `This export includes only the **latest version** of each component's documentation:\n`;
175
+ components.forEach(component => {
176
+ const latest = component.latest || component.versions[0];
177
+ if (latest) {
178
+ fullContent += `- **${component.title}**: version ${latest.version}\n`;
179
+ }
180
+ });
181
+ fullContent += `\n`;
182
+ fullContent += `### AI-Friendly Documentation Formats\n\n`;
183
+ fullContent += `We provide multiple formats optimized for AI consumption:\n\n`;
184
+ fullContent += `- **${siteUrl}/llms.txt**: Curated overview following the llms.txt standard - start here for a quick introduction\n`;
185
+ fullContent += `- **${siteUrl}/llms-full.txt**: Complete documentation export (this file) - comprehensive reference with all pages\n`;
186
+ fullContent += `- **Component-specific exports**: Focused documentation for individual products:\n`;
187
+ // Only advertise exports for components that actually have pages — a
188
+ // `<name>-full.txt` is generated per component below, so listing corpus-less
189
+ // landing/utility components (e.g. data-platform, self-managed, search) would
190
+ // point the AI index at a 404.
191
+ componentsWithExports(components, pages).forEach(component => {
192
+ fullContent += ` - \`${siteUrl}/${component.name}-full.txt\`: ${component.title}\n`;
193
+ });
194
+ fullContent += `- **Individual markdown pages**: Each HTML page has a corresponding .md file (e.g., \`/docs/page.html\` → \`/docs/page.md\`)\n\n`;
195
+ fullContent += `### Accessing Versioned Content\n\n`;
196
+ fullContent += `For components with versioned documentation (like Redpanda Self-Managed), older versions can be accessed by replacing the version segment in the URL:\n`;
197
+ fullContent += `- Latest: \`${siteUrl}/current/page-path\`\n`;
198
+ fullContent += `- Specific version: \`${siteUrl}/24.3/page-path\`, \`${siteUrl}/25.1/page-path\`, etc.\n\n`;
199
+ fullContent += `Available versioned components: ${components.filter(c => c.versions.length > 1).map(c => c.name).join(', ')}\n\n`;
200
+ fullContent += `---\n\n`;
232
201
 
233
202
  // Sort pages by URL for consistent ordering
234
- componentPages.sort((a, b) => {
203
+ pages.sort((a, b) => {
235
204
  const urlA = a.pub?.url || '';
236
205
  const urlB = b.pub?.url || '';
237
206
  return urlA.localeCompare(urlB);
238
207
  });
239
208
 
240
- let componentContent = `# ${component.title} - Full Markdown Export\n\n`;
241
- componentContent += `> This file contains all ${component.title} documentation pages in markdown format for AI agent consumption.\n`;
242
- componentContent += `> Generated from ${componentPages.length} pages on ${new Date().toISOString()}\n`;
243
- componentContent += `> Component: ${component.name} | Version: ${latest.version}\n`;
244
- componentContent += `> Site: ${siteUrl}\n\n`;
245
- componentContent += `## About This Export\n\n`;
246
- componentContent += `This export includes the **latest version** (${latest.version}) of the ${component.title} documentation.\n\n`;
247
- componentContent += `### AI-Friendly Documentation Formats\n\n`;
248
- componentContent += `We provide multiple formats optimized for AI consumption:\n\n`;
249
- componentContent += `- **${siteUrl}/llms.txt**: Curated overview of all Redpanda documentation\n`;
250
- componentContent += `- **${siteUrl}/llms-full.txt**: Complete documentation export with all components\n`;
251
- componentContent += `- **${siteUrl}/${componentName}-full.txt**: This file - ${component.title} documentation only\n`;
252
- componentContent += `- **Individual markdown pages**: Each HTML page has a corresponding .md file\n\n`;
253
-
254
- if (component.versions.length > 1) {
255
- componentContent += `### Accessing Older Versions\n\n`;
256
- componentContent += `This component has versioned documentation. Older versions can be accessed by replacing the version segment in the URL:\n`;
257
- componentContent += `- Latest: \`${siteUrl}/current/page-path\`\n`;
258
- componentContent += `- Specific version: \`${siteUrl}/24.3/page-path\`, \`${siteUrl}/25.1/page-path\`, etc.\n\n`;
259
- }
260
-
261
- componentContent += `---\n\n`;
262
-
263
- // Add all pages
264
- componentPages.forEach((page, index) => {
209
+ pages.forEach((page, index) => {
265
210
  const mdUrl = page.pub?.url ? toMarkdownUrl(page.pub.url) : '';
266
211
  const pageUrl = mdUrl ? `${siteUrl}${mdUrl}` : 'unknown';
267
212
  const pageTitle = page.asciidoc?.doctitle || page.src?.stem || 'Untitled';
268
213
 
269
- componentContent += `# Page ${index + 1}: ${pageTitle}\n\n`;
270
- componentContent += `**URL**: ${pageUrl}\n\n`;
271
- componentContent += `---\n\n`;
214
+ fullContent += `# Page ${index + 1}: ${pageTitle}\n\n`;
215
+ fullContent += `**URL**: ${pageUrl}\n\n`;
216
+ fullContent += `---\n\n`;
272
217
  // Strip metadata (directive, source comments) from page content
273
- componentContent += stripMarkdownMetadata(page.markdownContents);
274
- componentContent += `\n\n---\n\n`;
218
+ fullContent += stripMarkdownMetadata(page.markdownContents);
219
+ fullContent += `\n\n---\n\n`;
275
220
  });
276
221
 
277
- // Add component-specific full.txt file to site root
222
+ // Add llms-full.txt to site root
278
223
  siteCatalog.addFile({
279
- contents: Buffer.from(componentContent, 'utf8'),
280
- out: { path: `${componentName}-full.txt` },
224
+ contents: Buffer.from(fullContent, 'utf8'),
225
+ out: { path: 'llms-full.txt' },
226
+ });
227
+ logger.info(`Generated llms-full.txt with ${pages.length} pages`);
228
+
229
+ // Generate component-specific full.txt files
230
+ logger.info('Generating component-specific full.txt files...');
231
+
232
+ // Group pages by component
233
+ pages.forEach(page => {
234
+ const componentName = page.src.component;
235
+ if (!componentGroups.has(componentName)) {
236
+ componentGroups.set(componentName, []);
237
+ }
238
+ componentGroups.get(componentName).push(page);
281
239
  });
282
- logger.info(`Generated ${componentName}-full.txt with ${componentPages.length} pages`);
240
+
241
+ // Generate a full.txt file for each component
242
+ componentGroups.forEach((componentPages, componentName) => {
243
+ const component = components.find(c => c.name === componentName);
244
+ if (!component) return;
245
+
246
+ const latest = component.latest || component.versions[0];
247
+ if (!latest) return;
248
+
249
+ // Sort pages by URL for consistent ordering
250
+ componentPages.sort((a, b) => {
251
+ const urlA = a.pub?.url || '';
252
+ const urlB = b.pub?.url || '';
253
+ return urlA.localeCompare(urlB);
254
+ });
255
+
256
+ let componentContent = `# ${component.title} - Full Markdown Export\n\n`;
257
+ componentContent += `> This file contains all ${component.title} documentation pages in markdown format for AI agent consumption.\n`;
258
+ componentContent += `> Generated from ${componentPages.length} pages on ${new Date().toISOString()}\n`;
259
+ componentContent += `> Component: ${component.name} | Version: ${latest.version}\n`;
260
+ componentContent += `> Site: ${siteUrl}\n\n`;
261
+ componentContent += `## About This Export\n\n`;
262
+ componentContent += `This export includes the **latest version** (${latest.version}) of the ${component.title} documentation.\n\n`;
263
+ componentContent += `### AI-Friendly Documentation Formats\n\n`;
264
+ componentContent += `We provide multiple formats optimized for AI consumption:\n\n`;
265
+ componentContent += `- **${siteUrl}/llms.txt**: Curated overview of all Redpanda documentation\n`;
266
+ componentContent += `- **${siteUrl}/llms-full.txt**: Complete documentation export with all components\n`;
267
+ componentContent += `- **${siteUrl}/${componentName}-full.txt**: This file - ${component.title} documentation only\n`;
268
+ componentContent += `- **Individual markdown pages**: Each HTML page has a corresponding .md file\n\n`;
269
+
270
+ if (component.versions.length > 1) {
271
+ componentContent += `### Accessing Older Versions\n\n`;
272
+ componentContent += `This component has versioned documentation. Older versions can be accessed by replacing the version segment in the URL:\n`;
273
+ componentContent += `- Latest: \`${siteUrl}/current/page-path\`\n`;
274
+ componentContent += `- Specific version: \`${siteUrl}/24.3/page-path\`, \`${siteUrl}/25.1/page-path\`, etc.\n\n`;
275
+ }
276
+
277
+ componentContent += `---\n\n`;
278
+
279
+ // Add all pages
280
+ componentPages.forEach((page, index) => {
281
+ const mdUrl = page.pub?.url ? toMarkdownUrl(page.pub.url) : '';
282
+ const pageUrl = mdUrl ? `${siteUrl}${mdUrl}` : 'unknown';
283
+ const pageTitle = page.asciidoc?.doctitle || page.src?.stem || 'Untitled';
284
+
285
+ componentContent += `# Page ${index + 1}: ${pageTitle}\n\n`;
286
+ componentContent += `**URL**: ${pageUrl}\n\n`;
287
+ componentContent += `---\n\n`;
288
+ // Strip metadata (directive, source comments) from page content
289
+ componentContent += stripMarkdownMetadata(page.markdownContents);
290
+ componentContent += `\n\n---\n\n`;
291
+ });
292
+
293
+ // Add component-specific full.txt file to site root
294
+ siteCatalog.addFile({
295
+ contents: Buffer.from(componentContent, 'utf8'),
296
+ out: { path: `${componentName}-full.txt` },
297
+ });
298
+ logger.info(`Generated ${componentName}-full.txt with ${componentPages.length} pages`);
299
+ });
300
+ }
301
+
302
+ // Generate page index files for every published version (not only the latest),
303
+ // so the llms.txt tree lists every page the sitemap lists.
304
+ const pageIndexes = buildPageIndexes({
305
+ pages: allPages,
306
+ components,
307
+ siteUrl,
308
+ toMarkdownUrl,
283
309
  });
310
+ pageIndexes.forEach((index) => {
311
+ siteCatalog.addFile({
312
+ contents: Buffer.from(index.contents, 'utf8'),
313
+ out: { path: index.path },
314
+ });
315
+ });
316
+ const indexedPageCount = pageIndexes.reduce((n, index) => n + index.pageCount, 0);
317
+ logger.info(`Generated ${pageIndexes.length} page index files covering ${indexedPageCount} pages`);
284
318
 
285
319
  // Add llms.txt to site root (using content extracted earlier)
286
320
  if (llmsPage && llmsPage.llmsTxtContent) {
@@ -297,6 +331,15 @@ module.exports.register = function () {
297
331
  llmsTxtContent = truncateAtNewline(llmsTxtContent, MAX_LLMS_TXT_CHARS - 100) + '\n\n[Content truncated due to size limits]';
298
332
  }
299
333
 
334
+ // The page index list is never truncated: an index missing from the root
335
+ // leaves its pages unreachable from llms.txt.
336
+ if (pageIndexes.length) {
337
+ llmsTxtContent = llmsTxtContent + '\n\n' + renderPageIndexSection(pageIndexes, components);
338
+ if (llmsTxtContent.length >= MAX_LLMS_TXT_CHARS) {
339
+ logger.warn(`llms.txt with page index list is ${llmsTxtContent.length} chars, over the ${MAX_LLMS_TXT_CHARS} char target. Shorten the llms page or raise the per-index size limit.`);
340
+ }
341
+ }
342
+
300
343
  // Generate navigation section with component sitemaps and key sections
301
344
  const navSection = generateNavigationSection(siteUrl, contentCatalog, components);
302
345
 
@@ -350,6 +393,10 @@ module.exports.register = function () {
350
393
  }
351
394
  });
352
395
 
396
+ pageIndexes.forEach((index) => {
397
+ if (index.lastModified) gitDates.set(index.path, index.lastModified);
398
+ });
399
+
353
400
  addToSitemap(contentCatalog, siteCatalog, siteUrl, gitDates, logger);
354
401
  } catch (err) {
355
402
  logger.warn(`Failed to add llms.txt to sitemap: ${err.message}`);
@@ -398,7 +445,8 @@ function addToSitemap(contentCatalog, siteCatalog, siteUrl, gitDates, logger) {
398
445
  const filename = file.out.path;
399
446
  return filename === 'llms.txt' ||
400
447
  filename === 'llms-full.txt' ||
401
- filename.endsWith('-full.txt');
448
+ filename.endsWith('-full.txt') ||
449
+ /(^|\/)llms(-\d+)?\.txt$/.test(filename);
402
450
  })
403
451
  .map(file => file.out.path)
404
452
  .sort(); // Sort for consistent ordering
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@redpanda-data/docs-extensions-and-macros",
3
- "version": "5.49.0",
3
+ "version": "5.51.0",
4
4
  "description": "Antora extensions and macros developed for Redpanda documentation.",
5
5
  "keywords": [
6
6
  "antora",
@@ -116,7 +116,16 @@ function lintStrings (options) {
116
116
  // still exists at HEAD, in any touched file, was edited or moved, not
117
117
  // removed.
118
118
  const removedFiles = removedBySurface[surfaceName] ? [...removedBySurface[surfaceName].keys()] : []
119
- const head = surface.extract({ repo: repoPath, files: new Set([...files.keys(), ...removedFiles]), log })
119
+ const touchedFiles = new Set([...files.keys(), ...removedFiles])
120
+ // A surface whose extraction covers the whole surface anyway (the
121
+ // properties extractor parses every config file on each run) supplies
122
+ // context from all of it, so a property in an untouched file that
123
+ // names, or is named by, a changed one is still on its page. Review
124
+ // scope stays the touched files either way.
125
+ const everything = surface.contextScope === 'surface' ? surface.extract({ repo: repoPath, log }) : null
126
+ const head = everything
127
+ ? everything.filter((d) => touchedFiles.has(d.file))
128
+ : surface.extract({ repo: repoPath, files: touchedFiles, log })
120
129
  headBySurface[surfaceName] = head
121
130
  const declarations = []
122
131
  for (const decl of head) {
@@ -128,7 +137,7 @@ function lintStrings (options) {
128
137
  continue
129
138
  }
130
139
  declarations.push(decl)
131
- pending.push(pendingEntry(decl))
140
+ pending.push({ ...pendingEntry(decl), context: pageContext(decl, everything || head) })
132
141
  }
133
142
  results.push(runRules(declarations, rulesFor(surface), { skipRules, onlyRules }))
134
143
  }
@@ -186,6 +195,59 @@ function touches (decl, lineSet) {
186
195
  return spansOf(decl).some(([start, end]) => spanIntersects(start, end, lineSet))
187
196
  }
188
197
 
198
+ const CONTEXT_LIMIT = 15
199
+ const CONTEXT_STRING_LIMIT = 600
200
+
201
+ /**
202
+ * The other strings a reader sees alongside this one, so a review can judge
203
+ * a string against its page rather than on its own: a flag's usage next to
204
+ * its command's Long text and examples, a property that names another
205
+ * property, the fields around an API field.
206
+ *
207
+ * - rpk: every other string declared in the same file, which is the
208
+ * command's Short, Long and flags (rpk keeps one command per file).
209
+ * - api: the declarations nearest by line, which are the enclosing message
210
+ * or rpc and its neighbors.
211
+ * - properties and metrics: declarations whose string names this one, or
212
+ * that this string names, plus (metrics) its neighbors in the same group.
213
+ *
214
+ * Drawn from declarations already extracted for the diff, so it costs no
215
+ * extra extraction. Deterministic, capped, and order-stable.
216
+ */
217
+ function pageContext (decl, all) {
218
+ const others = all.filter((d) => d !== decl && d.string != null)
219
+ const byDistance = (d) => Math.abs((d.line_start || 0) - (decl.line_start || 0))
220
+ const sameFile = others.filter((d) => d.file === decl.file)
221
+ let picked
222
+ if (decl.surface === 'rpk') {
223
+ // The command's Short and Long always, then the flags nearest this
224
+ // declaration, so a changed flag late in a long command keeps its
225
+ // neighbors when the cap applies. Shown in source order.
226
+ const isText = (d) => d.meta && (d.meta.kind === 'short' || d.meta.kind === 'long')
227
+ const text = sameFile.filter(isText)
228
+ const flags = sameFile.filter((d) => !isText(d)).sort((a, b) => byDistance(a) - byDistance(b))
229
+ picked = [...text, ...flags].slice(0, CONTEXT_LIMIT)
230
+ .sort((a, b) => (a.line_start || 0) - (b.line_start || 0))
231
+ } else if (decl.surface === 'api') {
232
+ picked = sameFile.sort((a, b) => byDistance(a) - byDistance(b)).slice(0, 6)
233
+ } else {
234
+ const mentions = (text, name) => Boolean(text && name) &&
235
+ new RegExp(`(^|[^A-Za-z0-9_])${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}($|[^A-Za-z0-9_])`).test(text)
236
+ const related = others.filter((d) => mentions(d.string, decl.name) || mentions(decl.string, d.name))
237
+ const neighbors = decl.surface === 'metrics'
238
+ ? sameFile.filter((d) => !related.includes(d)).sort((a, b) => byDistance(a) - byDistance(b)).slice(0, 2)
239
+ : []
240
+ picked = [...related, ...neighbors]
241
+ }
242
+ return picked.slice(0, CONTEXT_LIMIT).map((d) => ({
243
+ name: d.name,
244
+ kind: (d.meta && d.meta.kind) || d.surface,
245
+ file: d.file,
246
+ line_start: d.line_start,
247
+ string: d.string.length > CONTEXT_STRING_LIMIT ? `${d.string.slice(0, CONTEXT_STRING_LIMIT)}...` : d.string
248
+ }))
249
+ }
250
+
189
251
  /**
190
252
  * Stable identity for one version of one doc string. A PR review records the
191
253
  * fingerprints it has seen, and a later push skips any declaration whose
@@ -467,7 +529,7 @@ function runCli (options) {
467
529
  process.exitCode = options.strict && result.summary.errors > 0 ? 1 : 0
468
530
  }
469
531
 
470
- module.exports = { lintStrings, formatHuman, runCli, SURFACES, rulesFor, fingerprint, readFingerprints }
532
+ module.exports = { lintStrings, formatHuman, runCli, SURFACES, rulesFor, fingerprint, readFingerprints, pageContext }
471
533
 
472
534
  // Direct usage: node tools/lint-strings --repo <path> [--surface a,b]
473
535
  // [--diff <base>] [--format json|human] [--strict]
@@ -100,7 +100,9 @@ function maskNonProse (text) {
100
100
  const CODE_TOKEN_PATTERNS = Object.freeze([
101
101
  { id: 'identifier', re: /\b[a-z][a-z0-9]*(?:_[a-z0-9]+)+\b/g },
102
102
  { id: 'flag', re: /(?<![\w-])--[a-z][a-z0-9]*(?:-[a-z0-9]+)*\b/g },
103
- { id: 'path', re: /(?<![\w/])\/[a-z][a-zA-Z0-9_.-]*(?:\/[a-zA-Z0-9_.{}-]+)+\/?/g }
103
+ // A path may not end in a period, so "served from /v1/brokers." reports
104
+ // `/v1/brokers`, not the sentence's full stop with it.
105
+ { id: 'path', re: /(?<![\w/])\/[a-z][a-zA-Z0-9_.-]*(?:\/[a-zA-Z0-9_.{}-]*[a-zA-Z0-9_{}-])+\/?/g }
104
106
  ])
105
107
 
106
108
  /**
@@ -276,7 +278,12 @@ const COMMON_RULES = [
276
278
  description: 'Code value or field name in prose without inline code',
277
279
  severity: 'warning',
278
280
  check: (decl) => {
279
- const bare = findBareCodeTokens(decl.string, decl.name)
281
+ // A surface whose generator formats some code values itself declares
282
+ // which (convention.auto_inline_code); those are not findings there.
283
+ const auto = (decl.convention && decl.convention.auto_inline_code) || null
284
+ const bare = findBareCodeTokens(decl.string, decl.name).filter((b) => !auto ||
285
+ !((auto.kinds || []).includes(b.kind) ||
286
+ (b.kind === 'path' && (auto.path_prefixes || []).some((prefix) => b.token.startsWith(prefix)))))
280
287
  if (bare.length === 0) return []
281
288
  const list = bare.map((b) => `\`${b.token}\``).join(', ')
282
289
  return [{
@@ -245,6 +245,9 @@ module.exports = {
245
245
  // The extractor pairs each .cc with its .h, so a merge-base extraction needs
246
246
  // the whole directory, not only the files that lost lines.
247
247
  baseScope: 'directory',
248
+ // extract() runs the extractor over the whole repo and filters after, so
249
+ // page context can use every property at no extra cost.
250
+ contextScope: 'surface',
248
251
  convention: CONVENTION,
249
252
  extract,
250
253
  mapExtractorJson,
@@ -37,7 +37,15 @@ const CONVENTION = {
37
37
  case: 'sentence',
38
38
  terminal_period: false,
39
39
  verbatim_asciidoc: false,
40
- transformer: 'formatDescription'
40
+ transformer: 'formatDescription',
41
+ // Code values formatDescription wraps in inline code by itself, so a bare
42
+ // one in the source is not a finding. Long is a Go raw string and cannot
43
+ // hold a backtick at all, which is why the generator does this. Keep in
44
+ // step with the auto-backtick passes in tools/rpk-docs/generate-rpk-docs.js.
45
+ auto_inline_code: {
46
+ kinds: ['flag'],
47
+ path_prefixes: ['/etc/', '/var/', '/usr/', '/home/', '/tmp/', '~/.']
48
+ }
41
49
  }
42
50
 
43
51
  // Composite literal only: gofmt writes `cobra.Command{` with no space, while
@@ -1231,7 +1231,12 @@ breaks. Two files feed it, both mirrors of the `docs` repo:
1231
1231
  fields the rendering and link layers read. `cloud_supported` is the point: it is what
1232
1232
  makes the "no Cloud link to a property Cloud does not publish" assertion real rather
1233
1233
  than assumed. Descriptions are deliberately absent, because the corpus test drives prose from
1234
- the overrides file, and a full copy would be 695 KB of text that rots.
1234
+ the overrides file, and a full copy would be 695 KB of text that rots. The exceptions are
1235
+ overrides that act on the source description instead of replacing it: links with no prose of
1236
+ their own, which apply to the source description, and includes with no description of their
1237
+ own, which the generator appends to it. The snapshot keeps the source description for those
1238
+ properties. `.github/scripts/derive-property-snapshot.js` makes that call from the overrides
1239
+ file, and both the recipe below and `property-corpus-drift.sh` use it.
1235
1240
 
1236
1241
  Keep the overrides copy a faithful mirror. It previously doubled as
1237
1242
  `property-docs-overrides.test.js`'s fixture, someone hand-edited an `admin` entry into
@@ -1264,17 +1269,13 @@ TAG="$(git -C "$DOCS" ls-tree origin/main --name-only -- modules/reference/attac
1264
1269
  echo "Refreshing against $TAG"
1265
1270
 
1266
1271
  git -C "$DOCS" show "origin/main:modules/reference/attachments/redpanda-properties-${TAG}.json" \
1272
+ > /tmp/redpanda-properties.json
1273
+ node .github/scripts/derive-property-snapshot.js /tmp/redpanda-properties.json \
1274
+ __tests__/docs-data/property-overrides.json \
1267
1275
  | node -e '
1268
1276
  const fs = require("fs");
1269
- const src = JSON.parse(fs.readFileSync(0, "utf8"));
1270
- const keep = ["name","config_scope","type","cloud_supported","cloud_editable",
1271
- "cloud_readonly","cloud_byoc_only","is_deprecated","nullable"];
1272
- const properties = {};
1273
- for (const [k, v] of Object.entries(src.properties || {})) {
1274
- properties[k] = Object.fromEntries(keep.filter((f) => f in v).map((f) => [f, v[f]]));
1275
- }
1276
1277
  const out = JSON.parse(fs.readFileSync("__tests__/docs-data/property-snapshot.json", "utf8"));
1277
- out.properties = properties;
1278
+ out.properties = JSON.parse(fs.readFileSync(0, "utf8"));
1278
1279
  out.source_tag = process.argv[1];
1279
1280
  fs.writeFileSync("__tests__/docs-data/property-snapshot.json", JSON.stringify(out, null, 1) + "\n");
1280
1281
  ' "$TAG"
@@ -1569,6 +1569,26 @@ function formatDescription(desc, customTransformations = null, options = {}) {
1569
1569
  return placeholder
1570
1570
  })
1571
1571
 
1572
+ // Bare URLs become inline code: endpoints (http://localhost:9644) and links
1573
+ // alike are code values, and an unformatted URL in help prose reads as
1574
+ // broken markup. Not trailing sentence punctuation, and not a URL that is
1575
+ // already an AsciiDoc link macro (url[text]), which backticks would break.
1576
+ //
1577
+ // Protected as a placeholder straight away, before any other pass runs.
1578
+ // Every later pass matches inside a URL it does not know is one: the path
1579
+ // passes backticked /tmp/file out of https://host/tmp/file, the short-flag
1580
+ // pass took -a out of /foo-a/, and the issue-reference pass rewrote #NNNN
1581
+ // anchors into link macros.
1582
+ protectedDesc = protectedDesc.replace(/(?<![`\w/]|\]\()https?:\/\/[^\s`<>"'()[\]]+/g, (url, offset, whole) => {
1583
+ if (whole[offset + url.length] === '[') return url // AsciiDoc link macro
1584
+ const trail = url.match(/[.,;:!?]+$/)
1585
+ const core = trail ? url.slice(0, -trail[0].length) : url
1586
+ if (core.length <= 'https://'.length) return url
1587
+ const placeholder = `__INLINE_CODE_${inlineCode.length}__`
1588
+ inlineCode.push(`\`${decodeHtmlEntities(core)}\``)
1589
+ return placeholder + (trail ? trail[0] : '')
1590
+ })
1591
+
1572
1592
  // Known top-level rpk subcommands (for accurate command detection)
1573
1593
  const rpkSubcommands = new Set([
1574
1594
  'ai', 'check', 'cloud', 'cluster', 'connect', 'container', 'debug',
@@ -1713,6 +1733,17 @@ function formatDescription(desc, customTransformations = null, options = {}) {
1713
1733
  }).join('`')
1714
1734
  result = result.replace(/(?<!`|-)(-[a-zA-Z])(?![a-zA-Z-])/g, '`$1`')
1715
1735
 
1736
+ // A bare command followed by its flags ("run rpk topic list -r") is one
1737
+ // command, but the passes above wrap the command path and each flag
1738
+ // separately. Merge flag spans into the command span before them:
1739
+ // `rpk topic list` `-r` -> `rpk topic list -r`. The quoted form never
1740
+ // splits, because its whole span is protected before flags are wrapped.
1741
+ let merged
1742
+ do {
1743
+ merged = result
1744
+ result = result.replace(/`(rpk(?: [a-z][-a-z0-9]*)+(?: -{1,2}[a-zA-Z][-a-zA-Z0-9]*)*)` `(-{1,2}[a-zA-Z][-a-zA-Z0-9]*)`/g, '`$1 $2`')
1745
+ } while (result !== merged)
1746
+
1716
1747
  // Add backticks around environment variables
1717
1748
  result = result.replace(/(?<!`)(\$[A-Z_][A-Z0-9_]*)/g, '`$1`')
1718
1749
 
@@ -1729,12 +1760,15 @@ function formatDescription(desc, customTransformations = null, options = {}) {
1729
1760
  result = result.replace(/`(\$[A-Z_][A-Z0-9_]*)`(\/(?:[^\s`]*[^\s`).,;:!?])?)/g, '`$1$2`')
1730
1761
 
1731
1762
  // Add backticks around file paths (but not if already backticked)
1732
- // Must check both the slash and the path aren't already inside backticks
1733
- result = result.replace(/(?<![`/])(\/(?:etc|var|usr|home|tmp)\/[^\s,)`]+)/g, '`$1`')
1734
- result = result.replace(/(?<![`/])((?:etc|var|usr|home|tmp)\/[^\s,)`]+)/g, '`$1`')
1763
+ // Must check both the slash and the path aren't already inside backticks.
1764
+ // Like the $HOME merge above, a path may not END in sentence punctuation,
1765
+ // so "...in ~/.config/rpk/rpk.yaml." puts the period outside the span
1766
+ // instead of publishing `~/.config/rpk/rpk.yaml.`.
1767
+ result = result.replace(/(?<![`/])(\/(?:etc|var|usr|home|tmp)\/(?:[^\s,)`]*[^\s,)`.;:!?])?)/g, '`$1`')
1768
+ result = result.replace(/(?<![`/])((?:etc|var|usr|home|tmp)\/(?:[^\s,)`]*[^\s,)`.;:!?])?)/g, '`$1`')
1735
1769
 
1736
1770
  // Add backticks around home directory paths (~/.bashrc, ~/.zshrc, ~/.config/...)
1737
- result = result.replace(/(?<![`\w])(~\/\.[^\s,;:)`]+)/g, '`$1`')
1771
+ result = result.replace(/(?<![`\w])(~\/\.(?:[^\s,;:)`]*[^\s,;:)`.!?])?)/g, '`$1`')
1738
1772
 
1739
1773
  // Add backticks around common package names
1740
1774
  result = result.replace(/(?<![`\w-])(bash-completion)(?![`\w-])/g, '`$1`')