@redpanda-data/docs-extensions-and-macros 5.49.0 → 5.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extension-utils/llms-utils.js +338 -1
- package/extensions/REFERENCE.adoc +13 -1
- package/extensions/convert-llms-to-txt.js +168 -120
- package/package.json +1 -1
- package/tools/lint-strings/index.js +65 -3
- package/tools/lint-strings/rules/common.js +9 -2
- package/tools/lint-strings/surfaces/properties.js +3 -0
- package/tools/lint-strings/surfaces/rpk.js +9 -1
- package/tools/property-extractor/README.adoc +10 -9
- package/tools/rpk-docs/generate-rpk-docs.js +38 -4
|
@@ -37,9 +37,14 @@ function escapeRegExp(str) {
|
|
|
37
37
|
* Regex pattern to match and strip the llms directive from markdown content.
|
|
38
38
|
* Dynamically derived from LLMS_DIRECTIVE_BASE to stay in sync.
|
|
39
39
|
* Matches the blockquote format with optional component-specific suffix.
|
|
40
|
+
* The directive is written with a root-relative link, but links are made
|
|
41
|
+
* absolute before the exports strip it, so any prefix before /llms.txt matches.
|
|
40
42
|
*/
|
|
41
43
|
const LLMS_DIRECTIVE_REGEX = new RegExp(
|
|
42
|
-
`^> ${escapeRegExp(LLMS_DIRECTIVE_BASE)
|
|
44
|
+
`^> ${escapeRegExp(LLMS_DIRECTIVE_BASE).replace(
|
|
45
|
+
escapeRegExp('(/llms.txt)'),
|
|
46
|
+
'\\([^)\\s]*/llms\\.txt\\)'
|
|
47
|
+
)}.*$`,
|
|
43
48
|
'gm'
|
|
44
49
|
);
|
|
45
50
|
|
|
@@ -96,11 +101,343 @@ function componentsWithExports(components, pages) {
|
|
|
96
101
|
);
|
|
97
102
|
}
|
|
98
103
|
|
|
104
|
+
/**
|
|
105
|
+
* Maximum characters per page index file. The Agent-Friendly Documentation
|
|
106
|
+
* Spec passes an llms.txt index at 50,000 characters or less and recommends
|
|
107
|
+
* splitting a large index into section-level files that each stay under that
|
|
108
|
+
* limit. Leave a buffer below 50K.
|
|
109
|
+
*/
|
|
110
|
+
const MAX_INDEX_CHARS = 45000;
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Descriptions longer than this are cut at a word boundary so that one verbose
|
|
114
|
+
* page doesn't force an extra split.
|
|
115
|
+
*/
|
|
116
|
+
const MAX_INDEX_DESCRIPTION_CHARS = 200;
|
|
117
|
+
|
|
118
|
+
function oneLine(text) {
|
|
119
|
+
return String(text || '')
|
|
120
|
+
.replace(/<[^>]+>/g, '')
|
|
121
|
+
.replace(/\s+/g, ' ')
|
|
122
|
+
.trim();
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function truncateDescription(text) {
|
|
126
|
+
const clean = oneLine(text);
|
|
127
|
+
if (clean.length <= MAX_INDEX_DESCRIPTION_CHARS) return clean;
|
|
128
|
+
const cut = clean.slice(0, MAX_INDEX_DESCRIPTION_CHARS);
|
|
129
|
+
const lastSpace = cut.lastIndexOf(' ');
|
|
130
|
+
return `${(lastSpace > 0 ? cut.slice(0, lastSpace) : cut).replace(/[\s.,;:]+$/, '')}...`;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Square brackets in link text end the link early in markdown.
|
|
134
|
+
function escapeLinkText(text) {
|
|
135
|
+
return oneLine(text).replace(/([[\]])/g, '\\$1');
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* A URL made safe as a Markdown link destination. An unmatched `)` ends the
|
|
140
|
+
* destination early, and a space or `<` ends it outright, so those are
|
|
141
|
+
* percent-encoded. Encoding keeps the plain `[text](url)` shape llms.txt
|
|
142
|
+
* readers expect, where an angle-bracket destination would not.
|
|
143
|
+
*/
|
|
144
|
+
function linkDestination(url) {
|
|
145
|
+
return String(url).replace(/[()<> ]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Longest shared directory of a set of URL paths, always ending in `/`.
|
|
150
|
+
* `/streaming/26.1/manage/a/` and `/streaming/26.1/deploy/b/` share `/streaming/26.1/`.
|
|
151
|
+
*/
|
|
152
|
+
function commonDirectory(urls) {
|
|
153
|
+
const dirs = urls.map((url) => url.replace(/[^/]*\/?$/, '').split('/').filter(Boolean));
|
|
154
|
+
if (!dirs.length) return '/';
|
|
155
|
+
const shared = [];
|
|
156
|
+
for (let i = 0; i < dirs[0].length; i++) {
|
|
157
|
+
const segment = dirs[0][i];
|
|
158
|
+
if (!dirs.every((d) => d[i] === segment)) break;
|
|
159
|
+
shared.push(segment);
|
|
160
|
+
}
|
|
161
|
+
return shared.length ? `/${shared.join('/')}/` : '/';
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* URL of the component version's root, derived from how Antora built the page
|
|
166
|
+
* URL: the version root, then the module name (omitted for ROOT), then the
|
|
167
|
+
* page's relative path (index pages drop their own segment). Works for both
|
|
168
|
+
* indexified (`/a/b/`) and `.html` URLs, since the page is one segment either
|
|
169
|
+
* way, except an index page, whose `.html` URL has an `index.html` segment
|
|
170
|
+
* that its indexified URL does not.
|
|
171
|
+
* Returns null when the page lacks the source fields needed.
|
|
172
|
+
*/
|
|
173
|
+
function versionRootOf(page) {
|
|
174
|
+
const { module: moduleName, relative } = page.src || {};
|
|
175
|
+
if (!relative || !moduleName) return null;
|
|
176
|
+
const relSegments = relative.replace(/\.adoc$/, '').split('/').filter(Boolean);
|
|
177
|
+
const urlSegments = page.pub.url.split('/').filter(Boolean);
|
|
178
|
+
if (relSegments[relSegments.length - 1] === 'index') {
|
|
179
|
+
relSegments.pop();
|
|
180
|
+
// An index page keeps an `index.html` segment of its own when URLs are
|
|
181
|
+
// not indexified (`/streaming/26.1/index.html`), and has none when they
|
|
182
|
+
// are (`/streaming/26.1/`).
|
|
183
|
+
if (urlSegments[urlSegments.length - 1] === 'index.html') urlSegments.pop();
|
|
184
|
+
}
|
|
185
|
+
const tailCount = relSegments.length + (moduleName === 'ROOT' ? 0 : 1);
|
|
186
|
+
if (tailCount > urlSegments.length) return null;
|
|
187
|
+
const rootSegments = urlSegments.slice(0, urlSegments.length - tailCount);
|
|
188
|
+
return rootSegments.length ? `/${rootSegments.join('/')}/` : '/';
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function mostCommon(values) {
|
|
192
|
+
const counts = new Map();
|
|
193
|
+
values.forEach((v) => counts.set(v, (counts.get(v) || 0) + 1));
|
|
194
|
+
let best = null;
|
|
195
|
+
counts.forEach((count, value) => {
|
|
196
|
+
if (best === null || count > counts.get(best)) best = value;
|
|
197
|
+
});
|
|
198
|
+
return best;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* The directory segment directly under `dir` that contains `url`, or null when
|
|
203
|
+
* the page sits in `dir` itself. With indexified URLs a page `/a/b/` lives in
|
|
204
|
+
* `/a/`, so the last segment is the page, not a directory.
|
|
205
|
+
*/
|
|
206
|
+
function childSegment(url, dir) {
|
|
207
|
+
const rest = url.slice(dir.length).replace(/\/$/, '').replace(/\/index\.html$/, '');
|
|
208
|
+
const segments = rest.split('/').filter(Boolean);
|
|
209
|
+
return segments.length > 1 ? segments[0] : null;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function latestDate(dates) {
|
|
213
|
+
const times = dates.filter(Boolean).map((d) => new Date(d)).filter((d) => !isNaN(d));
|
|
214
|
+
return times.length ? new Date(Math.max(...times)).toISOString() : null;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function renderIndexFile({ heading, intro, entries }) {
|
|
218
|
+
let out = `# ${heading}\n\n`;
|
|
219
|
+
intro.forEach((line) => {
|
|
220
|
+
out += `> ${line}\n`;
|
|
221
|
+
});
|
|
222
|
+
out += `\n## Pages\n\n`;
|
|
223
|
+
entries.forEach((entry) => {
|
|
224
|
+
out += `${entry.line}\n`;
|
|
225
|
+
});
|
|
226
|
+
return out;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Pack entries into chunks whose rendered size stays under the limit.
|
|
231
|
+
* Used only when a single directory holds more pages than one file allows.
|
|
232
|
+
*/
|
|
233
|
+
function chunkEntries(entries, overhead, maxChars) {
|
|
234
|
+
const chunks = [];
|
|
235
|
+
let current = [];
|
|
236
|
+
let size = overhead;
|
|
237
|
+
entries.forEach((entry) => {
|
|
238
|
+
const lineSize = entry.line.length + 1;
|
|
239
|
+
if (current.length && size + lineSize > maxChars) {
|
|
240
|
+
chunks.push(current);
|
|
241
|
+
current = [];
|
|
242
|
+
size = overhead;
|
|
243
|
+
}
|
|
244
|
+
current.push(entry);
|
|
245
|
+
size += lineSize;
|
|
246
|
+
});
|
|
247
|
+
if (current.length) chunks.push(current);
|
|
248
|
+
return chunks;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Build page index files for every published component version.
|
|
253
|
+
*
|
|
254
|
+
* Each component version gets an index at `<version root>/llms.txt` listing
|
|
255
|
+
* every page as a markdown link to its .md URL. When a version's index would
|
|
256
|
+
* exceed `maxChars`, it is split by URL directory, recursively, so each file
|
|
257
|
+
* stays under the limit. The root llms.txt links to every file this returns,
|
|
258
|
+
* which keeps every page one hop from the root index.
|
|
259
|
+
*
|
|
260
|
+
* @param {Object} options
|
|
261
|
+
* @param {Array} options.pages - Antora pages that have markdown (page.markdownContents)
|
|
262
|
+
* @param {Array} options.components - Antora components
|
|
263
|
+
* @param {string} options.siteUrl - Absolute site URL without a trailing slash
|
|
264
|
+
* @param {Function} options.toMarkdownUrl - Maps a page's HTML URL to its .md URL
|
|
265
|
+
* @param {number} [options.maxChars] - Size limit per file
|
|
266
|
+
* @returns {Array<{path: string, url: string, title: string, component: string,
|
|
267
|
+
* version: string, isLatest: boolean, pageCount: number, contents: string}>}
|
|
268
|
+
*/
|
|
269
|
+
function buildPageIndexes({ pages, components, siteUrl, toMarkdownUrl, maxChars = MAX_INDEX_CHARS }) {
|
|
270
|
+
const componentsByName = new Map((components || []).map((c) => [c.name, c]));
|
|
271
|
+
const groups = new Map();
|
|
272
|
+
|
|
273
|
+
(pages || []).forEach((page) => {
|
|
274
|
+
const url = page.pub && page.pub.url;
|
|
275
|
+
if (!url || !page.out || !page.src) return;
|
|
276
|
+
const key = `${page.src.version}@${page.src.component}`;
|
|
277
|
+
if (!groups.has(key)) groups.set(key, []);
|
|
278
|
+
groups.get(key).push(page);
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
const indexes = [];
|
|
282
|
+
|
|
283
|
+
groups.forEach((groupPages) => {
|
|
284
|
+
const { component: componentName, version } = groupPages[0].src;
|
|
285
|
+
const component = componentsByName.get(componentName);
|
|
286
|
+
if (!component) return;
|
|
287
|
+
const componentVersion = (component.versions || []).find((v) => v.version === version) || {};
|
|
288
|
+
const latest = component.latest || (component.versions || [])[0] || {};
|
|
289
|
+
const isLatest = latest.version === version;
|
|
290
|
+
const versioned = (component.versions || []).length > 1;
|
|
291
|
+
const displayVersion = componentVersion.displayVersion || version;
|
|
292
|
+
const baseTitle = versioned
|
|
293
|
+
? `${component.title} ${displayVersion}`
|
|
294
|
+
: component.title;
|
|
295
|
+
|
|
296
|
+
const entries = groupPages
|
|
297
|
+
.map((page) => {
|
|
298
|
+
const title = page.asciidoc && (page.asciidoc.doctitle || page.asciidoc.navtitle);
|
|
299
|
+
const description = page.asciidoc && page.asciidoc.attributes && page.asciidoc.attributes.description;
|
|
300
|
+
const mdUrl = `${siteUrl}${toMarkdownUrl(page.pub.url)}`;
|
|
301
|
+
let line = `- [${escapeLinkText(title || page.src.stem)}](${linkDestination(mdUrl)})`;
|
|
302
|
+
if (description) line += `: ${truncateDescription(description)}`;
|
|
303
|
+
const modified = page.asciidoc && page.asciidoc.attributes && page.asciidoc.attributes['page-git-modified-date'];
|
|
304
|
+
return { url: page.pub.url, line, modified };
|
|
305
|
+
})
|
|
306
|
+
.sort((a, b) => a.url.localeCompare(b.url));
|
|
307
|
+
|
|
308
|
+
const roots = groupPages.map(versionRootOf).filter(Boolean);
|
|
309
|
+
const versionRoot = roots.length ? mostCommon(roots) : commonDirectory(entries.map((e) => e.url));
|
|
310
|
+
|
|
311
|
+
const intro = (scope) => {
|
|
312
|
+
const lines = [
|
|
313
|
+
`Index of ${scope} documentation pages. Each link points to the page's markdown version.`,
|
|
314
|
+
];
|
|
315
|
+
if (versioned && !isLatest) {
|
|
316
|
+
lines.push(`This is version ${displayVersion}. The latest version is ${latest.displayVersion || latest.version}.`);
|
|
317
|
+
}
|
|
318
|
+
lines.push(`For the full documentation index, see ${siteUrl}/llms.txt`);
|
|
319
|
+
return lines;
|
|
320
|
+
};
|
|
321
|
+
|
|
322
|
+
const makeIndex = (dir, dirEntries, label, part) => {
|
|
323
|
+
const heading = label ? `${baseTitle}: ${label}` : baseTitle;
|
|
324
|
+
const title = part > 1 ? `${heading} (part ${part})` : heading;
|
|
325
|
+
const path = `${dir.replace(/^\//, '')}${part > 1 ? `llms-${part}.txt` : 'llms.txt'}`;
|
|
326
|
+
return {
|
|
327
|
+
path,
|
|
328
|
+
url: `${siteUrl}/${path}`,
|
|
329
|
+
title,
|
|
330
|
+
component: componentName,
|
|
331
|
+
version,
|
|
332
|
+
isLatest,
|
|
333
|
+
pageCount: dirEntries.length,
|
|
334
|
+
lastModified: latestDate(dirEntries.map((e) => e.modified)),
|
|
335
|
+
contents: renderIndexFile({
|
|
336
|
+
heading: title,
|
|
337
|
+
intro: intro(label ? `${baseTitle} ${label}` : baseTitle),
|
|
338
|
+
entries: dirEntries,
|
|
339
|
+
}),
|
|
340
|
+
};
|
|
341
|
+
};
|
|
342
|
+
|
|
343
|
+
const split = (dir, dirEntries) => {
|
|
344
|
+
const label = dir.slice(versionRoot.length).replace(/\/$/, '');
|
|
345
|
+
const whole = makeIndex(dir, dirEntries, label, 1);
|
|
346
|
+
if (whole.contents.length <= maxChars) {
|
|
347
|
+
indexes.push(whole);
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
const children = new Map();
|
|
352
|
+
dirEntries.forEach((entry) => {
|
|
353
|
+
const child = childSegment(entry.url, dir);
|
|
354
|
+
if (child === null) return;
|
|
355
|
+
if (!children.has(child)) children.set(child, []);
|
|
356
|
+
children.get(child).push(entry);
|
|
357
|
+
});
|
|
358
|
+
|
|
359
|
+
// Peel off the largest subdirectories into their own indexes until the
|
|
360
|
+
// rest fits. Small subdirectories stay in this directory's index, so the
|
|
361
|
+
// split produces as few files as the limit allows.
|
|
362
|
+
const fits = (list) => makeIndex(dir, list, label, 1).contents.length <= maxChars;
|
|
363
|
+
const bySize = [...children.entries()].sort((a, b) => b[1].length - a[1].length);
|
|
364
|
+
const kept = new Set(bySize.map(([child]) => child));
|
|
365
|
+
const remaining = () => dirEntries.filter((entry) => {
|
|
366
|
+
const child = childSegment(entry.url, dir);
|
|
367
|
+
return child === null || kept.has(child);
|
|
368
|
+
});
|
|
369
|
+
for (const [child, childEntries] of bySize) {
|
|
370
|
+
if (fits(remaining())) break;
|
|
371
|
+
kept.delete(child);
|
|
372
|
+
split(`${dir}${child}/`, childEntries);
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
const rest = remaining();
|
|
376
|
+
if (!rest.length) return;
|
|
377
|
+
// A directory whose own pages still exceed the limit is split into numbered parts.
|
|
378
|
+
// Header size of the largest possible part, so no chunk overflows once rendered.
|
|
379
|
+
const overhead = makeIndex(dir, [], label, 99).contents.length;
|
|
380
|
+
chunkEntries(rest, overhead, maxChars).forEach((chunk, i) => indexes.push(makeIndex(dir, chunk, label, i + 1)));
|
|
381
|
+
};
|
|
382
|
+
|
|
383
|
+
split(versionRoot, entries);
|
|
384
|
+
});
|
|
385
|
+
|
|
386
|
+
// Latest versions first, then older versions newest first, each in URL order.
|
|
387
|
+
return indexes.sort((a, b) => {
|
|
388
|
+
if (a.component !== b.component) return a.component.localeCompare(b.component);
|
|
389
|
+
if (a.isLatest !== b.isLatest) return a.isLatest ? -1 : 1;
|
|
390
|
+
if (a.version !== b.version) return b.version.localeCompare(a.version, undefined, { numeric: true });
|
|
391
|
+
return a.path.localeCompare(b.path);
|
|
392
|
+
});
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* Replace markdown links to the root llms.txt with their plain link text.
|
|
397
|
+
* A root index that links to itself is read as a nested index by tools that
|
|
398
|
+
* walk llms.txt, which pushes every page index one level deeper than intended.
|
|
399
|
+
*/
|
|
400
|
+
function unlinkSelfReferences(content, siteUrl) {
|
|
401
|
+
const self = new RegExp(`\\[([^\\]]*)\\]\\((?:${escapeRegExp(siteUrl)})?/llms\\.txt\\)`, 'g');
|
|
402
|
+
return content.replace(self, '$1');
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/**
|
|
406
|
+
* Render the root llms.txt section that links to every page index file.
|
|
407
|
+
* Every index must be linked from the root so agents, and tools that follow
|
|
408
|
+
* one level of nested indexes, can reach every page.
|
|
409
|
+
*/
|
|
410
|
+
function renderPageIndexSection(indexes, components) {
|
|
411
|
+
const titles = new Map((components || []).map((c) => [c.name, c.title]));
|
|
412
|
+
let out = `## Page indexes\n\n`;
|
|
413
|
+
out += `Each file below lists pages as links to their markdown versions. Start with the latest version of a product unless you need a specific older version.\n\n`;
|
|
414
|
+
let currentComponent = null;
|
|
415
|
+
let olderHeadingShown = false;
|
|
416
|
+
indexes.forEach((index) => {
|
|
417
|
+
if (index.component !== currentComponent) {
|
|
418
|
+
currentComponent = index.component;
|
|
419
|
+
olderHeadingShown = false;
|
|
420
|
+
out += `${currentComponent === indexes[0].component ? '' : '\n'}### ${titles.get(index.component) || index.component}\n\n`;
|
|
421
|
+
}
|
|
422
|
+
if (!index.isLatest && !olderHeadingShown) {
|
|
423
|
+
olderHeadingShown = true;
|
|
424
|
+
out += `\nOlder versions:\n\n`;
|
|
425
|
+
}
|
|
426
|
+
out += `- [${escapeLinkText(index.title)}](${linkDestination(index.url)}): ${index.pageCount} ${index.pageCount === 1 ? 'page' : 'pages'}\n`;
|
|
427
|
+
});
|
|
428
|
+
return out;
|
|
429
|
+
}
|
|
430
|
+
|
|
99
431
|
module.exports = {
|
|
100
432
|
LLMS_DIRECTIVE_BASE,
|
|
101
433
|
LLMS_DIRECTIVE_REGEX,
|
|
102
434
|
SOURCE_COMMENT_REGEX,
|
|
435
|
+
MAX_INDEX_CHARS,
|
|
103
436
|
formatLlmsDirective,
|
|
437
|
+
linkDestination,
|
|
104
438
|
stripMarkdownMetadata,
|
|
105
439
|
componentsWithExports,
|
|
440
|
+
buildPageIndexes,
|
|
441
|
+
renderPageIndexSection,
|
|
442
|
+
unlinkSelfReferences,
|
|
106
443
|
};
|
|
@@ -1267,6 +1267,7 @@ The `convert-llms-to-txt` extension generates AI-optimized documentation exports
|
|
|
1267
1267
|
- Generates `llms.txt` from a curated `llms.adoc` page in the home component
|
|
1268
1268
|
- Creates `llms-full.txt` with all documentation pages
|
|
1269
1269
|
- Generates component-specific exports (e.g., `ROOT-full.txt`, `cloud-full.txt`)
|
|
1270
|
+
- Generates a page index (`llms.txt`) for every published component version, linked from the root `llms.txt`
|
|
1270
1271
|
- Unpublishes the HTML version of llms.adoc
|
|
1271
1272
|
- Replaces site-url attribute dynamically for preview builds
|
|
1272
1273
|
|
|
@@ -1290,7 +1291,8 @@ Create `modules/home/pages/llms.adoc` in your docs repository with curated docum
|
|
|
1290
1291
|
2. Place `llms.txt` at site root
|
|
1291
1292
|
3. Generate `llms-full.txt` with all pages
|
|
1292
1293
|
4. Generate component-specific full.txt files
|
|
1293
|
-
5.
|
|
1294
|
+
5. Generate a page index for every published component version and link each one from `llms.txt`
|
|
1295
|
+
6. Unpublish the HTML version
|
|
1294
1296
|
|
|
1295
1297
|
=== Generated files
|
|
1296
1298
|
|
|
@@ -1299,6 +1301,16 @@ Create `modules/home/pages/llms.adoc` in your docs repository with curated docum
|
|
|
1299
1301
|
- `{site-url}/ROOT-full.txt` - Self-Managed docs only
|
|
1300
1302
|
- `{site-url}/cloud-full.txt` - Cloud docs only
|
|
1301
1303
|
- `{site-url}/redpanda-connect-full.txt` - Connect docs only
|
|
1304
|
+
- `{site-url}/<component>/<version>/llms.txt` - Page index for one component version
|
|
1305
|
+
- `{site-url}/<component>/llms.txt` - Page index for a component without versions, such as `connect/llms.txt`
|
|
1306
|
+
|
|
1307
|
+
=== Page indexes
|
|
1308
|
+
|
|
1309
|
+
Each page index lists every published page in one component version as a markdown link to the page's `.md` URL, followed by the page description when it has one. Older versions get their own indexes, which say which version they cover and name the latest version.
|
|
1310
|
+
|
|
1311
|
+
An index that would exceed 45,000 characters is split by URL directory, for example `streaming/current/manage/llms.txt`. A directory that is still too large on its own is split into numbered parts (`llms.txt`, `llms-2.txt`, and so on). The Agent-Friendly Documentation Spec passes an `llms.txt` at 50,000 characters or less and recommends this kind of split for large sites.
|
|
1312
|
+
|
|
1313
|
+
The root `llms.txt` links to every page index in a *Page indexes* section. The index list is never truncated, because an index missing from the root leaves its pages unreachable from `llms.txt`. Page indexes do not link to other page indexes, so every page is one hop from the root.
|
|
1302
1314
|
|
|
1303
1315
|
=== Attributes
|
|
1304
1316
|
|
|
@@ -3,7 +3,13 @@
|
|
|
3
3
|
const { raiseListenerLimit } = require('./util/raise-listener-limit')
|
|
4
4
|
|
|
5
5
|
const { toMarkdownUrl } = require('../extension-utils/url-utils');
|
|
6
|
-
const {
|
|
6
|
+
const {
|
|
7
|
+
stripMarkdownMetadata,
|
|
8
|
+
componentsWithExports,
|
|
9
|
+
buildPageIndexes,
|
|
10
|
+
renderPageIndexSection,
|
|
11
|
+
unlinkSelfReferences,
|
|
12
|
+
} = require('../extension-utils/llms-utils');
|
|
7
13
|
|
|
8
14
|
/**
|
|
9
15
|
* Extracts markdown from llms.adoc page and generates AI-friendly documentation exports.
|
|
@@ -18,6 +24,9 @@ const { stripMarkdownMetadata, componentsWithExports } = require('../extension-u
|
|
|
18
24
|
* 5. Places llms.txt (markdown) at site root
|
|
19
25
|
* 6. Generates llms-full.txt with markdown from latest versions of all components
|
|
20
26
|
* 7. Generates component-specific full.txt files (e.g., redpanda-full.txt, cloud-full.txt)
|
|
27
|
+
* 8. Generates page index files (<version root>/llms.txt) for every published component
|
|
28
|
+
* version, split by URL directory to stay under the size limit, and links each
|
|
29
|
+
* one from the root llms.txt so every page is one hop from the root index
|
|
21
30
|
*
|
|
22
31
|
* Must run after convert-to-markdown extension to access page.markdownContents.
|
|
23
32
|
*/
|
|
@@ -110,6 +119,8 @@ module.exports.register = function () {
|
|
|
110
119
|
logger.info('Unpublished llms HTML page');
|
|
111
120
|
}
|
|
112
121
|
|
|
122
|
+
content = unlinkSelfReferences(content, siteUrl);
|
|
123
|
+
|
|
113
124
|
// Store cleaned markdown content for adding after llms-full.txt
|
|
114
125
|
llmsPage.llmsTxtContent = content;
|
|
115
126
|
|
|
@@ -142,145 +153,168 @@ module.exports.register = function () {
|
|
|
142
153
|
return latestVersions.has(pageKey);
|
|
143
154
|
});
|
|
144
155
|
|
|
145
|
-
if (!
|
|
146
|
-
logger.warn('No pages with markdown content found
|
|
156
|
+
if (!allPages.length) {
|
|
157
|
+
logger.warn('No pages with markdown content found, skipping llms.txt generation');
|
|
147
158
|
return;
|
|
148
159
|
}
|
|
149
160
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
let fullContent = `# Redpanda Documentation - Full Markdown Export\n\n`;
|
|
153
|
-
fullContent += `> This file contains all documentation pages in markdown format for AI agent consumption.\n`;
|
|
154
|
-
fullContent += `> Generated from ${pages.length} pages on ${new Date().toISOString()}\n`;
|
|
155
|
-
fullContent += `> Site: ${siteUrl}\n\n`;
|
|
156
|
-
fullContent += `## About This Export\n\n`;
|
|
157
|
-
fullContent += `This export includes only the **latest version** of each component's documentation:\n`;
|
|
158
|
-
components.forEach(component => {
|
|
159
|
-
const latest = component.latest || component.versions[0];
|
|
160
|
-
if (latest) {
|
|
161
|
-
fullContent += `- **${component.title}**: version ${latest.version}\n`;
|
|
162
|
-
}
|
|
163
|
-
});
|
|
164
|
-
fullContent += `\n`;
|
|
165
|
-
fullContent += `### AI-Friendly Documentation Formats\n\n`;
|
|
166
|
-
fullContent += `We provide multiple formats optimized for AI consumption:\n\n`;
|
|
167
|
-
fullContent += `- **${siteUrl}/llms.txt**: Curated overview following the llms.txt standard - start here for a quick introduction\n`;
|
|
168
|
-
fullContent += `- **${siteUrl}/llms-full.txt**: Complete documentation export (this file) - comprehensive reference with all pages\n`;
|
|
169
|
-
fullContent += `- **Component-specific exports**: Focused documentation for individual products:\n`;
|
|
170
|
-
// Only advertise exports for components that actually have pages — a
|
|
171
|
-
// `<name>-full.txt` is generated per component below, so listing corpus-less
|
|
172
|
-
// landing/utility components (e.g. data-platform, self-managed, search) would
|
|
173
|
-
// point the AI index at a 404.
|
|
174
|
-
componentsWithExports(components, pages).forEach(component => {
|
|
175
|
-
fullContent += ` - \`${siteUrl}/${component.name}-full.txt\`: ${component.title}\n`;
|
|
176
|
-
});
|
|
177
|
-
fullContent += `- **Individual markdown pages**: Each HTML page has a corresponding .md file (e.g., \`/docs/page.html\` → \`/docs/page.md\`)\n\n`;
|
|
178
|
-
fullContent += `### Accessing Versioned Content\n\n`;
|
|
179
|
-
fullContent += `For components with versioned documentation (like Redpanda Self-Managed), older versions can be accessed by replacing the version segment in the URL:\n`;
|
|
180
|
-
fullContent += `- Latest: \`${siteUrl}/current/page-path\`\n`;
|
|
181
|
-
fullContent += `- Specific version: \`${siteUrl}/24.3/page-path\`, \`${siteUrl}/25.1/page-path\`, etc.\n\n`;
|
|
182
|
-
fullContent += `Available versioned components: ${components.filter(c => c.versions.length > 1).map(c => c.name).join(', ')}\n\n`;
|
|
183
|
-
fullContent += `---\n\n`;
|
|
184
|
-
|
|
185
|
-
// Sort pages by URL for consistent ordering
|
|
186
|
-
pages.sort((a, b) => {
|
|
187
|
-
const urlA = a.pub?.url || '';
|
|
188
|
-
const urlB = b.pub?.url || '';
|
|
189
|
-
return urlA.localeCompare(urlB);
|
|
190
|
-
});
|
|
191
|
-
|
|
192
|
-
pages.forEach((page, index) => {
|
|
193
|
-
const mdUrl = page.pub?.url ? toMarkdownUrl(page.pub.url) : '';
|
|
194
|
-
const pageUrl = mdUrl ? `${siteUrl}${mdUrl}` : 'unknown';
|
|
195
|
-
const pageTitle = page.asciidoc?.doctitle || page.src?.stem || 'Untitled';
|
|
196
|
-
|
|
197
|
-
fullContent += `# Page ${index + 1}: ${pageTitle}\n\n`;
|
|
198
|
-
fullContent += `**URL**: ${pageUrl}\n\n`;
|
|
199
|
-
fullContent += `---\n\n`;
|
|
200
|
-
// Strip metadata (directive, source comments) from page content
|
|
201
|
-
fullContent += stripMarkdownMetadata(page.markdownContents);
|
|
202
|
-
fullContent += `\n\n---\n\n`;
|
|
203
|
-
});
|
|
204
|
-
|
|
205
|
-
// Add llms-full.txt to site root
|
|
206
|
-
siteCatalog.addFile({
|
|
207
|
-
contents: Buffer.from(fullContent, 'utf8'),
|
|
208
|
-
out: { path: 'llms-full.txt' },
|
|
209
|
-
});
|
|
210
|
-
logger.info(`Generated llms-full.txt with ${pages.length} pages`);
|
|
211
|
-
|
|
212
|
-
// Generate component-specific full.txt files
|
|
213
|
-
logger.info('Generating component-specific full.txt files...');
|
|
161
|
+
// The full exports cover the latest versions only, but page indexes cover
|
|
162
|
+
// every published version, so an empty latest set skips only the exports.
|
|
214
163
|
const componentGroups = new Map();
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
164
|
+
if (!pages.length) {
|
|
165
|
+
logger.warn('No pages with markdown content found in latest versions, skipping llms-full.txt generation');
|
|
166
|
+
} else {
|
|
167
|
+
logger.info(`Filtered to ${pages.length} pages from ${latestVersions.size} latest component versions (from ${allPages.length} total pages)`);
|
|
168
|
+
|
|
169
|
+
let fullContent = `# Redpanda Documentation - Full Markdown Export\n\n`;
|
|
170
|
+
fullContent += `> This file contains all documentation pages in markdown format for AI agent consumption.\n`;
|
|
171
|
+
fullContent += `> Generated from ${pages.length} pages on ${new Date().toISOString()}\n`;
|
|
172
|
+
fullContent += `> Site: ${siteUrl}\n\n`;
|
|
173
|
+
fullContent += `## About This Export\n\n`;
|
|
174
|
+
fullContent += `This export includes only the **latest version** of each component's documentation:\n`;
|
|
175
|
+
components.forEach(component => {
|
|
176
|
+
const latest = component.latest || component.versions[0];
|
|
177
|
+
if (latest) {
|
|
178
|
+
fullContent += `- **${component.title}**: version ${latest.version}\n`;
|
|
179
|
+
}
|
|
180
|
+
});
|
|
181
|
+
fullContent += `\n`;
|
|
182
|
+
fullContent += `### AI-Friendly Documentation Formats\n\n`;
|
|
183
|
+
fullContent += `We provide multiple formats optimized for AI consumption:\n\n`;
|
|
184
|
+
fullContent += `- **${siteUrl}/llms.txt**: Curated overview following the llms.txt standard - start here for a quick introduction\n`;
|
|
185
|
+
fullContent += `- **${siteUrl}/llms-full.txt**: Complete documentation export (this file) - comprehensive reference with all pages\n`;
|
|
186
|
+
fullContent += `- **Component-specific exports**: Focused documentation for individual products:\n`;
|
|
187
|
+
// Only advertise exports for components that actually have pages — a
|
|
188
|
+
// `<name>-full.txt` is generated per component below, so listing corpus-less
|
|
189
|
+
// landing/utility components (e.g. data-platform, self-managed, search) would
|
|
190
|
+
// point the AI index at a 404.
|
|
191
|
+
componentsWithExports(components, pages).forEach(component => {
|
|
192
|
+
fullContent += ` - \`${siteUrl}/${component.name}-full.txt\`: ${component.title}\n`;
|
|
193
|
+
});
|
|
194
|
+
fullContent += `- **Individual markdown pages**: Each HTML page has a corresponding .md file (e.g., \`/docs/page.html\` → \`/docs/page.md\`)\n\n`;
|
|
195
|
+
fullContent += `### Accessing Versioned Content\n\n`;
|
|
196
|
+
fullContent += `For components with versioned documentation (like Redpanda Self-Managed), older versions can be accessed by replacing the version segment in the URL:\n`;
|
|
197
|
+
fullContent += `- Latest: \`${siteUrl}/current/page-path\`\n`;
|
|
198
|
+
fullContent += `- Specific version: \`${siteUrl}/24.3/page-path\`, \`${siteUrl}/25.1/page-path\`, etc.\n\n`;
|
|
199
|
+
fullContent += `Available versioned components: ${components.filter(c => c.versions.length > 1).map(c => c.name).join(', ')}\n\n`;
|
|
200
|
+
fullContent += `---\n\n`;
|
|
232
201
|
|
|
233
202
|
// Sort pages by URL for consistent ordering
|
|
234
|
-
|
|
203
|
+
pages.sort((a, b) => {
|
|
235
204
|
const urlA = a.pub?.url || '';
|
|
236
205
|
const urlB = b.pub?.url || '';
|
|
237
206
|
return urlA.localeCompare(urlB);
|
|
238
207
|
});
|
|
239
208
|
|
|
240
|
-
|
|
241
|
-
componentContent += `> This file contains all ${component.title} documentation pages in markdown format for AI agent consumption.\n`;
|
|
242
|
-
componentContent += `> Generated from ${componentPages.length} pages on ${new Date().toISOString()}\n`;
|
|
243
|
-
componentContent += `> Component: ${component.name} | Version: ${latest.version}\n`;
|
|
244
|
-
componentContent += `> Site: ${siteUrl}\n\n`;
|
|
245
|
-
componentContent += `## About This Export\n\n`;
|
|
246
|
-
componentContent += `This export includes the **latest version** (${latest.version}) of the ${component.title} documentation.\n\n`;
|
|
247
|
-
componentContent += `### AI-Friendly Documentation Formats\n\n`;
|
|
248
|
-
componentContent += `We provide multiple formats optimized for AI consumption:\n\n`;
|
|
249
|
-
componentContent += `- **${siteUrl}/llms.txt**: Curated overview of all Redpanda documentation\n`;
|
|
250
|
-
componentContent += `- **${siteUrl}/llms-full.txt**: Complete documentation export with all components\n`;
|
|
251
|
-
componentContent += `- **${siteUrl}/${componentName}-full.txt**: This file - ${component.title} documentation only\n`;
|
|
252
|
-
componentContent += `- **Individual markdown pages**: Each HTML page has a corresponding .md file\n\n`;
|
|
253
|
-
|
|
254
|
-
if (component.versions.length > 1) {
|
|
255
|
-
componentContent += `### Accessing Older Versions\n\n`;
|
|
256
|
-
componentContent += `This component has versioned documentation. Older versions can be accessed by replacing the version segment in the URL:\n`;
|
|
257
|
-
componentContent += `- Latest: \`${siteUrl}/current/page-path\`\n`;
|
|
258
|
-
componentContent += `- Specific version: \`${siteUrl}/24.3/page-path\`, \`${siteUrl}/25.1/page-path\`, etc.\n\n`;
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
componentContent += `---\n\n`;
|
|
262
|
-
|
|
263
|
-
// Add all pages
|
|
264
|
-
componentPages.forEach((page, index) => {
|
|
209
|
+
pages.forEach((page, index) => {
|
|
265
210
|
const mdUrl = page.pub?.url ? toMarkdownUrl(page.pub.url) : '';
|
|
266
211
|
const pageUrl = mdUrl ? `${siteUrl}${mdUrl}` : 'unknown';
|
|
267
212
|
const pageTitle = page.asciidoc?.doctitle || page.src?.stem || 'Untitled';
|
|
268
213
|
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
214
|
+
fullContent += `# Page ${index + 1}: ${pageTitle}\n\n`;
|
|
215
|
+
fullContent += `**URL**: ${pageUrl}\n\n`;
|
|
216
|
+
fullContent += `---\n\n`;
|
|
272
217
|
// Strip metadata (directive, source comments) from page content
|
|
273
|
-
|
|
274
|
-
|
|
218
|
+
fullContent += stripMarkdownMetadata(page.markdownContents);
|
|
219
|
+
fullContent += `\n\n---\n\n`;
|
|
275
220
|
});
|
|
276
221
|
|
|
277
|
-
// Add
|
|
222
|
+
// Add llms-full.txt to site root
|
|
278
223
|
siteCatalog.addFile({
|
|
279
|
-
contents: Buffer.from(
|
|
280
|
-
out: { path:
|
|
224
|
+
contents: Buffer.from(fullContent, 'utf8'),
|
|
225
|
+
out: { path: 'llms-full.txt' },
|
|
226
|
+
});
|
|
227
|
+
logger.info(`Generated llms-full.txt with ${pages.length} pages`);
|
|
228
|
+
|
|
229
|
+
// Generate component-specific full.txt files
|
|
230
|
+
logger.info('Generating component-specific full.txt files...');
|
|
231
|
+
|
|
232
|
+
// Group pages by component
|
|
233
|
+
pages.forEach(page => {
|
|
234
|
+
const componentName = page.src.component;
|
|
235
|
+
if (!componentGroups.has(componentName)) {
|
|
236
|
+
componentGroups.set(componentName, []);
|
|
237
|
+
}
|
|
238
|
+
componentGroups.get(componentName).push(page);
|
|
281
239
|
});
|
|
282
|
-
|
|
240
|
+
|
|
241
|
+
// Generate a full.txt file for each component
|
|
242
|
+
componentGroups.forEach((componentPages, componentName) => {
|
|
243
|
+
const component = components.find(c => c.name === componentName);
|
|
244
|
+
if (!component) return;
|
|
245
|
+
|
|
246
|
+
const latest = component.latest || component.versions[0];
|
|
247
|
+
if (!latest) return;
|
|
248
|
+
|
|
249
|
+
// Sort pages by URL for consistent ordering
|
|
250
|
+
componentPages.sort((a, b) => {
|
|
251
|
+
const urlA = a.pub?.url || '';
|
|
252
|
+
const urlB = b.pub?.url || '';
|
|
253
|
+
return urlA.localeCompare(urlB);
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
let componentContent = `# ${component.title} - Full Markdown Export\n\n`;
|
|
257
|
+
componentContent += `> This file contains all ${component.title} documentation pages in markdown format for AI agent consumption.\n`;
|
|
258
|
+
componentContent += `> Generated from ${componentPages.length} pages on ${new Date().toISOString()}\n`;
|
|
259
|
+
componentContent += `> Component: ${component.name} | Version: ${latest.version}\n`;
|
|
260
|
+
componentContent += `> Site: ${siteUrl}\n\n`;
|
|
261
|
+
componentContent += `## About This Export\n\n`;
|
|
262
|
+
componentContent += `This export includes the **latest version** (${latest.version}) of the ${component.title} documentation.\n\n`;
|
|
263
|
+
componentContent += `### AI-Friendly Documentation Formats\n\n`;
|
|
264
|
+
componentContent += `We provide multiple formats optimized for AI consumption:\n\n`;
|
|
265
|
+
componentContent += `- **${siteUrl}/llms.txt**: Curated overview of all Redpanda documentation\n`;
|
|
266
|
+
componentContent += `- **${siteUrl}/llms-full.txt**: Complete documentation export with all components\n`;
|
|
267
|
+
componentContent += `- **${siteUrl}/${componentName}-full.txt**: This file - ${component.title} documentation only\n`;
|
|
268
|
+
componentContent += `- **Individual markdown pages**: Each HTML page has a corresponding .md file\n\n`;
|
|
269
|
+
|
|
270
|
+
if (component.versions.length > 1) {
|
|
271
|
+
componentContent += `### Accessing Older Versions\n\n`;
|
|
272
|
+
componentContent += `This component has versioned documentation. Older versions can be accessed by replacing the version segment in the URL:\n`;
|
|
273
|
+
componentContent += `- Latest: \`${siteUrl}/current/page-path\`\n`;
|
|
274
|
+
componentContent += `- Specific version: \`${siteUrl}/24.3/page-path\`, \`${siteUrl}/25.1/page-path\`, etc.\n\n`;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
componentContent += `---\n\n`;
|
|
278
|
+
|
|
279
|
+
// Add all pages
|
|
280
|
+
componentPages.forEach((page, index) => {
|
|
281
|
+
const mdUrl = page.pub?.url ? toMarkdownUrl(page.pub.url) : '';
|
|
282
|
+
const pageUrl = mdUrl ? `${siteUrl}${mdUrl}` : 'unknown';
|
|
283
|
+
const pageTitle = page.asciidoc?.doctitle || page.src?.stem || 'Untitled';
|
|
284
|
+
|
|
285
|
+
componentContent += `# Page ${index + 1}: ${pageTitle}\n\n`;
|
|
286
|
+
componentContent += `**URL**: ${pageUrl}\n\n`;
|
|
287
|
+
componentContent += `---\n\n`;
|
|
288
|
+
// Strip metadata (directive, source comments) from page content
|
|
289
|
+
componentContent += stripMarkdownMetadata(page.markdownContents);
|
|
290
|
+
componentContent += `\n\n---\n\n`;
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
// Add component-specific full.txt file to site root
|
|
294
|
+
siteCatalog.addFile({
|
|
295
|
+
contents: Buffer.from(componentContent, 'utf8'),
|
|
296
|
+
out: { path: `${componentName}-full.txt` },
|
|
297
|
+
});
|
|
298
|
+
logger.info(`Generated ${componentName}-full.txt with ${componentPages.length} pages`);
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
// Generate page index files for every published version (not only the latest),
|
|
303
|
+
// so the llms.txt tree lists every page the sitemap lists.
|
|
304
|
+
const pageIndexes = buildPageIndexes({
|
|
305
|
+
pages: allPages,
|
|
306
|
+
components,
|
|
307
|
+
siteUrl,
|
|
308
|
+
toMarkdownUrl,
|
|
283
309
|
});
|
|
310
|
+
pageIndexes.forEach((index) => {
|
|
311
|
+
siteCatalog.addFile({
|
|
312
|
+
contents: Buffer.from(index.contents, 'utf8'),
|
|
313
|
+
out: { path: index.path },
|
|
314
|
+
});
|
|
315
|
+
});
|
|
316
|
+
const indexedPageCount = pageIndexes.reduce((n, index) => n + index.pageCount, 0);
|
|
317
|
+
logger.info(`Generated ${pageIndexes.length} page index files covering ${indexedPageCount} pages`);
|
|
284
318
|
|
|
285
319
|
// Add llms.txt to site root (using content extracted earlier)
|
|
286
320
|
if (llmsPage && llmsPage.llmsTxtContent) {
|
|
@@ -297,6 +331,15 @@ module.exports.register = function () {
|
|
|
297
331
|
llmsTxtContent = truncateAtNewline(llmsTxtContent, MAX_LLMS_TXT_CHARS - 100) + '\n\n[Content truncated due to size limits]';
|
|
298
332
|
}
|
|
299
333
|
|
|
334
|
+
// The page index list is never truncated: an index missing from the root
|
|
335
|
+
// leaves its pages unreachable from llms.txt.
|
|
336
|
+
if (pageIndexes.length) {
|
|
337
|
+
llmsTxtContent = llmsTxtContent + '\n\n' + renderPageIndexSection(pageIndexes, components);
|
|
338
|
+
if (llmsTxtContent.length >= MAX_LLMS_TXT_CHARS) {
|
|
339
|
+
logger.warn(`llms.txt with page index list is ${llmsTxtContent.length} chars, over the ${MAX_LLMS_TXT_CHARS} char target. Shorten the llms page or raise the per-index size limit.`);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
300
343
|
// Generate navigation section with component sitemaps and key sections
|
|
301
344
|
const navSection = generateNavigationSection(siteUrl, contentCatalog, components);
|
|
302
345
|
|
|
@@ -350,6 +393,10 @@ module.exports.register = function () {
|
|
|
350
393
|
}
|
|
351
394
|
});
|
|
352
395
|
|
|
396
|
+
pageIndexes.forEach((index) => {
|
|
397
|
+
if (index.lastModified) gitDates.set(index.path, index.lastModified);
|
|
398
|
+
});
|
|
399
|
+
|
|
353
400
|
addToSitemap(contentCatalog, siteCatalog, siteUrl, gitDates, logger);
|
|
354
401
|
} catch (err) {
|
|
355
402
|
logger.warn(`Failed to add llms.txt to sitemap: ${err.message}`);
|
|
@@ -398,7 +445,8 @@ function addToSitemap(contentCatalog, siteCatalog, siteUrl, gitDates, logger) {
|
|
|
398
445
|
const filename = file.out.path;
|
|
399
446
|
return filename === 'llms.txt' ||
|
|
400
447
|
filename === 'llms-full.txt' ||
|
|
401
|
-
filename.endsWith('-full.txt')
|
|
448
|
+
filename.endsWith('-full.txt') ||
|
|
449
|
+
/(^|\/)llms(-\d+)?\.txt$/.test(filename);
|
|
402
450
|
})
|
|
403
451
|
.map(file => file.out.path)
|
|
404
452
|
.sort(); // Sort for consistent ordering
|
package/package.json
CHANGED
|
@@ -116,7 +116,16 @@ function lintStrings (options) {
|
|
|
116
116
|
// still exists at HEAD, in any touched file, was edited or moved, not
|
|
117
117
|
// removed.
|
|
118
118
|
const removedFiles = removedBySurface[surfaceName] ? [...removedBySurface[surfaceName].keys()] : []
|
|
119
|
-
const
|
|
119
|
+
const touchedFiles = new Set([...files.keys(), ...removedFiles])
|
|
120
|
+
// A surface whose extraction covers the whole surface anyway (the
|
|
121
|
+
// properties extractor parses every config file on each run) supplies
|
|
122
|
+
// context from all of it, so a property in an untouched file that
|
|
123
|
+
// names, or is named by, a changed one is still on its page. Review
|
|
124
|
+
// scope stays the touched files either way.
|
|
125
|
+
const everything = surface.contextScope === 'surface' ? surface.extract({ repo: repoPath, log }) : null
|
|
126
|
+
const head = everything
|
|
127
|
+
? everything.filter((d) => touchedFiles.has(d.file))
|
|
128
|
+
: surface.extract({ repo: repoPath, files: touchedFiles, log })
|
|
120
129
|
headBySurface[surfaceName] = head
|
|
121
130
|
const declarations = []
|
|
122
131
|
for (const decl of head) {
|
|
@@ -128,7 +137,7 @@ function lintStrings (options) {
|
|
|
128
137
|
continue
|
|
129
138
|
}
|
|
130
139
|
declarations.push(decl)
|
|
131
|
-
pending.push(pendingEntry(decl))
|
|
140
|
+
pending.push({ ...pendingEntry(decl), context: pageContext(decl, everything || head) })
|
|
132
141
|
}
|
|
133
142
|
results.push(runRules(declarations, rulesFor(surface), { skipRules, onlyRules }))
|
|
134
143
|
}
|
|
@@ -186,6 +195,59 @@ function touches (decl, lineSet) {
|
|
|
186
195
|
return spansOf(decl).some(([start, end]) => spanIntersects(start, end, lineSet))
|
|
187
196
|
}
|
|
188
197
|
|
|
198
|
+
const CONTEXT_LIMIT = 15
|
|
199
|
+
const CONTEXT_STRING_LIMIT = 600
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* The other strings a reader sees alongside this one, so a review can judge
|
|
203
|
+
* a string against its page rather than on its own: a flag's usage next to
|
|
204
|
+
* its command's Long text and examples, a property that names another
|
|
205
|
+
* property, the fields around an API field.
|
|
206
|
+
*
|
|
207
|
+
* - rpk: every other string declared in the same file, which is the
|
|
208
|
+
* command's Short, Long and flags (rpk keeps one command per file).
|
|
209
|
+
* - api: the declarations nearest by line, which are the enclosing message
|
|
210
|
+
* or rpc and its neighbors.
|
|
211
|
+
* - properties and metrics: declarations whose string names this one, or
|
|
212
|
+
* that this string names, plus (metrics) its neighbors in the same group.
|
|
213
|
+
*
|
|
214
|
+
* Drawn from declarations already extracted for the diff, so it costs no
|
|
215
|
+
* extra extraction. Deterministic, capped, and order-stable.
|
|
216
|
+
*/
|
|
217
|
+
function pageContext (decl, all) {
|
|
218
|
+
const others = all.filter((d) => d !== decl && d.string != null)
|
|
219
|
+
const byDistance = (d) => Math.abs((d.line_start || 0) - (decl.line_start || 0))
|
|
220
|
+
const sameFile = others.filter((d) => d.file === decl.file)
|
|
221
|
+
let picked
|
|
222
|
+
if (decl.surface === 'rpk') {
|
|
223
|
+
// The command's Short and Long always, then the flags nearest this
|
|
224
|
+
// declaration, so a changed flag late in a long command keeps its
|
|
225
|
+
// neighbors when the cap applies. Shown in source order.
|
|
226
|
+
const isText = (d) => d.meta && (d.meta.kind === 'short' || d.meta.kind === 'long')
|
|
227
|
+
const text = sameFile.filter(isText)
|
|
228
|
+
const flags = sameFile.filter((d) => !isText(d)).sort((a, b) => byDistance(a) - byDistance(b))
|
|
229
|
+
picked = [...text, ...flags].slice(0, CONTEXT_LIMIT)
|
|
230
|
+
.sort((a, b) => (a.line_start || 0) - (b.line_start || 0))
|
|
231
|
+
} else if (decl.surface === 'api') {
|
|
232
|
+
picked = sameFile.sort((a, b) => byDistance(a) - byDistance(b)).slice(0, 6)
|
|
233
|
+
} else {
|
|
234
|
+
const mentions = (text, name) => Boolean(text && name) &&
|
|
235
|
+
new RegExp(`(^|[^A-Za-z0-9_])${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}($|[^A-Za-z0-9_])`).test(text)
|
|
236
|
+
const related = others.filter((d) => mentions(d.string, decl.name) || mentions(decl.string, d.name))
|
|
237
|
+
const neighbors = decl.surface === 'metrics'
|
|
238
|
+
? sameFile.filter((d) => !related.includes(d)).sort((a, b) => byDistance(a) - byDistance(b)).slice(0, 2)
|
|
239
|
+
: []
|
|
240
|
+
picked = [...related, ...neighbors]
|
|
241
|
+
}
|
|
242
|
+
return picked.slice(0, CONTEXT_LIMIT).map((d) => ({
|
|
243
|
+
name: d.name,
|
|
244
|
+
kind: (d.meta && d.meta.kind) || d.surface,
|
|
245
|
+
file: d.file,
|
|
246
|
+
line_start: d.line_start,
|
|
247
|
+
string: d.string.length > CONTEXT_STRING_LIMIT ? `${d.string.slice(0, CONTEXT_STRING_LIMIT)}...` : d.string
|
|
248
|
+
}))
|
|
249
|
+
}
|
|
250
|
+
|
|
189
251
|
/**
|
|
190
252
|
* Stable identity for one version of one doc string. A PR review records the
|
|
191
253
|
* fingerprints it has seen, and a later push skips any declaration whose
|
|
@@ -467,7 +529,7 @@ function runCli (options) {
|
|
|
467
529
|
process.exitCode = options.strict && result.summary.errors > 0 ? 1 : 0
|
|
468
530
|
}
|
|
469
531
|
|
|
470
|
-
module.exports = { lintStrings, formatHuman, runCli, SURFACES, rulesFor, fingerprint, readFingerprints }
|
|
532
|
+
module.exports = { lintStrings, formatHuman, runCli, SURFACES, rulesFor, fingerprint, readFingerprints, pageContext }
|
|
471
533
|
|
|
472
534
|
// Direct usage: node tools/lint-strings --repo <path> [--surface a,b]
|
|
473
535
|
// [--diff <base>] [--format json|human] [--strict]
|
|
@@ -100,7 +100,9 @@ function maskNonProse (text) {
|
|
|
100
100
|
const CODE_TOKEN_PATTERNS = Object.freeze([
|
|
101
101
|
{ id: 'identifier', re: /\b[a-z][a-z0-9]*(?:_[a-z0-9]+)+\b/g },
|
|
102
102
|
{ id: 'flag', re: /(?<![\w-])--[a-z][a-z0-9]*(?:-[a-z0-9]+)*\b/g },
|
|
103
|
-
|
|
103
|
+
// A path may not end in a period, so "served from /v1/brokers." reports
|
|
104
|
+
// `/v1/brokers`, not the sentence's full stop with it.
|
|
105
|
+
{ id: 'path', re: /(?<![\w/])\/[a-z][a-zA-Z0-9_.-]*(?:\/[a-zA-Z0-9_.{}-]*[a-zA-Z0-9_{}-])+\/?/g }
|
|
104
106
|
])
|
|
105
107
|
|
|
106
108
|
/**
|
|
@@ -276,7 +278,12 @@ const COMMON_RULES = [
|
|
|
276
278
|
description: 'Code value or field name in prose without inline code',
|
|
277
279
|
severity: 'warning',
|
|
278
280
|
check: (decl) => {
|
|
279
|
-
|
|
281
|
+
// A surface whose generator formats some code values itself declares
|
|
282
|
+
// which (convention.auto_inline_code); those are not findings there.
|
|
283
|
+
const auto = (decl.convention && decl.convention.auto_inline_code) || null
|
|
284
|
+
const bare = findBareCodeTokens(decl.string, decl.name).filter((b) => !auto ||
|
|
285
|
+
!((auto.kinds || []).includes(b.kind) ||
|
|
286
|
+
(b.kind === 'path' && (auto.path_prefixes || []).some((prefix) => b.token.startsWith(prefix)))))
|
|
280
287
|
if (bare.length === 0) return []
|
|
281
288
|
const list = bare.map((b) => `\`${b.token}\``).join(', ')
|
|
282
289
|
return [{
|
|
@@ -245,6 +245,9 @@ module.exports = {
|
|
|
245
245
|
// The extractor pairs each .cc with its .h, so a merge-base extraction needs
|
|
246
246
|
// the whole directory, not only the files that lost lines.
|
|
247
247
|
baseScope: 'directory',
|
|
248
|
+
// extract() runs the extractor over the whole repo and filters after, so
|
|
249
|
+
// page context can use every property at no extra cost.
|
|
250
|
+
contextScope: 'surface',
|
|
248
251
|
convention: CONVENTION,
|
|
249
252
|
extract,
|
|
250
253
|
mapExtractorJson,
|
|
@@ -37,7 +37,15 @@ const CONVENTION = {
|
|
|
37
37
|
case: 'sentence',
|
|
38
38
|
terminal_period: false,
|
|
39
39
|
verbatim_asciidoc: false,
|
|
40
|
-
transformer: 'formatDescription'
|
|
40
|
+
transformer: 'formatDescription',
|
|
41
|
+
// Code values formatDescription wraps in inline code by itself, so a bare
|
|
42
|
+
// one in the source is not a finding. Long is a Go raw string and cannot
|
|
43
|
+
// hold a backtick at all, which is why the generator does this. Keep in
|
|
44
|
+
// step with the auto-backtick passes in tools/rpk-docs/generate-rpk-docs.js.
|
|
45
|
+
auto_inline_code: {
|
|
46
|
+
kinds: ['flag'],
|
|
47
|
+
path_prefixes: ['/etc/', '/var/', '/usr/', '/home/', '/tmp/', '~/.']
|
|
48
|
+
}
|
|
41
49
|
}
|
|
42
50
|
|
|
43
51
|
// Composite literal only: gofmt writes `cobra.Command{` with no space, while
|
|
@@ -1231,7 +1231,12 @@ breaks. Two files feed it, both mirrors of the `docs` repo:
|
|
|
1231
1231
|
fields the rendering and link layers read. `cloud_supported` is the point: it is what
|
|
1232
1232
|
makes the "no Cloud link to a property Cloud does not publish" assertion real rather
|
|
1233
1233
|
than assumed. Descriptions are deliberately absent, because the corpus test drives prose from
|
|
1234
|
-
the overrides file, and a full copy would be 695 KB of text that rots.
|
|
1234
|
+
the overrides file, and a full copy would be 695 KB of text that rots. The exceptions are
|
|
1235
|
+
overrides that act on the source description instead of replacing it: links with no prose of
|
|
1236
|
+
their own, which apply to the source description, and includes with no description of their
|
|
1237
|
+
own, which the generator appends to it. The snapshot keeps the source description for those
|
|
1238
|
+
properties. `.github/scripts/derive-property-snapshot.js` makes that call from the overrides
|
|
1239
|
+
file, and both the recipe below and `property-corpus-drift.sh` use it.
|
|
1235
1240
|
|
|
1236
1241
|
Keep the overrides copy a faithful mirror. It previously doubled as
|
|
1237
1242
|
`property-docs-overrides.test.js`'s fixture, someone hand-edited an `admin` entry into
|
|
@@ -1264,17 +1269,13 @@ TAG="$(git -C "$DOCS" ls-tree origin/main --name-only -- modules/reference/attac
|
|
|
1264
1269
|
echo "Refreshing against $TAG"
|
|
1265
1270
|
|
|
1266
1271
|
git -C "$DOCS" show "origin/main:modules/reference/attachments/redpanda-properties-${TAG}.json" \
|
|
1272
|
+
> /tmp/redpanda-properties.json
|
|
1273
|
+
node .github/scripts/derive-property-snapshot.js /tmp/redpanda-properties.json \
|
|
1274
|
+
__tests__/docs-data/property-overrides.json \
|
|
1267
1275
|
| node -e '
|
|
1268
1276
|
const fs = require("fs");
|
|
1269
|
-
const src = JSON.parse(fs.readFileSync(0, "utf8"));
|
|
1270
|
-
const keep = ["name","config_scope","type","cloud_supported","cloud_editable",
|
|
1271
|
-
"cloud_readonly","cloud_byoc_only","is_deprecated","nullable"];
|
|
1272
|
-
const properties = {};
|
|
1273
|
-
for (const [k, v] of Object.entries(src.properties || {})) {
|
|
1274
|
-
properties[k] = Object.fromEntries(keep.filter((f) => f in v).map((f) => [f, v[f]]));
|
|
1275
|
-
}
|
|
1276
1277
|
const out = JSON.parse(fs.readFileSync("__tests__/docs-data/property-snapshot.json", "utf8"));
|
|
1277
|
-
out.properties =
|
|
1278
|
+
out.properties = JSON.parse(fs.readFileSync(0, "utf8"));
|
|
1278
1279
|
out.source_tag = process.argv[1];
|
|
1279
1280
|
fs.writeFileSync("__tests__/docs-data/property-snapshot.json", JSON.stringify(out, null, 1) + "\n");
|
|
1280
1281
|
' "$TAG"
|
|
@@ -1569,6 +1569,26 @@ function formatDescription(desc, customTransformations = null, options = {}) {
|
|
|
1569
1569
|
return placeholder
|
|
1570
1570
|
})
|
|
1571
1571
|
|
|
1572
|
+
// Bare URLs become inline code: endpoints (http://localhost:9644) and links
|
|
1573
|
+
// alike are code values, and an unformatted URL in help prose reads as
|
|
1574
|
+
// broken markup. Not trailing sentence punctuation, and not a URL that is
|
|
1575
|
+
// already an AsciiDoc link macro (url[text]), which backticks would break.
|
|
1576
|
+
//
|
|
1577
|
+
// Protected as a placeholder straight away, before any other pass runs.
|
|
1578
|
+
// Every later pass matches inside a URL it does not know is one: the path
|
|
1579
|
+
// passes backticked /tmp/file out of https://host/tmp/file, the short-flag
|
|
1580
|
+
// pass took -a out of /foo-a/, and the issue-reference pass rewrote #NNNN
|
|
1581
|
+
// anchors into link macros.
|
|
1582
|
+
protectedDesc = protectedDesc.replace(/(?<![`\w/]|\]\()https?:\/\/[^\s`<>"'()[\]]+/g, (url, offset, whole) => {
|
|
1583
|
+
if (whole[offset + url.length] === '[') return url // AsciiDoc link macro
|
|
1584
|
+
const trail = url.match(/[.,;:!?]+$/)
|
|
1585
|
+
const core = trail ? url.slice(0, -trail[0].length) : url
|
|
1586
|
+
if (core.length <= 'https://'.length) return url
|
|
1587
|
+
const placeholder = `__INLINE_CODE_${inlineCode.length}__`
|
|
1588
|
+
inlineCode.push(`\`${decodeHtmlEntities(core)}\``)
|
|
1589
|
+
return placeholder + (trail ? trail[0] : '')
|
|
1590
|
+
})
|
|
1591
|
+
|
|
1572
1592
|
// Known top-level rpk subcommands (for accurate command detection)
|
|
1573
1593
|
const rpkSubcommands = new Set([
|
|
1574
1594
|
'ai', 'check', 'cloud', 'cluster', 'connect', 'container', 'debug',
|
|
@@ -1713,6 +1733,17 @@ function formatDescription(desc, customTransformations = null, options = {}) {
|
|
|
1713
1733
|
}).join('`')
|
|
1714
1734
|
result = result.replace(/(?<!`|-)(-[a-zA-Z])(?![a-zA-Z-])/g, '`$1`')
|
|
1715
1735
|
|
|
1736
|
+
// A bare command followed by its flags ("run rpk topic list -r") is one
|
|
1737
|
+
// command, but the passes above wrap the command path and each flag
|
|
1738
|
+
// separately. Merge flag spans into the command span before them:
|
|
1739
|
+
// `rpk topic list` `-r` -> `rpk topic list -r`. The quoted form never
|
|
1740
|
+
// splits, because its whole span is protected before flags are wrapped.
|
|
1741
|
+
let merged
|
|
1742
|
+
do {
|
|
1743
|
+
merged = result
|
|
1744
|
+
result = result.replace(/`(rpk(?: [a-z][-a-z0-9]*)+(?: -{1,2}[a-zA-Z][-a-zA-Z0-9]*)*)` `(-{1,2}[a-zA-Z][-a-zA-Z0-9]*)`/g, '`$1 $2`')
|
|
1745
|
+
} while (result !== merged)
|
|
1746
|
+
|
|
1716
1747
|
// Add backticks around environment variables
|
|
1717
1748
|
result = result.replace(/(?<!`)(\$[A-Z_][A-Z0-9_]*)/g, '`$1`')
|
|
1718
1749
|
|
|
@@ -1729,12 +1760,15 @@ function formatDescription(desc, customTransformations = null, options = {}) {
|
|
|
1729
1760
|
result = result.replace(/`(\$[A-Z_][A-Z0-9_]*)`(\/(?:[^\s`]*[^\s`).,;:!?])?)/g, '`$1$2`')
|
|
1730
1761
|
|
|
1731
1762
|
// Add backticks around file paths (but not if already backticked)
|
|
1732
|
-
// Must check both the slash and the path aren't already inside backticks
|
|
1733
|
-
|
|
1734
|
-
|
|
1763
|
+
// Must check both the slash and the path aren't already inside backticks.
|
|
1764
|
+
// Like the $HOME merge above, a path may not END in sentence punctuation,
|
|
1765
|
+
// so "...in ~/.config/rpk/rpk.yaml." puts the period outside the span
|
|
1766
|
+
// instead of publishing `~/.config/rpk/rpk.yaml.`.
|
|
1767
|
+
result = result.replace(/(?<![`/])(\/(?:etc|var|usr|home|tmp)\/(?:[^\s,)`]*[^\s,)`.;:!?])?)/g, '`$1`')
|
|
1768
|
+
result = result.replace(/(?<![`/])((?:etc|var|usr|home|tmp)\/(?:[^\s,)`]*[^\s,)`.;:!?])?)/g, '`$1`')
|
|
1735
1769
|
|
|
1736
1770
|
// Add backticks around home directory paths (~/.bashrc, ~/.zshrc, ~/.config/...)
|
|
1737
|
-
result = result.replace(/(?<![`\w])(~\/\.[^\s,;:)`]
|
|
1771
|
+
result = result.replace(/(?<![`\w])(~\/\.(?:[^\s,;:)`]*[^\s,;:)`.!?])?)/g, '`$1`')
|
|
1738
1772
|
|
|
1739
1773
|
// Add backticks around common package names
|
|
1740
1774
|
result = result.replace(/(?<![`\w-])(bash-completion)(?![`\w-])/g, '`$1`')
|