@redpanda-data/docs-extensions-and-macros 5.13.0 → 5.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/doc-tools.js +39 -0
- package/package.json +1 -1
- package/tools/generate-cli-docs.js +15 -4
- package/tools/redpanda-connect/generate-rpcn-connector-docs.js +275 -33
- package/tools/redpanda-connect/helpers/descriptionInclude.js +24 -0
- package/tools/redpanda-connect/helpers/index.js +3 -0
- package/tools/redpanda-connect/helpers/renderConnectDescription.js +125 -0
- package/tools/redpanda-connect/helpers/summaryAttribute.js +22 -0
- package/tools/redpanda-connect/metadata-utils.js +183 -35
- package/tools/redpanda-connect/migrate-descriptions-to-partials.js +275 -0
- package/tools/redpanda-connect/normalize-metadata.js +6 -4
- package/tools/redpanda-connect/pr-summary-formatter.js +51 -0
- package/tools/redpanda-connect/rpcn-connector-docs-handler.js +37 -7
- package/tools/redpanda-connect/templates/connector.hbs +17 -4
- package/tools/redpanda-connect/templates/descriptions-partials.hbs +21 -0
- package/tools/redpanda-connect/templates/intro.hbs +1 -1
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* One-time migration that makes published connector pages consume the
|
|
5
|
+
* regenerated description partial.
|
|
6
|
+
*
|
|
7
|
+
* Connector reference pages are one-time first drafts: the summary, the
|
|
8
|
+
* "Introduced in version" line and the description prose were frozen into
|
|
9
|
+
* `modules/components/pages/<type>/<name>.adoc` when the page was drafted, and
|
|
10
|
+
* nothing has refreshed them since. The generator now writes the same content
|
|
11
|
+
* to `modules/components/partials/descriptions/<type>/<name>.adoc` on every
|
|
12
|
+
* run, with two tag regions. This script rewires existing pages to it:
|
|
13
|
+
*
|
|
14
|
+
* header: include::...[tag=attrs] inside the page's `// tag::meta[]` region
|
|
15
|
+
* body: include::...[tag=body] where the frozen prose is today
|
|
16
|
+
*
|
|
17
|
+
* The header rewire is mechanical and safe: it swaps one `:description:` line
|
|
18
|
+
* for an include that sets the same attribute from the partial.
|
|
19
|
+
*
|
|
20
|
+
* The body rewire is a rewrite of published prose, so it is guarded. The page's
|
|
21
|
+
* intro region is only replaced when every non-blank line of it also appears in
|
|
22
|
+
* the partial's body region. Pages whose intro carries anything the partial does
|
|
23
|
+
* not reproduce (hand-added prose, a hand-written config listing, an extra
|
|
24
|
+
* admonition) are reported and left alone for a human, because migrating them
|
|
25
|
+
* would silently delete published content.
|
|
26
|
+
*
|
|
27
|
+
* Usage (run from the docs repo root, after a generator run so the partials
|
|
28
|
+
* exist):
|
|
29
|
+
* npx doc-tools generate migrate-rpcn-descriptions # dry run
|
|
30
|
+
* npx doc-tools generate migrate-rpcn-descriptions --write # apply
|
|
31
|
+
* npx doc-tools generate migrate-rpcn-descriptions --skip-body # headers only
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
const fs = require('fs');
|
|
35
|
+
const path = require('path');
|
|
36
|
+
const { descriptionIncludeLine, normalizeTypeDir, flattenToAttributeValue } = require('./metadata-utils.js');
|
|
37
|
+
|
|
38
|
+
// The page's meta tag region: the block single-source stubs inherit
|
|
39
|
+
// :description: from, so the attrs include belongs inside it.
|
|
40
|
+
const META_TAG_OPEN = /^\/\/\s*tag::meta\[\]\s*$/;
|
|
41
|
+
const META_TAG_CLOSE = /^\/\/\s*end::meta\[\]\s*$/;
|
|
42
|
+
|
|
43
|
+
// The intro region starts after the component-type dropdown macro, which every
|
|
44
|
+
// connector page carries directly below its header.
|
|
45
|
+
const INTRO_START = /^component_type_dropdown::\[\]\s*$/;
|
|
46
|
+
|
|
47
|
+
// ...and ends at the first structural element after it. `include::` covers the
|
|
48
|
+
// metadata, fields and examples partials; `[tabs]` covers the generated config
|
|
49
|
+
// listing; `== ` covers the first real section. Fenced code blocks are
|
|
50
|
+
// deliberately NOT terminators: a hand-written config listing inside the intro
|
|
51
|
+
// has to stay inside the region so the content check below sees it and refuses.
|
|
52
|
+
const INTRO_END = /^(?:==\s+\S|include::|\[tabs\]|\/\/\s*end::single-source)/;
|
|
53
|
+
|
|
54
|
+
/** Index of the intro anchor, or the end of the file when there is none. */
|
|
55
|
+
function introOf (lines) {
|
|
56
|
+
const at = lines.findIndex((l) => INTRO_START.test(l));
|
|
57
|
+
return at === -1 ? lines.length : at;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Extract the lines of one tag region from a partial. */
|
|
61
|
+
function tagRegion (partial, tag) {
|
|
62
|
+
const lines = partial.split('\n');
|
|
63
|
+
const open = lines.findIndex((l) => new RegExp(`^//\\s*tag::${tag}\\[\\]\\s*$`).test(l));
|
|
64
|
+
if (open === -1) return null;
|
|
65
|
+
const close = lines.findIndex((l, i) => i > open && new RegExp(`^//\\s*end::${tag}\\[\\]\\s*$`).test(l));
|
|
66
|
+
if (close === -1) return null;
|
|
67
|
+
return lines.slice(open + 1, close);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Lines that carry content: blanks and AsciiDoc comments do not count. */
|
|
71
|
+
function contentLines (lines) {
|
|
72
|
+
return lines.map((l) => l.trim()).filter((l) => l !== '' && !l.startsWith('//'));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* True when the partial's body region reproduces every content line of the
|
|
77
|
+
* page's intro region. This is the guard that keeps the migration from
|
|
78
|
+
* deleting published prose the generator does not know about.
|
|
79
|
+
*/
|
|
80
|
+
function bodyCoversIntro (introLines, bodyLines) {
|
|
81
|
+
const body = new Set(contentLines(bodyLines));
|
|
82
|
+
return contentLines(introLines).every((l) => body.has(l));
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Migrate published connector pages onto the regenerated description partial.
|
|
87
|
+
* @param {object} [options]
|
|
88
|
+
* @param {boolean} [options.write=false] Apply changes (otherwise dry run).
|
|
89
|
+
* @param {boolean} [options.skipBody=false] Only rewire page headers.
|
|
90
|
+
* @param {string} [options.pagesRoot]
|
|
91
|
+
* @param {string} [options.partialsRoot]
|
|
92
|
+
* @returns {{headers:string[], bodies:string[], skipped:Array<{page:string, reason:string}>}}
|
|
93
|
+
*/
|
|
94
|
+
function migrateDescriptionsToPartials ({
|
|
95
|
+
write = false,
|
|
96
|
+
skipBody = false,
|
|
97
|
+
pagesRoot,
|
|
98
|
+
partialsRoot,
|
|
99
|
+
} = {}) {
|
|
100
|
+
const pages = pagesRoot || path.resolve(process.cwd(), 'modules/components/pages');
|
|
101
|
+
const partials = partialsRoot || path.resolve(process.cwd(), 'modules/components/partials/descriptions');
|
|
102
|
+
const results = { headers: [], bodies: [], skipped: [] };
|
|
103
|
+
if (!fs.existsSync(pages)) return results;
|
|
104
|
+
|
|
105
|
+
for (const typeDirName of fs.readdirSync(pages).sort()) {
|
|
106
|
+
const typeDirPath = path.join(pages, typeDirName);
|
|
107
|
+
if (!fs.statSync(typeDirPath).isDirectory()) continue;
|
|
108
|
+
// The pages directory and the partials directory can spell a component
|
|
109
|
+
// family differently (pages/rate_limits vs the rate-limits data key), so
|
|
110
|
+
// resolve the partial through the same normalizer the generator writes
|
|
111
|
+
// with rather than by string equality.
|
|
112
|
+
const partialTypeDir = normalizeTypeDir(typeDirName);
|
|
113
|
+
|
|
114
|
+
for (const file of fs.readdirSync(typeDirPath).sort()) {
|
|
115
|
+
if (!file.endsWith('.adoc')) continue;
|
|
116
|
+
const name = file.slice(0, -'.adoc'.length);
|
|
117
|
+
const key = `${typeDirName}/${name}`;
|
|
118
|
+
const pagePath = path.join(typeDirPath, file);
|
|
119
|
+
const partialPath = path.join(partials, partialTypeDir, file);
|
|
120
|
+
|
|
121
|
+
if (!fs.existsSync(partialPath)) {
|
|
122
|
+
results.skipped.push({ page: key, reason: 'no generated description partial' });
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
const partial = fs.readFileSync(partialPath, 'utf8');
|
|
126
|
+
const attrs = tagRegion(partial, 'attrs');
|
|
127
|
+
const body = tagRegion(partial, 'body');
|
|
128
|
+
if (!attrs || !body || contentLines(body).length === 0) {
|
|
129
|
+
// A blanked partial (description removed upstream) has both regions
|
|
130
|
+
// but no content. Wiring a page to it would publish an empty page.
|
|
131
|
+
results.skipped.push({ page: key, reason: 'description partial is empty' });
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const includeAttrs = descriptionIncludeLine({ typeDir: partialTypeDir, name }, 'attrs');
|
|
136
|
+
const includeBody = descriptionIncludeLine({ typeDir: partialTypeDir, name }, 'body');
|
|
137
|
+
|
|
138
|
+
let lines = fs.readFileSync(pagePath, 'utf8').split('\n');
|
|
139
|
+
let changed = false;
|
|
140
|
+
|
|
141
|
+
// --- header ---------------------------------------------------------
|
|
142
|
+
if (!lines.some((l) => l.trim() === includeAttrs)) {
|
|
143
|
+
const open = lines.findIndex((l) => META_TAG_OPEN.test(l));
|
|
144
|
+
const close = lines.findIndex((l, i) => i > open && META_TAG_CLOSE.test(l));
|
|
145
|
+
const attrValue = contentLines(attrs)
|
|
146
|
+
.find((l) => l.startsWith(':description:'));
|
|
147
|
+
const pageAttr = lines.slice(0, introOf(lines)).find((l) => l.startsWith(':description:'));
|
|
148
|
+
if (!attrValue) {
|
|
149
|
+
// The connector has no summary, so the attrs region sets nothing.
|
|
150
|
+
// Replacing a hand-written :description: with it would drop the
|
|
151
|
+
// page's meta description.
|
|
152
|
+
results.skipped.push({ page: key, reason: 'attrs region sets no :description:' });
|
|
153
|
+
} else if (pageAttr && flattenToAttributeValue(pageAttr) !== flattenToAttributeValue(attrValue)) {
|
|
154
|
+
// The page's meta description is not the connector summary, so it
|
|
155
|
+
// was authored by hand (often tuned for search) and swapping in the
|
|
156
|
+
// include would silently discard that editorial work. 13 pages in
|
|
157
|
+
// rp-connect-docs are in this state. A human decides these: either
|
|
158
|
+
// move the wording upstream into the summary, or leave the page
|
|
159
|
+
// header alone.
|
|
160
|
+
results.skipped.push({ page: key, reason: 'hand-authored :description: differs from the summary (needs a human)' });
|
|
161
|
+
} else if (open !== -1 && close > open) {
|
|
162
|
+
lines.splice(open + 1, close - open - 1, includeAttrs);
|
|
163
|
+
results.headers.push(key);
|
|
164
|
+
changed = true;
|
|
165
|
+
} else {
|
|
166
|
+
// Header = everything above the dropdown macro, not "up to the
|
|
167
|
+
// first blank line": a handful of pages carry a blank line inside
|
|
168
|
+
// their attribute list, which is how they ended up with two
|
|
169
|
+
// :description: lines in the first place.
|
|
170
|
+
const attrLine = lines.findIndex((l) => l.startsWith(':description:'));
|
|
171
|
+
if (attrLine !== -1 && attrLine < introOf(lines)) {
|
|
172
|
+
lines.splice(attrLine, 1, '// tag::meta[]', includeAttrs, '// end::meta[]');
|
|
173
|
+
results.headers.push(key);
|
|
174
|
+
changed = true;
|
|
175
|
+
} else {
|
|
176
|
+
results.skipped.push({ page: key, reason: 'no header :description: to replace' });
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// A handful of pages carry a blank line inside their header, so the
|
|
182
|
+
// page-header backfill inserted its meta block above a `:description:`
|
|
183
|
+
// it could not see and the page ended up with two. AsciiDoc takes the
|
|
184
|
+
// last assignment, so leaving the literal one there would make the
|
|
185
|
+
// include a no-op and the page would keep publishing frozen text.
|
|
186
|
+
if (lines.some((l) => l.trim() === includeAttrs)) {
|
|
187
|
+
for (let i = introOf(lines) - 1; i >= 0; i--) {
|
|
188
|
+
if (lines[i].startsWith(':description:')) {
|
|
189
|
+
lines.splice(i, 1);
|
|
190
|
+
changed = true;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// --- body -----------------------------------------------------------
|
|
196
|
+
if (!skipBody && !lines.some((l) => l.trim() === includeBody)) {
|
|
197
|
+
const start = lines.findIndex((l) => INTRO_START.test(l));
|
|
198
|
+
if (start === -1) {
|
|
199
|
+
results.skipped.push({ page: key, reason: 'no component_type_dropdown to anchor the intro' });
|
|
200
|
+
} else {
|
|
201
|
+
let end = lines.length;
|
|
202
|
+
for (let i = start + 1; i < lines.length; i++) {
|
|
203
|
+
if (INTRO_END.test(lines[i])) { end = i; break; }
|
|
204
|
+
}
|
|
205
|
+
const intro = lines.slice(start + 1, end);
|
|
206
|
+
if (contentLines(intro).length === 0) {
|
|
207
|
+
results.skipped.push({ page: key, reason: 'intro region is empty' });
|
|
208
|
+
} else if (!bodyCoversIntro(intro, body)) {
|
|
209
|
+
results.skipped.push({
|
|
210
|
+
page: key,
|
|
211
|
+
reason: 'intro prose is not reproduced by the partial (needs a human)',
|
|
212
|
+
});
|
|
213
|
+
} else {
|
|
214
|
+
// The page's own metadata include is emitted by the partial body
|
|
215
|
+
// too, so consume it rather than leaving a duplicate behind.
|
|
216
|
+
let tail = end;
|
|
217
|
+
const metaInclude = `include::connect:components:partial$metadata/${partialTypeDir}/${name}.adoc[]`;
|
|
218
|
+
if (lines[tail] && lines[tail].trim() === metaInclude &&
|
|
219
|
+
contentLines(body).includes(metaInclude)) {
|
|
220
|
+
tail++;
|
|
221
|
+
}
|
|
222
|
+
// Some pages carry the rest of the description BELOW the generated
|
|
223
|
+
// config listing, so replacing the intro alone would leave the
|
|
224
|
+
// include and the surviving prose both rendering it. Measured on
|
|
225
|
+
// caches/redpanda, where an Antora build reported the same xref
|
|
226
|
+
// twice after the naive rewrite. Anything outside the region that
|
|
227
|
+
// the partial body also renders means the description is split
|
|
228
|
+
// across the page, which a human has to untangle.
|
|
229
|
+
const outside = contentLines(lines.slice(0, start + 1).concat(lines.slice(tail)));
|
|
230
|
+
const inBody = new Set(contentLines(body));
|
|
231
|
+
if (outside.some((l) => inBody.has(l))) {
|
|
232
|
+
results.skipped.push({
|
|
233
|
+
page: key,
|
|
234
|
+
reason: 'description prose continues past the config listing (needs a human)',
|
|
235
|
+
});
|
|
236
|
+
} else {
|
|
237
|
+
lines.splice(start + 1, tail - start - 1, '', includeBody, '');
|
|
238
|
+
results.bodies.push(key);
|
|
239
|
+
changed = true;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
if (changed) {
|
|
246
|
+
console.log(`${write ? 'MIGRATE' : 'WOULD MIGRATE'}: ${key}`);
|
|
247
|
+
if (write) fs.writeFileSync(pagePath, lines.join('\n'), 'utf8');
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
console.log(
|
|
253
|
+
`\n${write ? 'Migrated' : 'Would migrate'} ${results.headers.length} page header(s) and ` +
|
|
254
|
+
`${results.bodies.length} page body/bodies; ${results.skipped.length} left alone.`
|
|
255
|
+
);
|
|
256
|
+
const byReason = new Map();
|
|
257
|
+
for (const s of results.skipped) byReason.set(s.reason, (byReason.get(s.reason) || 0) + 1);
|
|
258
|
+
for (const [reason, count] of [...byReason].sort((a, b) => b[1] - a[1])) {
|
|
259
|
+
console.log(` ${count} ${reason}`);
|
|
260
|
+
}
|
|
261
|
+
if (!write) console.log('Dry run only. Re-run with --write to apply.');
|
|
262
|
+
|
|
263
|
+
return results;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
module.exports = { migrateDescriptionsToPartials, tagRegion, bodyCoversIntro };
|
|
267
|
+
|
|
268
|
+
// Allow direct execution for local development; the supported entry point is
|
|
269
|
+
// `npx doc-tools generate migrate-rpcn-descriptions`.
|
|
270
|
+
if (require.main === module) {
|
|
271
|
+
migrateDescriptionsToPartials({
|
|
272
|
+
write: process.argv.includes('--write'),
|
|
273
|
+
skipBody: process.argv.includes('--skip-body'),
|
|
274
|
+
});
|
|
275
|
+
}
|
|
@@ -24,8 +24,10 @@ const FENCE_OPEN = /^(\s*)(```|~~~)(.*)$/;
|
|
|
24
24
|
const BULLET = /^(\s*[-*]\s+)(.*)$/;
|
|
25
25
|
// AsciiDoc literal/listing block delimiter. Content inside `----` blocks is
|
|
26
26
|
// rendered verbatim, so backticks would show up literally — such regions must
|
|
27
|
-
// pass through untouched (no field-name inline-coding).
|
|
28
|
-
|
|
27
|
+
// pass through untouched (no field-name inline-coding). Shared with the
|
|
28
|
+
// scanners in metadata-utils rather than redeclared: three copies of this had
|
|
29
|
+
// already drifted into disagreeing about indented delimiters.
|
|
30
|
+
const { BLOCK_DELIMITER } = require('./metadata-utils.js');
|
|
29
31
|
// A metadata field bullet: the leading token is a lowercase snake_case
|
|
30
32
|
// identifier (gcs_key, http_server_verb, header), optionally followed by a
|
|
31
33
|
// parenthetical annotation (for example "(RFC3339)"), and optionally a
|
|
@@ -71,12 +73,12 @@ function normalizeMetadataBlock (block) {
|
|
|
71
73
|
// AsciiDoc literal/listing block: pass the whole `----` … `----` region
|
|
72
74
|
// through verbatim. Field-name lines inside are rendered literally, so they
|
|
73
75
|
// must not be inline-coded (backticks would appear as visible characters).
|
|
74
|
-
if (BLOCK_DELIMITER.test(lines[i]
|
|
76
|
+
if (BLOCK_DELIMITER.test(lines[i])) {
|
|
75
77
|
out.push(lines[i]);
|
|
76
78
|
let j = i + 1;
|
|
77
79
|
for (; j < lines.length; j++) {
|
|
78
80
|
out.push(lines[j]);
|
|
79
|
-
if (BLOCK_DELIMITER.test(lines[j]
|
|
81
|
+
if (BLOCK_DELIMITER.test(lines[j])) break;
|
|
80
82
|
}
|
|
81
83
|
i = j;
|
|
82
84
|
continue;
|
|
@@ -115,6 +115,29 @@ function renderBloblangChanges(data, includeVersion = false) {
|
|
|
115
115
|
return lines;
|
|
116
116
|
}
|
|
117
117
|
|
|
118
|
+
/**
|
|
119
|
+
* Render upstream description-structure findings as a collapsed section of
|
|
120
|
+
* the PR summary. These are advisory (the fix belongs in the Connect
|
|
121
|
+
* source), but console.warn output dies in the collapsed workflow log, so
|
|
122
|
+
* they surface here where the PR reviewer actually looks.
|
|
123
|
+
* @param {Array<{connector: string, message: string}>} reports
|
|
124
|
+
* @returns {string[]} lines (empty when there is nothing to report)
|
|
125
|
+
*/
|
|
126
|
+
function renderDescriptionReports(reports) {
|
|
127
|
+
if (!Array.isArray(reports) || reports.length === 0) return [];
|
|
128
|
+
const lines = [];
|
|
129
|
+
lines.push('<details>');
|
|
130
|
+
lines.push(`<summary>:pencil2: ${reports.length} description(s) need structure fixes upstream in the Connect source</summary>`);
|
|
131
|
+
lines.push('');
|
|
132
|
+
for (const r of reports) {
|
|
133
|
+
lines.push(`- ${r.message}`);
|
|
134
|
+
}
|
|
135
|
+
lines.push('');
|
|
136
|
+
lines.push('</details>');
|
|
137
|
+
lines.push('');
|
|
138
|
+
return lines;
|
|
139
|
+
}
|
|
140
|
+
|
|
118
141
|
/**
|
|
119
142
|
* Generate PR summary for multiple releases
|
|
120
143
|
* @param {object} masterDiff - Master diff with releases array
|
|
@@ -153,6 +176,7 @@ function generateMultiVersionPRSummary(masterDiff, binaryAnalysis = null, drafte
|
|
|
153
176
|
lines.push('<!-- PR_SUMMARY_START -->');
|
|
154
177
|
lines.push('');
|
|
155
178
|
lines.push(...renderLostSectionWarnings(masterDiff?.lostSectionWarnings));
|
|
179
|
+
lines.push(...renderDescriptionReports(masterDiff?.descriptionReports));
|
|
156
180
|
lines.push('## Redpanda Connect Documentation Update');
|
|
157
181
|
lines.push('');
|
|
158
182
|
lines.push(`**Multi-Release Update:** ${startVersion} → ${endVersion}`);
|
|
@@ -559,6 +583,7 @@ function generatePRSummary(diffData, binaryAnalysis = null, draftedConnectors =
|
|
|
559
583
|
lines.push('<!-- PR_SUMMARY_START -->');
|
|
560
584
|
lines.push('');
|
|
561
585
|
lines.push(...renderLostSectionWarnings(diffData.lostSectionWarnings));
|
|
586
|
+
lines.push(...renderDescriptionReports(diffData.descriptionReports));
|
|
562
587
|
|
|
563
588
|
// Detect if this is a master diff
|
|
564
589
|
if (!isMultiVersion && diffData.releases && diffData.totalSummary) {
|
|
@@ -1326,9 +1351,35 @@ function printPRSummary(diffData, binaryAnalysis = null, draftedConnectors = nul
|
|
|
1326
1351
|
console.log('\n' + summary + '\n');
|
|
1327
1352
|
}
|
|
1328
1353
|
|
|
1354
|
+
/**
|
|
1355
|
+
* Merge one generateRpcnConnectorDocs result into the accumulators that feed
|
|
1356
|
+
* this PR summary.
|
|
1357
|
+
*
|
|
1358
|
+
* Every report key the generator returns has to be collected at every call
|
|
1359
|
+
* site, or the summary is quietly incomplete for whichever path forgot one.
|
|
1360
|
+
* The draft call site pushed lostSectionWarnings inline and forgot
|
|
1361
|
+
* descriptionReports, so structure reports for newly drafted connectors --
|
|
1362
|
+
* the ones most likely to need an upstream fix -- never reached the summary.
|
|
1363
|
+
* Doing the merge in one place is what makes that structural rather than
|
|
1364
|
+
* something each call site has to remember.
|
|
1365
|
+
*
|
|
1366
|
+
* @param {object} result generator result
|
|
1367
|
+
* @param {{descriptionReports: Array, lostSectionWarnings: Array}} into accumulators, mutated in place
|
|
1368
|
+
* @returns {{descriptionReports: Array, lostSectionWarnings: Array}} the same accumulators
|
|
1369
|
+
*/
|
|
1370
|
+
function collectGeneratorReports (result, into) {
|
|
1371
|
+
if (!result) return into;
|
|
1372
|
+
into.descriptionReports.push(...(result.descriptionReports || []));
|
|
1373
|
+
into.lostSectionWarnings.push(...(result.lostSectionWarnings || []));
|
|
1374
|
+
return into;
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1329
1377
|
module.exports = {
|
|
1378
|
+
collectGeneratorReports,
|
|
1330
1379
|
generatePRSummary,
|
|
1331
1380
|
generateMultiVersionPRSummary,
|
|
1332
1381
|
printPRSummary,
|
|
1382
|
+
renderDescriptionReports,
|
|
1383
|
+
renderLostSectionWarnings,
|
|
1333
1384
|
truncateToSentence
|
|
1334
1385
|
};
|
|
@@ -1234,7 +1234,15 @@ async function handleRpcnConnectorDocs (options) {
|
|
|
1234
1234
|
|
|
1235
1235
|
console.log('Generating connector partials...')
|
|
1236
1236
|
let partialsWritten, partialFiles
|
|
1237
|
+
const descriptionReports = []
|
|
1237
1238
|
const lostSectionWarnings = []
|
|
1239
|
+
// Both generator call sites (partials and drafts) feed the same PR summary,
|
|
1240
|
+
// so they collect through one function. Pushed inline, the draft call site
|
|
1241
|
+
// silently dropped descriptionReports for months: the structure reports for
|
|
1242
|
+
// newly drafted connectors, the ones most likely to need an upstream fix,
|
|
1243
|
+
// never reached the summary.
|
|
1244
|
+
const { collectGeneratorReports } = require('./pr-summary-formatter.js')
|
|
1245
|
+
const collect = (result) => collectGeneratorReports(result, { descriptionReports, lostSectionWarnings })
|
|
1238
1246
|
|
|
1239
1247
|
try {
|
|
1240
1248
|
const result = await generateRpcnConnectorDocs({
|
|
@@ -1245,14 +1253,16 @@ async function handleRpcnConnectorDocs (options) {
|
|
|
1245
1253
|
templateFields: options.templateFields,
|
|
1246
1254
|
templateExamples: options.templateExamples,
|
|
1247
1255
|
templateMetadata: options.templateMetadata,
|
|
1256
|
+
templateDescription: options.templateDescription,
|
|
1248
1257
|
templateBloblang: options.templateBloblang,
|
|
1249
1258
|
writeFullDrafts: false,
|
|
1259
|
+
pruneOrphanedDescriptions: !!options.pruneOrphanedDescriptions,
|
|
1250
1260
|
includeBloblang: !!options.includeBloblang,
|
|
1251
1261
|
csvMetadata
|
|
1252
1262
|
})
|
|
1253
1263
|
partialsWritten = result.partialsWritten
|
|
1254
1264
|
partialFiles = result.partialFiles
|
|
1255
|
-
|
|
1265
|
+
collect(result)
|
|
1256
1266
|
} catch (err) {
|
|
1257
1267
|
console.error(`Error: Failed to generate partials: ${err.message}`)
|
|
1258
1268
|
process.exit(1)
|
|
@@ -1883,6 +1893,12 @@ async function handleRpcnConnectorDocs (options) {
|
|
|
1883
1893
|
|
|
1884
1894
|
// Check each cloud-supported connector
|
|
1885
1895
|
// Filter to only check actual connector/component types that need individual pages
|
|
1896
|
+
// NOT the shared CONNECTOR_TYPE_DIRS: this list deliberately omits
|
|
1897
|
+
// rate_limits, per the comment below. rate_limits does have
|
|
1898
|
+
// per-component pages, so the omission looks like a bug and rate
|
|
1899
|
+
// limits are never checked for a cloud-docs stub -- but changing it
|
|
1900
|
+
// adds cloud-docs findings in a path with no test coverage, so it
|
|
1901
|
+
// needs its own ticket rather than riding along here.
|
|
1886
1902
|
const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'buffers', 'scanners', 'metrics', 'tracers']
|
|
1887
1903
|
|
|
1888
1904
|
for (const connectorKey of cloudSupportedSet) {
|
|
@@ -2035,6 +2051,7 @@ async function handleRpcnConnectorDocs (options) {
|
|
|
2035
2051
|
templateFields: options.templateFields,
|
|
2036
2052
|
templateExamples: options.templateExamples,
|
|
2037
2053
|
templateMetadata: options.templateMetadata,
|
|
2054
|
+
templateDescription: options.templateDescription,
|
|
2038
2055
|
templateIntro: options.templateIntro,
|
|
2039
2056
|
writeFullDrafts: true,
|
|
2040
2057
|
cgoOnly: binaryAnalysis?.cgoOnly || [],
|
|
@@ -2045,7 +2062,7 @@ async function handleRpcnConnectorDocs (options) {
|
|
|
2045
2062
|
fs.unlinkSync(tempDataPath)
|
|
2046
2063
|
draftsWritten = draftResult.draftsWritten
|
|
2047
2064
|
draftFiles = draftResult.draftFiles
|
|
2048
|
-
|
|
2065
|
+
collect(draftResult)
|
|
2049
2066
|
}
|
|
2050
2067
|
} catch (err) {
|
|
2051
2068
|
console.error(`Error: Could not draft missing: ${err.message}`)
|
|
@@ -2092,13 +2109,26 @@ async function handleRpcnConnectorDocs (options) {
|
|
|
2092
2109
|
|
|
2093
2110
|
// Generate PR summary
|
|
2094
2111
|
try {
|
|
2095
|
-
const { printPRSummary } = require('./pr-summary-formatter.js')
|
|
2112
|
+
const { printPRSummary, renderLostSectionWarnings, renderDescriptionReports } = require('./pr-summary-formatter.js')
|
|
2096
2113
|
// Use master diff if available, otherwise use single diff. Content-loss
|
|
2097
|
-
// warnings ride the diff object so they
|
|
2098
|
-
// of dying in the collapsed workflow log.
|
|
2114
|
+
// warnings and structure reports ride the diff object so they land in the
|
|
2115
|
+
// PR summary body instead of dying in the collapsed workflow log.
|
|
2099
2116
|
const summaryDiff = masterDiff || diffJson
|
|
2100
|
-
if (
|
|
2101
|
-
|
|
2117
|
+
if (summaryDiff) {
|
|
2118
|
+
if (lostSectionWarnings.length) summaryDiff.lostSectionWarnings = lostSectionWarnings
|
|
2119
|
+
if (descriptionReports.length) summaryDiff.descriptionReports = descriptionReports
|
|
2120
|
+
printPRSummary(summaryDiff, binaryAnalysis, draftFiles, masterDiff ? true : false)
|
|
2121
|
+
} else if (lostSectionWarnings.length || descriptionReports.length) {
|
|
2122
|
+
// No diff to summarize (no prior version, or versions match), but this
|
|
2123
|
+
// run still produced content-loss warnings or description-structure
|
|
2124
|
+
// reports that a reviewer needs to see -- print them on their own
|
|
2125
|
+
// instead of losing them because there was nothing to diff against.
|
|
2126
|
+
const lines = [
|
|
2127
|
+
...renderLostSectionWarnings(lostSectionWarnings),
|
|
2128
|
+
...renderDescriptionReports(descriptionReports)
|
|
2129
|
+
]
|
|
2130
|
+
console.log('\n' + lines.join('\n') + '\n')
|
|
2131
|
+
}
|
|
2102
2132
|
} catch (err) {
|
|
2103
2133
|
console.error(`Warning: Failed to generate PR summary: ${err.message}`)
|
|
2104
2134
|
}
|
|
@@ -3,23 +3,36 @@
|
|
|
3
3
|
:type: {{type}}
|
|
4
4
|
{{#if support}}:support: {{support}}{{/if}}
|
|
5
5
|
:categories: [{{join categories ", "}}]
|
|
6
|
-
{{#if
|
|
6
|
+
{{#if hasDescriptionAttrs}}
|
|
7
7
|
// tag::meta[]
|
|
8
|
-
|
|
8
|
+
{{{descriptionInclude this "attrs"}}}
|
|
9
|
+
// end::meta[]
|
|
10
|
+
{{else if summary}}
|
|
11
|
+
// tag::meta[]
|
|
12
|
+
:description: {{{summaryAttribute summary}}}
|
|
9
13
|
// end::meta[]
|
|
10
14
|
{{else if description}}
|
|
11
15
|
// tag::meta[]
|
|
12
|
-
:description: {{description}}
|
|
16
|
+
:description: {{{summaryAttribute description}}}
|
|
13
17
|
// end::meta[]
|
|
14
18
|
{{/if}}
|
|
15
|
-
{{!--
|
|
19
|
+
{{!-- The attrs include stays inside the meta tag region so single-source
|
|
20
|
+
stubs keep inheriting :description: as a header attribute, and the
|
|
21
|
+
value now refreshes with the connector instead of being frozen at
|
|
22
|
+
first draft. Component families with no description partial keep the
|
|
23
|
+
inline attribute.
|
|
24
|
+
No fallback line when neither exists: an emitted TODO would become
|
|
16
25
|
the attribute VALUE and stubs would inherit it into their meta tags.
|
|
17
26
|
Missing descriptions are the tracked editorial backlog
|
|
18
27
|
(cloud-docs .github/description-allowlist.txt). --}}
|
|
19
28
|
|
|
20
29
|
component_type_dropdown::[]
|
|
21
30
|
|
|
31
|
+
{{#if hasDescriptionPartial}}
|
|
32
|
+
{{{descriptionInclude this "body"}}}
|
|
33
|
+
{{else}}
|
|
22
34
|
{{> intro}}
|
|
35
|
+
{{/if}}
|
|
23
36
|
|
|
24
37
|
{{#if (or this.config.children children)}}
|
|
25
38
|
[tabs]
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{{#if (or summary description version)}}
|
|
2
|
+
// This content is autogenerated. Do not edit manually. The summary and description come from the connector's definition in the Connect source. To change them, update the connector upstream, or use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
|
|
3
|
+
//
|
|
4
|
+
// Pages consume this partial in two places so both stay fresh:
|
|
5
|
+
// header: include::...[tag=attrs] (sets :description: for search snippets and meta tags)
|
|
6
|
+
// body: include::...[tag=body] (renders the summary and description)
|
|
7
|
+
|
|
8
|
+
// tag::attrs[]
|
|
9
|
+
{{#if summary}}:description: {{{summaryAttribute summary}}}
|
|
10
|
+
{{/if}}// end::attrs[]
|
|
11
|
+
|
|
12
|
+
// tag::body[]
|
|
13
|
+
{{#if summary}}{{{summary}}}
|
|
14
|
+
|
|
15
|
+
{{/if}}{{#if version}}ifndef::env-cloud[]
|
|
16
|
+
Introduced in version {{version}}.
|
|
17
|
+
endif::[]
|
|
18
|
+
|
|
19
|
+
{{/if}}{{{renderConnectDescription this}}}
|
|
20
|
+
// end::body[]
|
|
21
|
+
{{/if}}
|