@redpanda-data/docs-extensions-and-macros 4.9.0 → 4.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/doc-tools.js +154 -2
- package/package.json +1 -1
- package/tools/redpanda-connect/generate-rpcn-connector-docs.js +96 -2
- package/tools/redpanda-connect/helpers/bloblangExample.js +42 -0
- package/tools/redpanda-connect/helpers/index.js +4 -3
- package/tools/redpanda-connect/helpers/renderConnectFields.js +32 -5
- package/tools/redpanda-connect/report-delta.js +101 -0
- package/tools/redpanda-connect/templates/bloblang-function.hbs +28 -0
package/bin/doc-tools.js
CHANGED
|
@@ -676,6 +676,7 @@ automation
|
|
|
676
676
|
.description('Generate RPCN connector docs and diff changes since the last version')
|
|
677
677
|
.option('-d, --data-dir <path>', 'Directory where versioned connect JSON files live', path.resolve(process.cwd(), 'docs-data'))
|
|
678
678
|
.option('--old-data <path>', 'Optional override for old data file (for diff)')
|
|
679
|
+
.option('--update-whats-new', 'Update whats-new.adoc with new section from diff JSON')
|
|
679
680
|
.option('-f, --fetch-connectors', 'Fetch latest connector data using rpk')
|
|
680
681
|
.option('-m, --draft-missing', 'Generate full-doc drafts for connectors missing in output')
|
|
681
682
|
.option('--csv <path>', 'Path to connector metadata CSV file', 'internal/plugins/info.csv')
|
|
@@ -683,7 +684,9 @@ automation
|
|
|
683
684
|
.option('--template-intro <path>', 'Intro section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/intro.hbs'))
|
|
684
685
|
.option('--template-fields <path>', 'Fields section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/fields-partials.hbs'))
|
|
685
686
|
.option('--template-examples <path>', 'Examples section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/examples-partials.hbs'))
|
|
687
|
+
.option('--template-bloblang <path>', 'Custom Handlebars template for bloblang function/method partials')
|
|
686
688
|
.option('--overrides <path>', 'Optional JSON file with overrides')
|
|
689
|
+
.option('--include-bloblang', 'Include Bloblang functions and methods in generation')
|
|
687
690
|
.action(async (options) => {
|
|
688
691
|
requireTool('rpk', {
|
|
689
692
|
versionFlag: '--version',
|
|
@@ -744,7 +747,9 @@ automation
|
|
|
744
747
|
templateIntro: options.templateIntro,
|
|
745
748
|
templateFields: options.templateFields,
|
|
746
749
|
templateExamples: options.templateExamples,
|
|
747
|
-
|
|
750
|
+
templateBloblang: options.templateBloblang,
|
|
751
|
+
writeFullDrafts: false,
|
|
752
|
+
includeBloblang: !!options.includeBloblang
|
|
748
753
|
});
|
|
749
754
|
partialsWritten = result.partialsWritten;
|
|
750
755
|
partialFiles = result.partialFiles;
|
|
@@ -826,10 +831,13 @@ automation
|
|
|
826
831
|
}
|
|
827
832
|
|
|
828
833
|
let oldIndex = {};
|
|
834
|
+
let oldVersion = null;
|
|
829
835
|
if (options.oldData && fs.existsSync(options.oldData)) {
|
|
830
836
|
oldIndex = JSON.parse(fs.readFileSync(options.oldData, 'utf8'));
|
|
837
|
+
const m = options.oldData.match(/connect-([\d.]+)\.json$/);
|
|
838
|
+
if (m) oldVersion = m[1];
|
|
831
839
|
} else {
|
|
832
|
-
|
|
840
|
+
oldVersion = getAntoraValue('asciidoc.attributes.latest-connect-version');
|
|
833
841
|
if (oldVersion) {
|
|
834
842
|
const oldPath = path.join(dataDir, `connect-${oldVersion}.json`);
|
|
835
843
|
if (fs.existsSync(oldPath)) {
|
|
@@ -841,6 +849,21 @@ automation
|
|
|
841
849
|
const newIndex = JSON.parse(fs.readFileSync(dataFile, 'utf8'));
|
|
842
850
|
printDeltaReport(oldIndex, newIndex);
|
|
843
851
|
|
|
852
|
+
// Generate JSON diff file for whats-new.adoc
|
|
853
|
+
const { generateConnectorDiffJson } = require('../tools/redpanda-connect/report-delta.js');
|
|
854
|
+
const diffJson = generateConnectorDiffJson(
|
|
855
|
+
oldIndex,
|
|
856
|
+
newIndex,
|
|
857
|
+
{
|
|
858
|
+
oldVersion: oldVersion || '',
|
|
859
|
+
newVersion,
|
|
860
|
+
timestamp
|
|
861
|
+
}
|
|
862
|
+
);
|
|
863
|
+
const diffPath = path.join(dataDir, `connect-diff-${(oldVersion || 'unknown')}_to_${newVersion}.json`);
|
|
864
|
+
fs.writeFileSync(diffPath, JSON.stringify(diffJson, null, 2), 'utf8');
|
|
865
|
+
console.log(`✅ Connector diff JSON written to: ${diffPath}`);
|
|
866
|
+
|
|
844
867
|
function logCollapsed(label, filesArray, maxToShow = 10) {
|
|
845
868
|
console.log(` • ${label}: ${filesArray.length} total`);
|
|
846
869
|
const sample = filesArray.slice(0, maxToShow);
|
|
@@ -872,6 +895,135 @@ automation
|
|
|
872
895
|
logCollapsed('Draft files', draftFiles, 5);
|
|
873
896
|
}
|
|
874
897
|
|
|
898
|
+
// Optionally update whats-new.adoc
|
|
899
|
+
if (options.updateWhatsNew) {
|
|
900
|
+
try {
|
|
901
|
+
const whatsNewPath = path.join(findRepoRoot(), 'modules/get-started/pages/whats-new.adoc');
|
|
902
|
+
if (!fs.existsSync(whatsNewPath)) {
|
|
903
|
+
console.error(`❌ Unable to update release notes: 'whats-new.adoc' was not found at: ${whatsNewPath}\nPlease ensure this file exists and is tracked in your repository.`);
|
|
904
|
+
return;
|
|
905
|
+
}
|
|
906
|
+
// Find the diff JSON file we just wrote
|
|
907
|
+
const diffPath = path.join(dataDir, `connect-diff-${(oldVersion || 'unknown')}_to_${newVersion}.json`);
|
|
908
|
+
if (!fs.existsSync(diffPath)) {
|
|
909
|
+
console.error(`❌ Unable to update release notes: The connector diff JSON was not found at: ${diffPath}\nPlease ensure the diff was generated successfully before updating release notes.`);
|
|
910
|
+
return;
|
|
911
|
+
}
|
|
912
|
+
let diff;
|
|
913
|
+
try {
|
|
914
|
+
diff = JSON.parse(fs.readFileSync(diffPath, 'utf8'));
|
|
915
|
+
} catch (jsonErr) {
|
|
916
|
+
console.error(`❌ Unable to parse connector diff JSON at ${diffPath}: ${jsonErr.message}\nPlease check the file for syntax errors or corruption.`);
|
|
917
|
+
return;
|
|
918
|
+
}
|
|
919
|
+
let whatsNewContent;
|
|
920
|
+
try {
|
|
921
|
+
whatsNewContent = fs.readFileSync(whatsNewPath, 'utf8');
|
|
922
|
+
} catch (readErr) {
|
|
923
|
+
console.error(`❌ Unable to read whats-new.adoc at ${whatsNewPath}: ${readErr.message}\nPlease check file permissions and try again.`);
|
|
924
|
+
return;
|
|
925
|
+
}
|
|
926
|
+
const whatsNew = whatsNewContent;
|
|
927
|
+
// Regex to find section for this version
|
|
928
|
+
const versionTitle = `== Version ${diff.comparison.newVersion}`;
|
|
929
|
+
const versionRe = new RegExp(`^== Version ${diff.comparison.newVersion.replace(/[-.]/g, '\\$&')}(?:\\r?\\n|$)`, 'm');
|
|
930
|
+
const match = versionRe.exec(whatsNew);
|
|
931
|
+
let startIdx = match ? match.index : -1;
|
|
932
|
+
let endIdx = -1;
|
|
933
|
+
if (startIdx !== -1) {
|
|
934
|
+
// Find the start of the next version section
|
|
935
|
+
const rest = whatsNew.slice(startIdx + 1);
|
|
936
|
+
const nextMatch = /^== Version /m.exec(rest);
|
|
937
|
+
endIdx = nextMatch ? startIdx + 1 + nextMatch.index : whatsNew.length;
|
|
938
|
+
}
|
|
939
|
+
// Compose new section
|
|
940
|
+
let section = `\n== Version ${diff.comparison.newVersion}\n\n=== Component updates\n\n`;
|
|
941
|
+
// Add link to full release notes for this connector version after version heading and before component updates
|
|
942
|
+
let releaseNotesLink = '';
|
|
943
|
+
if (diff.comparison && diff.comparison.newVersion) {
|
|
944
|
+
releaseNotesLink = `link:https://github.com/redpanda-data/connect/releases/tag/v${diff.comparison.newVersion}[See the full release notes^].\n\n`;
|
|
945
|
+
}
|
|
946
|
+
section = `\n== Version ${diff.comparison.newVersion}\n\n${releaseNotesLink}=== Component updates\n\n`;
|
|
947
|
+
// New components
|
|
948
|
+
if (diff.details.newComponents && diff.details.newComponents.length) {
|
|
949
|
+
section += 'This release adds the following new components:\n\n';
|
|
950
|
+
// Group by type
|
|
951
|
+
const byType = {};
|
|
952
|
+
for (const comp of diff.details.newComponents) {
|
|
953
|
+
if (!byType[comp.type]) byType[comp.type] = [];
|
|
954
|
+
byType[comp.type].push(comp);
|
|
955
|
+
}
|
|
956
|
+
for (const [type, comps] of Object.entries(byType)) {
|
|
957
|
+
section += `* ${type.charAt(0).toUpperCase() + type.slice(1)}:\n`;
|
|
958
|
+
for (const comp of comps) {
|
|
959
|
+
section += `** xref:components:${type}/${comp.name}.adoc[\`${comp.name}\`]`;
|
|
960
|
+
if (comp.status) section += ` (${comp.status})`;
|
|
961
|
+
if (comp.description) section += `: ${comp.description}`;
|
|
962
|
+
section += '\n';
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
// New fields
|
|
968
|
+
if (diff.details.newFields && diff.details.newFields.length) {
|
|
969
|
+
section += '\nThis release adds support for the following new fields:\n\n';
|
|
970
|
+
// Group new fields by component type
|
|
971
|
+
const fieldsByType = {};
|
|
972
|
+
for (const field of diff.details.newFields) {
|
|
973
|
+
// component: "inputs:kafka", field: "timely_nacks_maximum_wait"
|
|
974
|
+
const [type, compName] = field.component.split(':');
|
|
975
|
+
if (!fieldsByType[type]) fieldsByType[type] = [];
|
|
976
|
+
fieldsByType[type].push({
|
|
977
|
+
compName,
|
|
978
|
+
field: field.field,
|
|
979
|
+
description: field.description || '',
|
|
980
|
+
});
|
|
981
|
+
}
|
|
982
|
+
for (const [type, fields] of Object.entries(fieldsByType)) {
|
|
983
|
+
section += `* ${type.charAt(0).toUpperCase() + type.slice(1)} components\n`;
|
|
984
|
+
// Group by component name
|
|
985
|
+
const byComp = {};
|
|
986
|
+
for (const f of fields) {
|
|
987
|
+
if (!byComp[f.compName]) byComp[f.compName] = [];
|
|
988
|
+
byComp[f.compName].push(f);
|
|
989
|
+
}
|
|
990
|
+
for (const [comp, compFields] of Object.entries(byComp)) {
|
|
991
|
+
section += `** xref:components:${type}/${comp}.adoc['${comp}']`;
|
|
992
|
+
if (compFields.length === 1) {
|
|
993
|
+
const f = compFields[0];
|
|
994
|
+
section += `: xref:components:${type}/${comp}.adoc#${f.field}['${f.field}']`;
|
|
995
|
+
if (f.description) section += ` - ${f.description}`;
|
|
996
|
+
section += '\n';
|
|
997
|
+
} else {
|
|
998
|
+
section += '\n';
|
|
999
|
+
for (const f of compFields) {
|
|
1000
|
+
section += `*** xref:components:${type}/${comp}.adoc#${f.field}['${f.field}']`;
|
|
1001
|
+
if (f.description) section += ` - ${f.description}`;
|
|
1002
|
+
section += '\n';
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
1008
|
+
let updated;
|
|
1009
|
+
if (startIdx !== -1) {
|
|
1010
|
+
// Replace the existing section
|
|
1011
|
+
updated = whatsNew.slice(0, startIdx) + section + '\n' + whatsNew.slice(endIdx);
|
|
1012
|
+
console.log(`♻️ whats-new.adoc: replaced section for Version ${diff.comparison.newVersion}`);
|
|
1013
|
+
} else {
|
|
1014
|
+
// Insert above first version heading
|
|
1015
|
+
const versionHeading = /^== Version /m;
|
|
1016
|
+
const firstMatch = versionHeading.exec(whatsNew);
|
|
1017
|
+
let insertIdx = firstMatch ? firstMatch.index : 0;
|
|
1018
|
+
updated = whatsNew.slice(0, insertIdx) + section + '\n' + whatsNew.slice(insertIdx);
|
|
1019
|
+
console.log(`✅ whats-new.adoc updated with Version ${diff.comparison.newVersion}`);
|
|
1020
|
+
}
|
|
1021
|
+
fs.writeFileSync(whatsNewPath, updated, 'utf8');
|
|
1022
|
+
} catch (err) {
|
|
1023
|
+
console.error(`❌ Failed to update whats-new.adoc: ${err.message}`);
|
|
1024
|
+
}
|
|
1025
|
+
}
|
|
1026
|
+
|
|
875
1027
|
console.log('\n📄 Summary:');
|
|
876
1028
|
console.log(` • Run time: ${timestamp}`);
|
|
877
1029
|
console.log(` • Version used: ${newVersion}`);
|
package/package.json
CHANGED
|
@@ -209,6 +209,31 @@ function resolveReferences(obj, root) {
|
|
|
209
209
|
* When generating full drafts, components with a `status` of `'deprecated'` are skipped.
|
|
210
210
|
*/
|
|
211
211
|
async function generateRpcnConnectorDocs(options) {
|
|
212
|
+
// Types and output folders for bloblang function/method partials
|
|
213
|
+
const bloblangTypes = [
|
|
214
|
+
{ key: 'bloblang-functions', folder: 'bloblang-functions' },
|
|
215
|
+
{ key: 'bloblang-methods', folder: 'bloblang-methods' }
|
|
216
|
+
];
|
|
217
|
+
// Recursively mark is_beta on any field/component with description starting with BETA:
|
|
218
|
+
function markBeta(obj) {
|
|
219
|
+
if (!obj || typeof obj !== 'object') return;
|
|
220
|
+
if (Array.isArray(obj)) {
|
|
221
|
+
obj.forEach(markBeta);
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
// Mark as beta if description starts with 'BETA:' (case-insensitive, trims leading whitespace)
|
|
225
|
+
if (typeof obj.description === 'string' && /^\s*BETA:\s*/i.test(obj.description)) {
|
|
226
|
+
obj.is_beta = true;
|
|
227
|
+
}
|
|
228
|
+
// Recurse into children/config/fields
|
|
229
|
+
if (Array.isArray(obj.children)) obj.children.forEach(markBeta);
|
|
230
|
+
if (obj.config && Array.isArray(obj.config.children)) obj.config.children.forEach(markBeta);
|
|
231
|
+
// For connector/component arrays
|
|
232
|
+
for (const key of Object.keys(obj)) {
|
|
233
|
+
if (Array.isArray(obj[key])) obj[key].forEach(markBeta);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
212
237
|
const {
|
|
213
238
|
data,
|
|
214
239
|
overrides,
|
|
@@ -216,6 +241,7 @@ async function generateRpcnConnectorDocs(options) {
|
|
|
216
241
|
templateIntro,
|
|
217
242
|
templateFields,
|
|
218
243
|
templateExamples,
|
|
244
|
+
templateBloblang,
|
|
219
245
|
writeFullDrafts
|
|
220
246
|
} = options;
|
|
221
247
|
|
|
@@ -223,16 +249,38 @@ async function generateRpcnConnectorDocs(options) {
|
|
|
223
249
|
const raw = fs.readFileSync(data, 'utf8');
|
|
224
250
|
const ext = path.extname(data).toLowerCase();
|
|
225
251
|
const dataObj = ext === '.json' ? JSON.parse(raw) : yaml.parse(raw);
|
|
252
|
+
// Mark beta fields/components before overrides
|
|
253
|
+
markBeta(dataObj);
|
|
226
254
|
|
|
227
255
|
// Apply overrides if provided
|
|
228
256
|
if (overrides) {
|
|
229
257
|
const ovRaw = fs.readFileSync(overrides, 'utf8');
|
|
230
258
|
const ovObj = JSON.parse(ovRaw);
|
|
231
|
-
|
|
232
259
|
// Resolve any $ref references in the overrides
|
|
233
260
|
const resolvedOverrides = resolveReferences(ovObj, ovObj);
|
|
234
|
-
|
|
235
261
|
mergeOverrides(dataObj, resolvedOverrides);
|
|
262
|
+
|
|
263
|
+
// Special: merge bloblang_methods and bloblang_functions from overrides into main data
|
|
264
|
+
for (const [overrideKey, mainKey] of [
|
|
265
|
+
['bloblang_methods', 'bloblang-methods'],
|
|
266
|
+
['bloblang_functions', 'bloblang-functions']
|
|
267
|
+
]) {
|
|
268
|
+
if (Array.isArray(resolvedOverrides[overrideKey])) {
|
|
269
|
+
if (!Array.isArray(dataObj[mainKey])) dataObj[mainKey] = [];
|
|
270
|
+
// Merge by name
|
|
271
|
+
const mainArr = dataObj[mainKey];
|
|
272
|
+
const overrideArr = resolvedOverrides[overrideKey];
|
|
273
|
+
for (const overrideItem of overrideArr) {
|
|
274
|
+
if (!overrideItem.name) continue;
|
|
275
|
+
const idx = mainArr.findIndex(i => i.name === overrideItem.name);
|
|
276
|
+
if (idx !== -1) {
|
|
277
|
+
mainArr[idx] = { ...mainArr[idx], ...overrideItem };
|
|
278
|
+
} else {
|
|
279
|
+
mainArr.push(overrideItem);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
236
284
|
}
|
|
237
285
|
|
|
238
286
|
// Compile the “main” template (used when writeFullDrafts = true)
|
|
@@ -260,6 +308,7 @@ async function generateRpcnConnectorDocs(options) {
|
|
|
260
308
|
const fieldsOutRoot = path.join(outputRoot, 'fields');
|
|
261
309
|
const examplesOutRoot = path.join(outputRoot, 'examples');
|
|
262
310
|
const draftsRoot = path.join(outputRoot, 'drafts');
|
|
311
|
+
const configExamplesRoot = path.resolve(process.cwd(), 'modules/components/examples');
|
|
263
312
|
|
|
264
313
|
if (!writeFullDrafts) {
|
|
265
314
|
fs.mkdirSync(fieldsOutRoot, { recursive: true });
|
|
@@ -332,6 +381,51 @@ async function generateRpcnConnectorDocs(options) {
|
|
|
332
381
|
}
|
|
333
382
|
}
|
|
334
383
|
|
|
384
|
+
// Bloblang function/method partials (only if includeBloblang is true)
|
|
385
|
+
if (options.includeBloblang) {
|
|
386
|
+
for (const { key, folder } of bloblangTypes) {
|
|
387
|
+
const items = dataObj[key];
|
|
388
|
+
if (!Array.isArray(items)) continue;
|
|
389
|
+
const outRoot = path.join(outputRoot, folder);
|
|
390
|
+
fs.mkdirSync(outRoot, { recursive: true });
|
|
391
|
+
// Use custom or default template
|
|
392
|
+
const bloblangTemplatePath = templateBloblang || path.resolve(__dirname, './templates/bloblang-function.hbs');
|
|
393
|
+
const bloblangTemplate = handlebars.compile(fs.readFileSync(bloblangTemplatePath, 'utf8'));
|
|
394
|
+
for (const fn of items) {
|
|
395
|
+
if (!fn.name) continue;
|
|
396
|
+
const adoc = bloblangTemplate(fn);
|
|
397
|
+
const outPath = path.join(outRoot, `${fn.name}.adoc`);
|
|
398
|
+
fs.writeFileSync(outPath, adoc, 'utf8');
|
|
399
|
+
partialsWritten++;
|
|
400
|
+
partialFiles.push(path.relative(process.cwd(), outPath));
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
// Common/Advanced config snippet YAMLs in modules/components/examples
|
|
406
|
+
const commonConfig = helpers.commonConfig;
|
|
407
|
+
const advancedConfig = helpers.advancedConfig;
|
|
408
|
+
for (const [type, items] of Object.entries(dataObj)) {
|
|
409
|
+
if (!Array.isArray(items)) continue;
|
|
410
|
+
for (const item of items) {
|
|
411
|
+
if (!item.name || !item.config || !Array.isArray(item.config.children)) continue;
|
|
412
|
+
// Common config
|
|
413
|
+
const commonYaml = commonConfig(type, item.name, item.config.children);
|
|
414
|
+
const commonPath = path.join(configExamplesRoot, 'common', type, `${item.name}.yaml`);
|
|
415
|
+
fs.mkdirSync(path.dirname(commonPath), { recursive: true });
|
|
416
|
+
fs.writeFileSync(commonPath, commonYaml.toString(), 'utf8');
|
|
417
|
+
partialsWritten++;
|
|
418
|
+
partialFiles.push(path.relative(process.cwd(), commonPath));
|
|
419
|
+
// Advanced config
|
|
420
|
+
const advYaml = advancedConfig(type, item.name, item.config.children);
|
|
421
|
+
const advPath = path.join(configExamplesRoot, 'advanced', type, `${item.name}.yaml`);
|
|
422
|
+
fs.mkdirSync(path.dirname(advPath), { recursive: true });
|
|
423
|
+
fs.writeFileSync(advPath, advYaml.toString(), 'utf8');
|
|
424
|
+
partialsWritten++;
|
|
425
|
+
partialFiles.push(path.relative(process.cwd(), advPath));
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
335
429
|
return {
|
|
336
430
|
partialsWritten,
|
|
337
431
|
draftsWritten,
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// Bloblang example formatting helper for Handlebars
|
|
2
|
+
function bloblangExample(example) {
|
|
3
|
+
if (typeof example === 'object' && example !== null && example.mapping) {
|
|
4
|
+
let codeBlock = '';
|
|
5
|
+
if (example.summary && example.summary.trim()) {
|
|
6
|
+
codeBlock += `# ${example.summary.trim().replace(/\n/g, '\n# ')}\n\n`;
|
|
7
|
+
}
|
|
8
|
+
if (typeof example.mapping === 'string') {
|
|
9
|
+
codeBlock += example.mapping.trim() + '\n';
|
|
10
|
+
}
|
|
11
|
+
if (Array.isArray(example.results)) {
|
|
12
|
+
for (const pair of example.results) {
|
|
13
|
+
if (Array.isArray(pair) && pair.length === 2) {
|
|
14
|
+
codeBlock += `\n# In: ${pair[0]}\n# Out: ${pair[1]}\n`;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
return `[,coffeescript]\n----\n${codeBlock.trim()}\n----\n`;
|
|
19
|
+
} else {
|
|
20
|
+
let exStr = '';
|
|
21
|
+
if (typeof example === 'string') {
|
|
22
|
+
exStr = example;
|
|
23
|
+
} else if (typeof example === 'object' && example !== null) {
|
|
24
|
+
if (example.code) {
|
|
25
|
+
exStr = example.code;
|
|
26
|
+
} else if (example.example) {
|
|
27
|
+
exStr = example.example;
|
|
28
|
+
} else {
|
|
29
|
+
try {
|
|
30
|
+
exStr = require('yaml').stringify(example).trim();
|
|
31
|
+
} catch {
|
|
32
|
+
exStr = JSON.stringify(example, null, 2);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
} else {
|
|
36
|
+
exStr = String(example);
|
|
37
|
+
}
|
|
38
|
+
return `[source,coffeescript]\n----\n${exStr}\n----\n`;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
module.exports = bloblangExample;
|
|
@@ -4,10 +4,10 @@ module.exports = {
|
|
|
4
4
|
uppercase: require('./uppercase.js'),
|
|
5
5
|
eq: require('./eq.js'),
|
|
6
6
|
ne: require('./ne.js'),
|
|
7
|
-
join:
|
|
7
|
+
join: require('./join.js'),
|
|
8
8
|
or: require('./or.js'),
|
|
9
|
-
toYaml:
|
|
10
|
-
isObject:
|
|
9
|
+
toYaml: require('./toYaml.js'),
|
|
10
|
+
isObject: require('./isObject.js'),
|
|
11
11
|
renderYamlList: require('./renderYamlList.js'),
|
|
12
12
|
renderConnectFields: require('./renderConnectFields.js'),
|
|
13
13
|
renderConnectExamples: require('./renderConnectExamples.js'),
|
|
@@ -16,4 +16,5 @@ module.exports = {
|
|
|
16
16
|
buildConfigYaml: require('./buildConfigYaml.js'),
|
|
17
17
|
commonConfig: require('./commonConfig.js'),
|
|
18
18
|
advancedConfig: require('./advancedConfig.js'),
|
|
19
|
+
bloblangExample: require('./bloblangExample.js'),
|
|
19
20
|
};
|
|
@@ -28,7 +28,10 @@ module.exports = function renderConnectFields(children, prefix = '') {
|
|
|
28
28
|
|
|
29
29
|
// Normalize types
|
|
30
30
|
let displayType;
|
|
31
|
-
|
|
31
|
+
const isArrayTitle = typeof child.name === 'string' && child.name.endsWith('[]');
|
|
32
|
+
if (isArrayTitle) {
|
|
33
|
+
displayType = 'array<object>';
|
|
34
|
+
} else if (child.type === 'string' && child.kind === 'array') {
|
|
32
35
|
displayType = 'array';
|
|
33
36
|
} else if (child.type === 'unknown' && child.kind === 'map') {
|
|
34
37
|
displayType = 'object';
|
|
@@ -42,14 +45,38 @@ module.exports = function renderConnectFields(children, prefix = '') {
|
|
|
42
45
|
|
|
43
46
|
let block = '';
|
|
44
47
|
const isArray = child.kind === 'array';
|
|
48
|
+
// Only append [] if kind is array and name does not already end with []
|
|
49
|
+
const nameWithArray = (typeof child.name === 'string' && isArray && !child.name.endsWith('[]'))
|
|
50
|
+
? `${child.name}[]`
|
|
51
|
+
: child.name;
|
|
45
52
|
const currentPath = prefix
|
|
46
|
-
? `${prefix}.${
|
|
47
|
-
: `${
|
|
53
|
+
? `${prefix}.${nameWithArray}`
|
|
54
|
+
: `${nameWithArray}`;
|
|
48
55
|
|
|
49
56
|
block += `=== \`${currentPath}\`\n\n`;
|
|
50
57
|
|
|
51
|
-
|
|
52
|
-
|
|
58
|
+
// --- Beta badge logic (now uses is_beta) ---
|
|
59
|
+
let desc = child.description || '';
|
|
60
|
+
if (child.is_beta) {
|
|
61
|
+
// Remove any leading "BETA:" label (case-insensitive, trims leading whitespace)
|
|
62
|
+
desc = 'badge::[label=Beta, size=large, tooltip={page-beta-text}]\n\n' + desc.replace(/^\s*BETA:\s*/i, '');
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// --- Interpolation support notice ---
|
|
66
|
+
const interpolationNotice = 'This field supports xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions].';
|
|
67
|
+
if (child.interpolation === true) {
|
|
68
|
+
// Only add if not already present (case-insensitive)
|
|
69
|
+
const descLower = desc.toLowerCase();
|
|
70
|
+
if (!descLower.includes('interpolation functions')) {
|
|
71
|
+
desc = desc.trim() + (desc.trim() ? '\n\n' : '') + interpolationNotice;
|
|
72
|
+
}
|
|
73
|
+
} else {
|
|
74
|
+
// If interpolation is not true, remove the notice if present
|
|
75
|
+
desc = desc.replace(new RegExp(interpolationNotice.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'), '').replace(/\n{2,}/g, '\n\n');
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
if (desc) {
|
|
79
|
+
block += `${desc}\n\n`;
|
|
53
80
|
}
|
|
54
81
|
if (child.is_secret) {
|
|
55
82
|
block += `include::redpanda-connect:components:partial$secret_warning.adoc[]\n\n`;
|
|
@@ -1,3 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Generate a JSON diff report between two connector index objects.
|
|
3
|
+
* @param {object} oldIndex - Previous version connector index
|
|
4
|
+
* @param {object} newIndex - Current version connector index
|
|
5
|
+
* @param {object} opts - { oldVersion, newVersion, timestamp }
|
|
6
|
+
* @returns {object} JSON diff report
|
|
7
|
+
*/
|
|
8
|
+
function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
|
|
9
|
+
const oldMap = buildComponentMap(oldIndex);
|
|
10
|
+
const newMap = buildComponentMap(newIndex);
|
|
11
|
+
|
|
12
|
+
// New components
|
|
13
|
+
const newComponentKeys = Object.keys(newMap).filter(k => !(k in oldMap));
|
|
14
|
+
const newComponents = newComponentKeys.map(key => {
|
|
15
|
+
const [type, name] = key.split(':');
|
|
16
|
+
const raw = newMap[key].raw;
|
|
17
|
+
return {
|
|
18
|
+
name,
|
|
19
|
+
type,
|
|
20
|
+
status: raw.status || raw.type || '',
|
|
21
|
+
version: raw.version || raw.introducedInVersion || '',
|
|
22
|
+
description: raw.description || ''
|
|
23
|
+
};
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
// Removed components
|
|
27
|
+
const removedComponentKeys = Object.keys(oldMap).filter(k => !(k in newMap));
|
|
28
|
+
const removedComponents = removedComponentKeys.map(key => {
|
|
29
|
+
const [type, name] = key.split(':');
|
|
30
|
+
const raw = oldMap[key].raw;
|
|
31
|
+
return {
|
|
32
|
+
name,
|
|
33
|
+
type,
|
|
34
|
+
status: raw.status || raw.type || '',
|
|
35
|
+
version: raw.version || raw.introducedInVersion || '',
|
|
36
|
+
description: raw.description || ''
|
|
37
|
+
};
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
// New fields under existing components
|
|
41
|
+
const newFields = [];
|
|
42
|
+
Object.keys(newMap).forEach(cKey => {
|
|
43
|
+
if (!(cKey in oldMap)) return;
|
|
44
|
+
const oldFields = new Set(oldMap[cKey].fields || []);
|
|
45
|
+
const newFieldsArr = newMap[cKey].fields || [];
|
|
46
|
+
newFieldsArr.forEach(fName => {
|
|
47
|
+
if (!oldFields.has(fName)) {
|
|
48
|
+
const [type, compName] = cKey.split(':');
|
|
49
|
+
let rawFieldObj = null;
|
|
50
|
+
if (type === 'config') {
|
|
51
|
+
rawFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
|
|
52
|
+
} else {
|
|
53
|
+
rawFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
|
|
54
|
+
}
|
|
55
|
+
newFields.push({
|
|
56
|
+
component: cKey,
|
|
57
|
+
field: fName,
|
|
58
|
+
introducedIn: rawFieldObj && (rawFieldObj.introducedInVersion || rawFieldObj.version),
|
|
59
|
+
description: rawFieldObj && rawFieldObj.description
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
// Removed fields under existing components
|
|
66
|
+
const removedFields = [];
|
|
67
|
+
Object.keys(oldMap).forEach(cKey => {
|
|
68
|
+
if (!(cKey in newMap)) return;
|
|
69
|
+
const newFieldsSet = new Set(newMap[cKey].fields || []);
|
|
70
|
+
const oldFieldsArr = oldMap[cKey].fields || [];
|
|
71
|
+
oldFieldsArr.forEach(fName => {
|
|
72
|
+
if (!newFieldsSet.has(fName)) {
|
|
73
|
+
removedFields.push({
|
|
74
|
+
component: cKey,
|
|
75
|
+
field: fName
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
return {
|
|
82
|
+
comparison: {
|
|
83
|
+
oldVersion: opts.oldVersion || '',
|
|
84
|
+
newVersion: opts.newVersion || '',
|
|
85
|
+
timestamp: opts.timestamp || new Date().toISOString()
|
|
86
|
+
},
|
|
87
|
+
summary: {
|
|
88
|
+
newComponents: newComponents.length,
|
|
89
|
+
removedComponents: removedComponents.length,
|
|
90
|
+
newFields: newFields.length,
|
|
91
|
+
removedFields: removedFields.length
|
|
92
|
+
},
|
|
93
|
+
details: {
|
|
94
|
+
newComponents,
|
|
95
|
+
removedComponents,
|
|
96
|
+
newFields,
|
|
97
|
+
removedFields
|
|
98
|
+
}
|
|
99
|
+
};
|
|
100
|
+
}
|
|
1
101
|
// tools/redpanda-connect/report-delta.js
|
|
2
102
|
'use strict';
|
|
3
103
|
|
|
@@ -149,4 +249,5 @@ module.exports = {
|
|
|
149
249
|
buildComponentMap,
|
|
150
250
|
getRpkConnectVersion,
|
|
151
251
|
printDeltaReport,
|
|
252
|
+
generateConnectorDiffJson,
|
|
152
253
|
};
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// This content is autogenerated. Do not edit manually. To override descriptions, use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
|
|
2
|
+
|
|
3
|
+
= {{name}}
|
|
4
|
+
|
|
5
|
+
{{#if signature}}
|
|
6
|
+
*Signature*: `{{signature}}`
|
|
7
|
+
{{/if}}
|
|
8
|
+
|
|
9
|
+
{{#if description}}
|
|
10
|
+
{{description}}
|
|
11
|
+
{{/if}}
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
{{#if examples}}
|
|
15
|
+
== Examples
|
|
16
|
+
|
|
17
|
+
{{#each examples}}
|
|
18
|
+
{{{bloblangExample this}}}
|
|
19
|
+
{{/each}}
|
|
20
|
+
{{/if}}
|
|
21
|
+
|
|
22
|
+
{{#if related}}
|
|
23
|
+
== Related
|
|
24
|
+
|
|
25
|
+
{{#each related}}
|
|
26
|
+
* {{this}}
|
|
27
|
+
{{/each}}
|
|
28
|
+
{{/if}}
|