@redpanda-data/docs-extensions-and-macros 4.7.4 → 4.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/doc-tools.js +90 -20
- package/package.json +1 -1
- package/tools/property-extractor/Makefile +64 -24
- package/tools/property-extractor/generate-handlebars-docs.js +344 -0
- package/tools/property-extractor/helpers/and.js +10 -0
- package/tools/property-extractor/helpers/eq.js +9 -0
- package/tools/property-extractor/helpers/formatPropertyValue.js +128 -0
- package/tools/property-extractor/helpers/formatUnits.js +26 -0
- package/tools/property-extractor/helpers/index.js +13 -0
- package/tools/property-extractor/helpers/join.js +18 -0
- package/tools/property-extractor/helpers/ne.js +9 -0
- package/tools/property-extractor/helpers/not.js +8 -0
- package/tools/property-extractor/helpers/or.js +10 -0
- package/tools/property-extractor/helpers/renderPropertyExample.js +42 -0
- package/tools/property-extractor/package-lock.json +77 -0
- package/tools/property-extractor/package.json +6 -0
- package/tools/property-extractor/property_extractor.py +1163 -20
- package/tools/property-extractor/requirements.txt +1 -0
- package/tools/property-extractor/templates/deprecated-properties.hbs +25 -0
- package/tools/property-extractor/templates/deprecated-property.hbs +7 -0
- package/tools/property-extractor/templates/property-page.hbs +22 -0
- package/tools/property-extractor/templates/property.hbs +70 -0
- package/tools/property-extractor/templates/topic-property.hbs +59 -0
- package/tools/property-extractor/topic_property_extractor.py +630 -0
- package/tools/property-extractor/transformers.py +80 -4
- package/tools/property-extractor/json-to-asciidoc/generate_docs.py +0 -466
package/bin/doc-tools.js
CHANGED
|
@@ -98,7 +98,7 @@ function requireCmd(cmd, help, versionFlag = '--version') {
|
|
|
98
98
|
* @param {number} [minMinor=10] - Minimum required minor version of Python.
|
|
99
99
|
*/
|
|
100
100
|
function requirePython(minMajor = 3, minMinor = 10) {
|
|
101
|
-
const candidates = ['python3', 'python'];
|
|
101
|
+
const candidates = ['python3', 'python', 'python3.12', 'python3.11', 'python3.10'];
|
|
102
102
|
for (const p of candidates) {
|
|
103
103
|
try {
|
|
104
104
|
const out = execSync(`${p} --version`, { encoding: 'utf8' }).trim();
|
|
@@ -234,12 +234,16 @@ For more details, visit: https://github.com/norwoodj/helm-docs
|
|
|
234
234
|
/**
|
|
235
235
|
* Ensures all dependencies required for generating property documentation are installed.
|
|
236
236
|
*
|
|
237
|
-
* Checks for the presence of `make`, Python 3.10 or newer, C++ compiler, and C++ standard library headers.
|
|
237
|
+
* Checks for the presence of `make`, Python 3.10 or newer, Node.js, C++ compiler, and C++ standard library headers.
|
|
238
238
|
* Exits the process with an error message if any dependency is missing.
|
|
239
239
|
*/
|
|
240
240
|
function verifyPropertyDependencies() {
|
|
241
241
|
requireCmd('make', 'Your OS package manager');
|
|
242
242
|
requirePython();
|
|
243
|
+
|
|
244
|
+
// Check for Node.js (required for Handlebars templates)
|
|
245
|
+
requireCmd('node', 'https://nodejs.org/en/download/ or use your package manager (e.g., brew install node)');
|
|
246
|
+
requireCmd('npm', 'Usually installed with Node.js');
|
|
243
247
|
|
|
244
248
|
// Check for C++ compiler
|
|
245
249
|
let cppCompiler = null;
|
|
@@ -463,25 +467,23 @@ function runClusterDocs(mode, tag, options) {
|
|
|
463
467
|
if (r.status !== 0) process.exit(r.status);
|
|
464
468
|
}
|
|
465
469
|
|
|
466
|
-
// helper to diff two
|
|
467
|
-
function diffDirs(kind, oldTag, newTag) {
|
|
468
|
-
const
|
|
469
|
-
const newDir = path.join('autogenerated', newTag, kind);
|
|
470
|
-
const diffDir = path.join('autogenerated', 'diffs', kind, `${oldTag}_to_${newTag}`);
|
|
470
|
+
// helper to diff two temporary directories
|
|
471
|
+
function diffDirs(kind, oldTag, newTag, oldTempDir, newTempDir) {
|
|
472
|
+
const diffDir = path.join('tmp', 'diffs', kind, `${oldTag}_to_${newTag}`);
|
|
471
473
|
const patch = path.join(diffDir, 'changes.patch');
|
|
472
474
|
|
|
473
|
-
if (!fs.existsSync(
|
|
474
|
-
console.error(`❌ Cannot diff: missing ${
|
|
475
|
+
if (!fs.existsSync(oldTempDir)) {
|
|
476
|
+
console.error(`❌ Cannot diff: missing ${oldTempDir}`);
|
|
475
477
|
process.exit(1);
|
|
476
478
|
}
|
|
477
|
-
if (!fs.existsSync(
|
|
478
|
-
console.error(`❌ Cannot diff: missing ${
|
|
479
|
+
if (!fs.existsSync(newTempDir)) {
|
|
480
|
+
console.error(`❌ Cannot diff: missing ${newTempDir}`);
|
|
479
481
|
process.exit(1);
|
|
480
482
|
}
|
|
481
483
|
|
|
482
484
|
fs.mkdirSync(diffDir, { recursive: true });
|
|
483
485
|
|
|
484
|
-
const cmd = `diff -ru "${
|
|
486
|
+
const cmd = `diff -ru "${oldTempDir}" "${newTempDir}" > "${patch}" || true`;
|
|
485
487
|
const res = spawnSync(cmd, { stdio: 'inherit', shell: true });
|
|
486
488
|
|
|
487
489
|
if (res.error) {
|
|
@@ -489,6 +491,10 @@ function diffDirs(kind, oldTag, newTag) {
|
|
|
489
491
|
process.exit(1);
|
|
490
492
|
}
|
|
491
493
|
console.log(`✅ Wrote patch: ${patch}`);
|
|
494
|
+
|
|
495
|
+
// Clean up temporary directories
|
|
496
|
+
fs.rmSync(oldTempDir, { recursive: true, force: true });
|
|
497
|
+
fs.rmSync(newTempDir, { recursive: true, force: true });
|
|
492
498
|
}
|
|
493
499
|
|
|
494
500
|
automation
|
|
@@ -747,15 +753,56 @@ automation
|
|
|
747
753
|
.description('Generate JSON and AsciiDoc documentation for Redpanda configuration properties')
|
|
748
754
|
.option('--tag <tag>', 'Git tag or branch to extract from', 'dev')
|
|
749
755
|
.option('--diff <oldTag>', 'Also diff autogenerated properties from <oldTag> → <tag>')
|
|
756
|
+
.option('--overrides <path>', 'Optional JSON file with property description overrides')
|
|
757
|
+
.option('--output-dir <dir>', 'Where to write all generated files', 'modules/reference')
|
|
758
|
+
.option('--template-property-page <path>', 'Custom Handlebars template for property page layout')
|
|
759
|
+
.option('--template-property <path>', 'Custom Handlebars template for individual property sections')
|
|
760
|
+
.option('--template-deprecated <path>', 'Custom Handlebars template for deprecated properties page')
|
|
761
|
+
.option('--template-deprecated-property <path>', 'Custom Handlebars template for individual deprecated property sections')
|
|
750
762
|
.action((options) => {
|
|
751
763
|
verifyPropertyDependencies();
|
|
752
764
|
|
|
753
765
|
const newTag = options.tag;
|
|
754
766
|
const oldTag = options.diff;
|
|
767
|
+
const overridesPath = options.overrides;
|
|
768
|
+
const outputDir = options.outputDir;
|
|
755
769
|
const cwd = path.resolve(__dirname, '../tools/property-extractor');
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
770
|
+
|
|
771
|
+
const make = (tag, overrides, templates = {}, outputDir = 'modules/reference/', tempDir = null) => {
|
|
772
|
+
console.log(`⏳ Building property docs for ${tag}${tempDir ? ' (for diff)' : ''}…`);
|
|
773
|
+
const args = ['build', `TAG=${tag}`];
|
|
774
|
+
|
|
775
|
+
// Pass all paths as environment variables for consistency
|
|
776
|
+
const env = { ...process.env };
|
|
777
|
+
if (overrides) {
|
|
778
|
+
env.OVERRIDES = path.resolve(overrides);
|
|
779
|
+
}
|
|
780
|
+
if (templates.propertyPage) {
|
|
781
|
+
env.TEMPLATE_PROPERTY_PAGE = path.resolve(templates.propertyPage);
|
|
782
|
+
}
|
|
783
|
+
if (templates.property) {
|
|
784
|
+
env.TEMPLATE_PROPERTY = path.resolve(templates.property);
|
|
785
|
+
}
|
|
786
|
+
if (templates.deprecated) {
|
|
787
|
+
env.TEMPLATE_DEPRECATED = path.resolve(templates.deprecated);
|
|
788
|
+
}
|
|
789
|
+
if (templates.deprecatedProperty) {
|
|
790
|
+
env.TEMPLATE_DEPRECATED_PROPERTY = path.resolve(templates.deprecatedProperty);
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
if (tempDir) {
|
|
794
|
+
// For diff purposes, generate to temporary directory
|
|
795
|
+
env.OUTPUT_ASCIIDOC_DIR = path.resolve(tempDir);
|
|
796
|
+
env.OUTPUT_JSON_DIR = path.resolve(tempDir, 'examples');
|
|
797
|
+
env.OUTPUT_AUTOGENERATED_DIR = path.resolve(tempDir);
|
|
798
|
+
} else {
|
|
799
|
+
// Normal generation - go directly to final destination
|
|
800
|
+
// Let Makefile calculate OUTPUT_ASCIIDOC_DIR as OUTPUT_AUTOGENERATED_DIR/pages
|
|
801
|
+
env.OUTPUT_JSON_DIR = path.resolve(outputDir, 'examples');
|
|
802
|
+
env.OUTPUT_AUTOGENERATED_DIR = path.resolve(outputDir);
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
const r = spawnSync('make', args, { cwd, stdio: 'inherit', env });
|
|
759
806
|
if (r.error) {
|
|
760
807
|
console.error(`❌ ${r.error.message}`);
|
|
761
808
|
process.exit(1);
|
|
@@ -763,15 +810,38 @@ automation
|
|
|
763
810
|
if (r.status !== 0) process.exit(r.status);
|
|
764
811
|
};
|
|
765
812
|
|
|
813
|
+
// Collect template options
|
|
814
|
+
const templates = {
|
|
815
|
+
propertyPage: options.templatePropertyPage,
|
|
816
|
+
property: options.templateProperty,
|
|
817
|
+
deprecated: options.templateDeprecated,
|
|
818
|
+
deprecatedProperty: options.templateDeprecatedProperty
|
|
819
|
+
};
|
|
820
|
+
|
|
821
|
+
let oldTempDir = null;
|
|
822
|
+
let newTempDir = null;
|
|
823
|
+
|
|
766
824
|
if (oldTag) {
|
|
767
|
-
|
|
768
|
-
|
|
825
|
+
// Generate old version to temporary directory for diff
|
|
826
|
+
oldTempDir = path.join('tmp', 'diff', `${oldTag}-properties`);
|
|
827
|
+
fs.mkdirSync(oldTempDir, { recursive: true });
|
|
828
|
+
make(oldTag, overridesPath, templates, outputDir, oldTempDir);
|
|
769
829
|
}
|
|
770
830
|
|
|
771
|
-
make(newTag);
|
|
772
|
-
|
|
773
831
|
if (oldTag) {
|
|
774
|
-
|
|
832
|
+
// Generate new version to temporary directory for diff
|
|
833
|
+
newTempDir = path.join('tmp', 'diff', `${newTag}-properties`);
|
|
834
|
+
fs.mkdirSync(newTempDir, { recursive: true });
|
|
835
|
+
make(newTag, overridesPath, templates, outputDir, newTempDir);
|
|
836
|
+
|
|
837
|
+
// Then generate new version to final destination
|
|
838
|
+
make(newTag, overridesPath, templates, outputDir);
|
|
839
|
+
|
|
840
|
+
// Compare the temporary directories
|
|
841
|
+
diffDirs('properties', oldTag, newTag, oldTempDir, newTempDir);
|
|
842
|
+
} else {
|
|
843
|
+
// No diff requested, just generate to final destination
|
|
844
|
+
make(newTag, overridesPath, templates, outputDir);
|
|
775
845
|
}
|
|
776
846
|
|
|
777
847
|
process.exit(0);
|
package/package.json
CHANGED
|
@@ -1,16 +1,28 @@
|
|
|
1
1
|
.PHONY: build venv clean redpanda-git treesitter generate-docs check
|
|
2
2
|
|
|
3
|
+
# --- Main build: venv, fetch code, build parser, extract & docgen ---
|
|
4
|
+
build: venv redpanda-git treesitter
|
|
5
|
+
@echo "🔧 Building with Redpanda tag: $(TAG)"
|
|
6
|
+
@mkdir -p $(TOOL_ROOT)/gen
|
|
7
|
+
@cd $(TOOL_ROOT) && \
|
|
8
|
+
$(PYTHON) -W ignore::FutureWarning property_extractor.py \
|
|
9
|
+
--recursive \
|
|
10
|
+
--path $(REDPANDA_SRC) \
|
|
11
|
+
--output gen/properties-output.json
|
|
12
|
+
@echo "✅ Cluster properties JSON generated at $(TOOL_ROOT)/gen/properties-output.json"
|
|
13
|
+
@$(MAKE) generate-docs
|
|
14
|
+
|
|
3
15
|
# Default tag (can be overridden via `make TAG=v25.1.1`)
|
|
4
16
|
TAG ?= dev
|
|
5
17
|
|
|
6
18
|
# Derive a “major.minor” or rc identifier from TAG for folder naming
|
|
7
19
|
VERSION := $(shell \
|
|
8
20
|
if echo "$(TAG)" | grep -qE '^v?[0-9]+\.[0-9]+'; then \
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
21
|
+
echo "$(TAG)" \
|
|
22
|
+
| sed -E 's/^v?([0-9]+\.[0-9]+)(\.[0-9]+)?(-rc[0-9]+)?.*/\1\3/' \
|
|
23
|
+
| sed 's/-rc/rc/'; \
|
|
12
24
|
else \
|
|
13
|
-
|
|
25
|
+
echo "$(TAG)"; \
|
|
14
26
|
fi)
|
|
15
27
|
|
|
16
28
|
# Paths
|
|
@@ -22,19 +34,31 @@ REDPANDA_SRC := $(TMP_ROOT)/redpanda
|
|
|
22
34
|
TREESITTER_DIR:= $(TOOL_ROOT)/tree-sitter/tree-sitter-cpp
|
|
23
35
|
VENV := $(TOOL_ROOT)/tmp/redpanda-property-extractor-venv
|
|
24
36
|
PYTHON := $(VENV)/bin/python
|
|
25
|
-
OUTPUT_DIR := $(REPO_ROOT)/autogenerated/$(TAG)/properties
|
|
26
37
|
TREE_SITTER := npx tree-sitter
|
|
27
38
|
|
|
39
|
+
# Output directory configuration (can be overridden by environment variables)
|
|
40
|
+
OUTPUT_AUTOGENERATED_DIR ?= $(REPO_ROOT)/modules/reference
|
|
41
|
+
OUTPUT_ASCIIDOC_DIR ?= $(OUTPUT_AUTOGENERATED_DIR)/pages
|
|
42
|
+
OUTPUT_JSON_DIR ?= $(OUTPUT_AUTOGENERATED_DIR)/examples
|
|
43
|
+
|
|
28
44
|
# --- Main build: venv, fetch code, build parser, extract & docgen ---
|
|
29
45
|
build: venv redpanda-git treesitter
|
|
30
46
|
@echo "🔧 Building with Redpanda tag: $(TAG)"
|
|
31
47
|
@mkdir -p $(TOOL_ROOT)/gen
|
|
48
|
+
@if [ -n "$(OVERRIDES)" ] && [ ! -f "$(OVERRIDES)" ]; then \
|
|
49
|
+
echo "❌ Error: Overrides file '$(OVERRIDES)' does not exist."; \
|
|
50
|
+
echo " Please check the path or omit the OVERRIDES variable to continue without overrides."; \
|
|
51
|
+
exit 1; \
|
|
52
|
+
fi
|
|
32
53
|
@cd $(TOOL_ROOT) && \
|
|
33
54
|
$(PYTHON) -W ignore::FutureWarning property_extractor.py \
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
55
|
+
--recursive \
|
|
56
|
+
--path $(REDPANDA_SRC) \
|
|
57
|
+
--output gen/properties-output.json \
|
|
58
|
+
--enhanced-output gen/$(TAG)-properties.json \
|
|
59
|
+
$(if $(OVERRIDES),$(if $(shell [ -f "$(OVERRIDES)" ] && echo exists),--overrides $(OVERRIDES),),)
|
|
37
60
|
@echo "✅ JSON generated at $(TOOL_ROOT)/gen/properties-output.json"
|
|
61
|
+
@echo "✅ Enhanced JSON (with overrides) generated at $(TOOL_ROOT)/gen/$(TAG)-properties.json"
|
|
38
62
|
@$(MAKE) generate-docs
|
|
39
63
|
|
|
40
64
|
# --- Ensure Python venv & dependencies ---
|
|
@@ -84,22 +108,38 @@ treesitter:
|
|
|
84
108
|
@echo "🔧 Generating parser in $(TREESITTER_DIR)…"
|
|
85
109
|
@cd "$(TREESITTER_DIR)" && npm install --silent && $(TREE_SITTER) generate
|
|
86
110
|
|
|
87
|
-
# ---
|
|
88
|
-
|
|
89
|
-
@echo "
|
|
90
|
-
@
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
@
|
|
95
|
-
@
|
|
111
|
+
# --- Install Node.js dependencies for Handlebars ---
|
|
112
|
+
node-deps:
|
|
113
|
+
@echo "📦 Installing Node.js dependencies…"
|
|
114
|
+
@cd $(TOOL_ROOT) && npm install --silent
|
|
115
|
+
|
|
116
|
+
# --- Turn the JSON into AsciiDoc pages under the specified output directories ---
|
|
117
|
+
generate-docs: node-deps
|
|
118
|
+
@echo "📝 Generating AsciiDoc pages in $(OUTPUT_ASCIIDOC_DIR)…"
|
|
119
|
+
@mkdir -p "$(OUTPUT_ASCIIDOC_DIR)"
|
|
120
|
+
@mkdir -p "$(OUTPUT_JSON_DIR)"
|
|
121
|
+
@# Use the enhanced properties file (with overrides) for documentation generation if it exists
|
|
122
|
+
@if [ -f "$(TOOL_ROOT)/gen/$(TAG)-properties.json" ]; then \
|
|
123
|
+
cd $(TOOL_ROOT) && \
|
|
124
|
+
node generate-handlebars-docs.js "gen/$(TAG)-properties.json" "$(OUTPUT_AUTOGENERATED_DIR)"; \
|
|
125
|
+
else \
|
|
126
|
+
cd $(TOOL_ROOT) && \
|
|
127
|
+
node generate-handlebars-docs.js "gen/properties-output.json" "$(OUTPUT_AUTOGENERATED_DIR)"; \
|
|
128
|
+
fi
|
|
129
|
+
@echo "📄 Copying properties JSON files to $(OUTPUT_JSON_DIR)…"
|
|
130
|
+
@if [ -f "$(TOOL_ROOT)/gen/$(TAG)-properties.json" ]; then \
|
|
131
|
+
cp "$(TOOL_ROOT)/gen/$(TAG)-properties.json" "$(OUTPUT_JSON_DIR)/"; \
|
|
132
|
+
fi
|
|
133
|
+
@echo "✅ Docs generated at $(OUTPUT_AUTOGENERATED_DIR)"
|
|
96
134
|
|
|
97
135
|
# --- Debug helper to print all the key paths/vars ---
|
|
98
136
|
check:
|
|
99
|
-
@echo "MODULE_ROOT:
|
|
100
|
-
@echo "TOOL_ROOT:
|
|
101
|
-
@echo "REDPANDA_SRC:
|
|
102
|
-
@echo "TREESITTER:
|
|
103
|
-
@echo "VENV:
|
|
104
|
-
@echo "PYTHON:
|
|
105
|
-
@echo "
|
|
137
|
+
@echo "MODULE_ROOT: $(MODULE_ROOT)"
|
|
138
|
+
@echo "TOOL_ROOT: $(TOOL_ROOT)"
|
|
139
|
+
@echo "REDPANDA_SRC: $(REDPANDA_SRC)"
|
|
140
|
+
@echo "TREESITTER: $(TREESITTER_DIR)"
|
|
141
|
+
@echo "VENV: $(VENV)"
|
|
142
|
+
@echo "PYTHON: $(PYTHON)"
|
|
143
|
+
@echo "OUTPUT_ASCIIDOC_DIR: $(OUTPUT_ASCIIDOC_DIR)"
|
|
144
|
+
@echo "OUTPUT_JSON_DIR: $(OUTPUT_JSON_DIR)"
|
|
145
|
+
@echo "OUTPUT_AUTOGENERATED_DIR: $(OUTPUT_AUTOGENERATED_DIR)"
|
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const handlebars = require('handlebars');
|
|
6
|
+
const helpers = require('./helpers');
|
|
7
|
+
|
|
8
|
+
// Register all helpers
|
|
9
|
+
Object.entries(helpers).forEach(([name, fn]) => {
|
|
10
|
+
if (typeof fn !== 'function') {
|
|
11
|
+
console.error(`❌ Helper "${name}" is not a function`);
|
|
12
|
+
process.exit(1);
|
|
13
|
+
}
|
|
14
|
+
handlebars.registerHelper(name, fn);
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Configuration mapping for different property types
|
|
19
|
+
*/
|
|
20
|
+
const PROPERTY_CONFIG = {
|
|
21
|
+
broker: {
|
|
22
|
+
pageTitle: 'Broker Configuration Properties',
|
|
23
|
+
pageAliases: ['reference:node-properties.adoc', 'reference:node-configuration-sample.adoc'],
|
|
24
|
+
description: 'Reference of broker configuration properties.',
|
|
25
|
+
intro: `Broker configuration properties are applied individually to each broker in a cluster. You can find and modify these properties in the \`redpanda.yaml\` configuration file.
|
|
26
|
+
|
|
27
|
+
For information on how to edit broker properties, see xref:manage:cluster-maintenance/node-property-configuration.adoc[].
|
|
28
|
+
|
|
29
|
+
NOTE: All broker properties require that you restart Redpanda for any update to take effect.`,
|
|
30
|
+
sectionTitle: 'Broker configuration',
|
|
31
|
+
groups: [
|
|
32
|
+
{
|
|
33
|
+
filter: (prop) => prop.config_scope === 'broker' && !prop.is_deprecated
|
|
34
|
+
}
|
|
35
|
+
],
|
|
36
|
+
filename: 'broker-properties.adoc'
|
|
37
|
+
},
|
|
38
|
+
cluster: {
|
|
39
|
+
pageTitle: 'Cluster Configuration Properties',
|
|
40
|
+
pageAliases: ['reference:tunable-properties.adoc', 'reference:cluster-properties.adoc'],
|
|
41
|
+
description: 'Cluster configuration properties list.',
|
|
42
|
+
intro: `Cluster configuration properties are the same for all brokers in a cluster, and are set at the cluster level.
|
|
43
|
+
|
|
44
|
+
For information on how to edit cluster properties, see xref:manage:cluster-maintenance/cluster-property-configuration.adoc[] or xref:manage:kubernetes/k-cluster-property-configuration.adoc[].
|
|
45
|
+
|
|
46
|
+
NOTE: Some cluster properties require that you restart the cluster for any updates to take effect. See the specific property details to identify whether or not a restart is required.`,
|
|
47
|
+
sectionTitle: 'Cluster configuration',
|
|
48
|
+
groups: [
|
|
49
|
+
{
|
|
50
|
+
filter: (prop) => prop.config_scope === 'cluster' && !prop.is_deprecated
|
|
51
|
+
}
|
|
52
|
+
],
|
|
53
|
+
filename: 'cluster-properties.adoc'
|
|
54
|
+
},
|
|
55
|
+
'object-storage': {
|
|
56
|
+
pageTitle: 'Object Storage Properties',
|
|
57
|
+
description: 'Reference of object storage properties.',
|
|
58
|
+
intro: `Object storage properties are a type of cluster property. For information on how to edit cluster properties, see xref:manage:cluster-maintenance/cluster-property-configuration.adoc[].
|
|
59
|
+
|
|
60
|
+
NOTE: Some object storage properties require that you restart the cluster for any updates to take effect. See the specific property details to identify whether or not a restart is required.`,
|
|
61
|
+
sectionTitle: 'Object storage configuration',
|
|
62
|
+
sectionIntro: 'Object storage properties should only be set if you enable xref:manage:tiered-storage.adoc[Tiered Storage].',
|
|
63
|
+
groups: [
|
|
64
|
+
{
|
|
65
|
+
filter: (prop) => prop.name && (
|
|
66
|
+
prop.name.includes('cloud_storage') ||
|
|
67
|
+
prop.name.includes('s3_') ||
|
|
68
|
+
prop.name.includes('azure_') ||
|
|
69
|
+
prop.name.includes('gcs_') ||
|
|
70
|
+
prop.name.includes('archival_') ||
|
|
71
|
+
prop.name.includes('remote_') ||
|
|
72
|
+
prop.name.includes('tiered_')
|
|
73
|
+
) && !prop.is_deprecated
|
|
74
|
+
}
|
|
75
|
+
],
|
|
76
|
+
filename: 'object-storage-properties.adoc'
|
|
77
|
+
},
|
|
78
|
+
topic: {
|
|
79
|
+
pageTitle: 'Topic Configuration Properties',
|
|
80
|
+
pageAliases: ['reference:topic-properties.adoc'],
|
|
81
|
+
description: 'Reference of topic configuration properties.',
|
|
82
|
+
intro: `A topic-level property sets a Redpanda or Kafka configuration for a particular topic.
|
|
83
|
+
|
|
84
|
+
Many topic-level properties have corresponding xref:manage:cluster-maintenance/cluster-property-configuration.adoc[cluster properties] that set a default value for all topics of a cluster. To customize the value for a topic, you can set a topic-level property that overrides the value of the corresponding cluster property.
|
|
85
|
+
|
|
86
|
+
NOTE: All topic properties take effect immediately after being set.`,
|
|
87
|
+
sectionTitle: 'Topic configuration',
|
|
88
|
+
groups: [
|
|
89
|
+
{
|
|
90
|
+
filter: (prop) => prop.config_scope === 'topic' && !prop.is_deprecated,
|
|
91
|
+
template: 'topic-property'
|
|
92
|
+
}
|
|
93
|
+
],
|
|
94
|
+
filename: 'topic-properties.adoc'
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
// "src/v/kafka/server/handlers/topics/types.cc": "topic"
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Gets template path, checking environment variables for custom paths first
|
|
102
|
+
*/
|
|
103
|
+
function getTemplatePath(defaultPath, envVar) {
|
|
104
|
+
const customPath = process.env[envVar];
|
|
105
|
+
if (customPath && fs.existsSync(customPath)) {
|
|
106
|
+
console.log(`📄 Using custom template: ${customPath}`);
|
|
107
|
+
return customPath;
|
|
108
|
+
}
|
|
109
|
+
return defaultPath;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Registers Handlebars partials from template files
|
|
114
|
+
*/
|
|
115
|
+
function registerPartials() {
|
|
116
|
+
const templatesDir = path.join(__dirname, 'templates');
|
|
117
|
+
|
|
118
|
+
// Register property partial
|
|
119
|
+
const propertyTemplatePath = getTemplatePath(
|
|
120
|
+
path.join(templatesDir, 'property.hbs'),
|
|
121
|
+
'TEMPLATE_PROPERTY'
|
|
122
|
+
);
|
|
123
|
+
const propertyTemplate = fs.readFileSync(propertyTemplatePath, 'utf8');
|
|
124
|
+
handlebars.registerPartial('property', propertyTemplate);
|
|
125
|
+
|
|
126
|
+
// Register topic property partial
|
|
127
|
+
const topicPropertyTemplatePath = getTemplatePath(
|
|
128
|
+
path.join(templatesDir, 'topic-property.hbs'),
|
|
129
|
+
'TEMPLATE_TOPIC_PROPERTY'
|
|
130
|
+
);
|
|
131
|
+
const topicPropertyTemplate = fs.readFileSync(topicPropertyTemplatePath, 'utf8');
|
|
132
|
+
handlebars.registerPartial('topic-property', topicPropertyTemplate);
|
|
133
|
+
|
|
134
|
+
// Register deprecated property partial
|
|
135
|
+
const deprecatedPropertyTemplatePath = getTemplatePath(
|
|
136
|
+
path.join(templatesDir, 'deprecated-property.hbs'),
|
|
137
|
+
'TEMPLATE_DEPRECATED_PROPERTY'
|
|
138
|
+
);
|
|
139
|
+
const deprecatedPropertyTemplate = fs.readFileSync(deprecatedPropertyTemplatePath, 'utf8');
|
|
140
|
+
handlebars.registerPartial('deprecated-property', deprecatedPropertyTemplate);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Generates documentation for a specific property type
|
|
145
|
+
*/
|
|
146
|
+
function generatePropertyDocs(properties, config, outputDir) {
|
|
147
|
+
const templatePath = getTemplatePath(
|
|
148
|
+
path.join(__dirname, 'templates', 'property-page.hbs'),
|
|
149
|
+
'TEMPLATE_PROPERTY_PAGE'
|
|
150
|
+
);
|
|
151
|
+
const template = handlebars.compile(fs.readFileSync(templatePath, 'utf8'));
|
|
152
|
+
|
|
153
|
+
// Filter and group properties according to configuration
|
|
154
|
+
const groups = config.groups.map(group => {
|
|
155
|
+
const filteredProperties = Object.values(properties)
|
|
156
|
+
.filter(prop => group.filter(prop))
|
|
157
|
+
.sort((a, b) => String(a.name || '').localeCompare(String(b.name || '')));
|
|
158
|
+
|
|
159
|
+
return {
|
|
160
|
+
title: group.title,
|
|
161
|
+
intro: group.intro,
|
|
162
|
+
properties: filteredProperties,
|
|
163
|
+
template: group.template || 'property' // Default to 'property' template
|
|
164
|
+
};
|
|
165
|
+
}).filter(group => group.properties.length > 0);
|
|
166
|
+
|
|
167
|
+
const data = {
|
|
168
|
+
...config,
|
|
169
|
+
groups
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
const output = template(data);
|
|
173
|
+
const outputPath = path.join(outputDir, config.filename);
|
|
174
|
+
|
|
175
|
+
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
|
176
|
+
fs.writeFileSync(outputPath, output, 'utf8');
|
|
177
|
+
|
|
178
|
+
console.log(`✅ Generated ${outputPath}`);
|
|
179
|
+
return groups.reduce((total, group) => total + group.properties.length, 0);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Generates deprecated properties documentation
|
|
184
|
+
*/
|
|
185
|
+
function generateDeprecatedDocs(properties, outputDir) {
|
|
186
|
+
const templatePath = getTemplatePath(
|
|
187
|
+
path.join(__dirname, 'templates', 'deprecated-properties.hbs'),
|
|
188
|
+
'TEMPLATE_DEPRECATED'
|
|
189
|
+
);
|
|
190
|
+
const template = handlebars.compile(fs.readFileSync(templatePath, 'utf8'));
|
|
191
|
+
|
|
192
|
+
const deprecatedProperties = Object.values(properties).filter(prop => prop.is_deprecated);
|
|
193
|
+
|
|
194
|
+
const brokerProperties = deprecatedProperties
|
|
195
|
+
.filter(prop => prop.config_scope === 'broker')
|
|
196
|
+
.sort((a, b) => String(a.name || '').localeCompare(String(b.name || '')));
|
|
197
|
+
|
|
198
|
+
const clusterProperties = deprecatedProperties
|
|
199
|
+
.filter(prop => prop.config_scope === 'cluster')
|
|
200
|
+
.sort((a, b) => String(a.name || '').localeCompare(String(b.name || '')));
|
|
201
|
+
|
|
202
|
+
const data = {
|
|
203
|
+
deprecated: deprecatedProperties.length > 0,
|
|
204
|
+
brokerProperties: brokerProperties.length > 0 ? brokerProperties : null,
|
|
205
|
+
clusterProperties: clusterProperties.length > 0 ? clusterProperties : null
|
|
206
|
+
};
|
|
207
|
+
|
|
208
|
+
const output = template(data);
|
|
209
|
+
const outputPath = path.join(outputDir, 'deprecated', 'partials', 'deprecated-properties.adoc');
|
|
210
|
+
|
|
211
|
+
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
|
212
|
+
fs.writeFileSync(outputPath, output, 'utf8');
|
|
213
|
+
|
|
214
|
+
console.log(`✅ Generated ${outputPath}`);
|
|
215
|
+
return deprecatedProperties.length;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Main function to generate all property documentation
|
|
220
|
+
*/
|
|
221
|
+
function generateAllDocs(inputFile, outputDir) {
|
|
222
|
+
// Register partials
|
|
223
|
+
registerPartials();
|
|
224
|
+
|
|
225
|
+
// Read input JSON
|
|
226
|
+
const data = JSON.parse(fs.readFileSync(inputFile, 'utf8'));
|
|
227
|
+
const properties = data.properties || {};
|
|
228
|
+
|
|
229
|
+
let totalProperties = 0;
|
|
230
|
+
let totalBrokerProperties = 0;
|
|
231
|
+
let totalClusterProperties = 0;
|
|
232
|
+
let totalObjectStorageProperties = 0;
|
|
233
|
+
let totalTopicProperties = 0;
|
|
234
|
+
|
|
235
|
+
// Generate each type of documentation
|
|
236
|
+
for (const [type, config] of Object.entries(PROPERTY_CONFIG)) {
|
|
237
|
+
const count = generatePropertyDocs(properties, config, path.join(outputDir, 'pages'));
|
|
238
|
+
totalProperties += count;
|
|
239
|
+
|
|
240
|
+
if (type === 'broker') totalBrokerProperties = count;
|
|
241
|
+
else if (type === 'cluster') totalClusterProperties = count;
|
|
242
|
+
else if (type === 'object-storage') totalObjectStorageProperties = count;
|
|
243
|
+
else if (type === 'topic') totalTopicProperties = count;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// Generate deprecated properties documentation
|
|
247
|
+
const deprecatedCount = generateDeprecatedDocs(properties, path.join(outputDir, 'pages'));
|
|
248
|
+
|
|
249
|
+
// Generate summary file
|
|
250
|
+
const allPropertiesContent = Object.keys(properties).sort().join('\n');
|
|
251
|
+
fs.writeFileSync(path.join(outputDir, 'all_properties.txt'), allPropertiesContent, 'utf8');
|
|
252
|
+
|
|
253
|
+
// Generate error reports
|
|
254
|
+
generateErrorReports(properties, outputDir);
|
|
255
|
+
|
|
256
|
+
console.log(`📊 Generation Summary:`);
|
|
257
|
+
console.log(` Total properties read: ${Object.keys(properties).length}`);
|
|
258
|
+
console.log(` Total Broker properties: ${totalBrokerProperties}`);
|
|
259
|
+
console.log(` Total Cluster properties: ${totalClusterProperties}`);
|
|
260
|
+
console.log(` Total Object Storage properties: ${totalObjectStorageProperties}`);
|
|
261
|
+
console.log(` Total Topic properties: ${totalTopicProperties}`);
|
|
262
|
+
console.log(` Total Deprecated properties: ${deprecatedCount}`);
|
|
263
|
+
|
|
264
|
+
return {
|
|
265
|
+
totalProperties: Object.keys(properties).length,
|
|
266
|
+
brokerProperties: totalBrokerProperties,
|
|
267
|
+
clusterProperties: totalClusterProperties,
|
|
268
|
+
objectStorageProperties: totalObjectStorageProperties,
|
|
269
|
+
topicProperties: totalTopicProperties,
|
|
270
|
+
deprecatedProperties: deprecatedCount
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Generate error reports for properties with missing or invalid data
|
|
276
|
+
*/
|
|
277
|
+
function generateErrorReports(properties, outputDir) {
|
|
278
|
+
const errorDir = path.join(outputDir, 'error');
|
|
279
|
+
fs.mkdirSync(errorDir, { recursive: true });
|
|
280
|
+
|
|
281
|
+
const emptyDescriptions = [];
|
|
282
|
+
const deprecatedProperties = [];
|
|
283
|
+
|
|
284
|
+
Object.values(properties).forEach(prop => {
|
|
285
|
+
if (!prop.description || prop.description.trim() === '') {
|
|
286
|
+
emptyDescriptions.push(prop.name);
|
|
287
|
+
}
|
|
288
|
+
if (prop.is_deprecated) {
|
|
289
|
+
deprecatedProperties.push(prop.name);
|
|
290
|
+
}
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
// Write error reports
|
|
294
|
+
if (emptyDescriptions.length > 0) {
|
|
295
|
+
fs.writeFileSync(
|
|
296
|
+
path.join(errorDir, 'empty_description.txt'),
|
|
297
|
+
emptyDescriptions.join('\n'),
|
|
298
|
+
'utf8'
|
|
299
|
+
);
|
|
300
|
+
const percentage = ((emptyDescriptions.length / Object.keys(properties).length) * 100).toFixed(2);
|
|
301
|
+
console.log(`You have ${emptyDescriptions.length} properties with empty description. Percentage of errors: ${percentage}%. Data written in 'empty_description.txt'.`);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
if (deprecatedProperties.length > 0) {
|
|
305
|
+
fs.writeFileSync(
|
|
306
|
+
path.join(errorDir, 'deprecated_properties.txt'),
|
|
307
|
+
deprecatedProperties.join('\n'),
|
|
308
|
+
'utf8'
|
|
309
|
+
);
|
|
310
|
+
const percentage = ((deprecatedProperties.length / Object.keys(properties).length) * 100).toFixed(2);
|
|
311
|
+
console.log(`You have ${deprecatedProperties.length} deprecated properties. Percentage of errors: ${percentage}%. Data written in 'deprecated_properties.txt'.`);
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
module.exports = {
|
|
316
|
+
generateAllDocs,
|
|
317
|
+
generatePropertyDocs,
|
|
318
|
+
generateDeprecatedDocs,
|
|
319
|
+
PROPERTY_CONFIG
|
|
320
|
+
};
|
|
321
|
+
|
|
322
|
+
// CLI interface
|
|
323
|
+
if (require.main === module) {
|
|
324
|
+
const args = process.argv.slice(2);
|
|
325
|
+
if (args.length < 2) {
|
|
326
|
+
console.error('Usage: node generate-handlebars-docs.js <input-file> <output-dir>');
|
|
327
|
+
process.exit(1);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
const [inputFile, outputDir] = args;
|
|
331
|
+
|
|
332
|
+
if (!fs.existsSync(inputFile)) {
|
|
333
|
+
console.error(`❌ Input file not found: ${inputFile}`);
|
|
334
|
+
process.exit(1);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
try {
|
|
338
|
+
generateAllDocs(inputFile, outputDir);
|
|
339
|
+
console.log('✅ Documentation generation completed successfully');
|
|
340
|
+
} catch (error) {
|
|
341
|
+
console.error(`❌ Error generating documentation: ${error.message}`);
|
|
342
|
+
process.exit(1);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Handlebars helper for logical AND
|
|
3
|
+
* @param {...*} args - Values to check
|
|
4
|
+
* @returns {boolean} True if all values are truthy per JavaScript semantics
|
|
5
|
+
*/
|
|
6
|
+
module.exports = function and(...args) {
|
|
7
|
+
// Remove the last argument which is the Handlebars options object
|
|
8
|
+
const values = args.slice(0, -1);
|
|
9
|
+
return values.every(val => Boolean(val));
|
|
10
|
+
};
|