@redpanda-data/docs-extensions-and-macros 4.2.5 → 4.4.0

package/tools/property-extractor/property_extractor.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+import logging
+import sys
+import os
+import json
+import re
+
+from pathlib import Path
+from file_pair import FilePair
+from tree_sitter import Language, Parser
+
+from parser import build_treesitter_cpp_library, extract_properties_from_file_pair
+from property_bag import PropertyBag
+from transformers import *
+
+logger = logging.getLogger("viewer")
+
+
+def validate_paths(options):
+    path = options.path
+
+    if not os.path.exists(path):
+        logger.error(f'Path does not exist: "{path}".')
+        sys.exit(1)
+
+    if options.definitions and not os.path.exists(options.definitions):
+        logger.error(
+            f'File with the type definitions not found: "{options.definitions}".'
+        )
+        sys.exit(1)
+
+
+def get_file_pairs(options):
+    path = Path(options.path)
+
+    file_iter = path.rglob("*.h") if options.recursive else path.rglob("*.h")
+
+    file_pairs = []
+
+    for i in file_iter:
+        if os.path.exists(i.with_suffix(".cc")):
+            file_pairs.append(FilePair(i.resolve(), i.with_suffix(".cc").resolve()))
+
+    return file_pairs
+
+
+def get_treesitter_cpp_parser_and_language(treesitter_dir, destination_path):
+    if not os.path.exists(destination_path):
+        build_treesitter_cpp_library(treesitter_dir, destination_path)
+
+    cpp = Language(destination_path, "cpp")
+
+    parser = Parser()
+    parser.set_language(cpp)
+
+    return parser, cpp
+
+
+def get_files_with_properties(file_pairs, treesitter_parser, cpp_language):
+    files_with_properties = []
+
+    for fp in file_pairs:
+        properties = extract_properties_from_file_pair(
+            treesitter_parser, cpp_language, fp
+        )
+        implementation_value = str(fp.implementation)
+
+        # List of file paths to check
+        file_paths = [
+            "src/v/config/configuration.cc",
+            "src/v/config/node_config.cc",
+            "src/v/kafka/client/configuration.cc",
+            "src/v/pandaproxy/rest/configuration.cc",
+            "src/v/pandaproxy/schema_registry/configuration.cc"
+        ]
+
+        # Check if any of the paths are in fp.implementation
+        if any(path in implementation_value for path in file_paths):
+            if len(properties) > 0:
+                files_with_properties.append((fp, properties))
+
+    
+
+    return files_with_properties
+
+
+def transform_files_with_properties(files_with_properties):
+    type_transformer = TypeTransformer()
+    transformers = [
+        EnterpriseTransformer(), ## this must be the first, as it modifies current data
+        TypeTransformer(),
+        MetaParamTransformer(),
+        BasicInfoTransformer(),
+        IsNullableTransformer(),
+        IsArrayTransformer(type_transformer),
+        NeedsRestartTransformer(),
+        GetsRestoredTransformer(),
+        VisibilityTransformer(),
+        DeprecatedTransformer(),
+        IsSecretTransformer(),
+        NumericBoundsTransformer(type_transformer),
+        DurationBoundsTransformer(type_transformer),
+        SimpleDefaultValuesTransformer(),
+        FriendlyDefaultTransformer(),
+        ExperimentalTransformer(),
+        AliasTransformer(),
+    ]
+
+    all_properties = PropertyBag()
+
+    for fp, properties in files_with_properties:
+        for name in properties:
+            # ignore private properties
+            if re.match(r"^_", name):
+                continue
+
+            property_definition = PropertyBag()
+
+            for transformer in transformers:
+                if transformer.accepts(properties[name], fp):
+                    transformer.parse(property_definition, properties[name], fp)
+
+            if len(property_definition) > 0:
+                all_properties[name] = property_definition
+
+    return all_properties
+
+
+# The definitions.json file contains type definitions that the extractor uses to standardize and centralize type information. After extracting and transforming the properties from the source code, the function merge_properties_and_definitions looks up each property's type in the definitions. If a property's type (or the type of its items, in the case of arrays) matches one of the definitions, the transformer replaces that type with a JSON pointer ( such as #/definitions/<type>) to the corresponding entry in definitions.json. The final JSON output then includes both a properties section (with types now referencing the definitions) and a definitions section, so that consumers of the output can easily resolve the full type information.
+def merge_properties_and_definitions(properties, definitions):
+    for name in properties:
+        property = properties[name]
+        # guard against missing "type"
+        prop_type = property.get("type")
+        if prop_type and prop_type in definitions:
+            properties[name]["type"] = "#/definitions/{prop_type}"
+        elif (
+            prop_type == "array"
+            and property.get("items", {}).get("type") in definitions
+        ):
+            properties[name]["items"]["type"] = (
+                f"#/definitions/{property['items']['type']}"
+            )
+
+    return dict(properties=properties, definitions=definitions)
+
+
+def main():
+    import argparse
+
+    def generate_options():
+        arg_parser = argparse.ArgumentParser(
+            description="Extract all properties from the Redpanda's source code and generate a JSON output with their definitions"
+        )
+        arg_parser.add_argument(
+            "--path",
+            type=str,
+            required=True,
+            help="Path to the Redpanda's source dir to extract the properties",
+        )
+
+        arg_parser.add_argument(
+            "--recursive", action="store_true", help="Scan the path recursively"
+        )
+
+        arg_parser.add_argument(
+            "--output",
+            type=str,
+            required=False,
+            help="File to store the JSON output. If no file is provided, the JSON will be printed to the standard output",
+        )
+
+        arg_parser.add_argument(
+            "--definitions",
+            type=str,
+            required=False,
+            default=os.path.dirname(os.path.realpath(__file__)) + "/definitions.json",
+            help='JSON file with the type definitions. This file will be merged in the output under the "definitions" field',
+        )
+
+        arg_parser.add_argument("-v", "--verbose", action="store_true")
+
+        return arg_parser
+
+    arg_parser = generate_options()
+    options, _ = arg_parser.parse_known_args()
+
+    if options.verbose:
+        logging.basicConfig(level="DEBUG")
+    else:
+        logging.basicConfig(level="INFO")
+
+    validate_paths(options)
+
+    file_pairs = get_file_pairs(options)
+
+    if not file_pairs:
+        logging.error("No h/cc file pairs were found")
+        sys.exit(-1)
+
+    definitions = None
+
+    if options.definitions:
+        try:
+            with open(options.definitions) as json_file:
+                definitions = json.load(json_file)
+        except json.JSONDecodeError as e:
+            logging.error(f"Failed to parse definitions file: {e}")
+            sys.exit(1)
+
+    treesitter_dir = os.path.join(os.getcwd(), "tree-sitter/tree-sitter-cpp")
+    destination_path = os.path.join(treesitter_dir, "tree-sitter-cpp.so")
+
+    if not os.path.exists(os.path.join(treesitter_dir, "src/parser.c")):
+        logging.error("Missing parser.c. Ensure Tree-sitter submodules are initialized.")
+        sys.exit(1)
+
+    treesitter_parser, cpp_language = get_treesitter_cpp_parser_and_language(
+        treesitter_dir, destination_path
+    )
+
+    files_with_properties = get_files_with_properties(
+        file_pairs, treesitter_parser, cpp_language
+    )
+    properties = transform_files_with_properties(files_with_properties)
+    properties_and_definitions = merge_properties_and_definitions(
+        properties, definitions
+    )
+
+    json_output = json.dumps(properties_and_definitions, indent=4, sort_keys=True)
+
+    if options.output:
+        try:
+            with open(options.output, "w+") as json_file:
+                json_file.write(json_output)
+        except IOError as e:
+            logging.error(f"Failed to write output file: {e}")
+            sys.exit(1)
+    else:
+        print(json_output)
+
+if __name__ == "__main__":
+    main()

package/tools/property-extractor/requirements.txt

@@ -0,0 +1,2 @@
+tree_sitter==0.21.1
+setuptools>=42.0.0

package/tools/property-extractor/tests/transformers_test.py

@@ -0,0 +1,376 @@
+import unittest
+from property_bag import PropertyBag
+from file_pair import FilePair
+from transformers import *
+
+
+def create_property_info(
+    name,
+    description,
+    declaration=None,
+    metadata=None,
+    default_value="{}",
+    default_value_type="initializer_list",
+):
+    info = PropertyBag()
+    info["name_in_file"] = name
+    info["declaration"] = declaration
+    info["params"] = []
+    info["params"].append(PropertyBag(value=name, type="string_literal"))
+    info["params"].append(PropertyBag(value=description, type="string_literal"))
+    info["params"].append(PropertyBag(value=metadata, type="initializer_list"))
+    info["params"].append(PropertyBag(value=default_value, type="initializer_list"))
+
+    return info
+
+
+class BasicInfoTransformerTestCase(unittest.TestCase):
+    def setUp(self):
+        self.transformer = BasicInfoTransformer()
+
+    def test_accepts_everything(self):
+        self.assertTrue(self.transformer.accepts(None, None))
+
+    def test_adds_name_description_and_defined_in(self):
+        property = PropertyBag()
+        info = create_property_info("test_property", "test description")
+        self.transformer.parse(property, info, FilePair("testfile.h", "testfile.cc"))
+
+        self.assertEqual("test_property", property["name"])
+        self.assertEqual("test description", property["description"])
+        self.assertEqual("testfile.cc", property["defined_in"])
+
+
+class IsArrayTransformerTestCase(unittest.TestCase):
+    def setUp(self):
+        self.transformer = IsArrayTransformer(TypeTransformer())
+
+    def test_accepts_vector(self):
+        self.assertTrue(
+            self.transformer.accepts(
+                create_property_info(
+                    "test_property", "test description", "std::vector<int>"
+                ),
+                None,
+            )
+        )
+
+    def test_rejects_other_properties(self):
+        for t in [
+            "int",
+            "std::optional",
+            "config::endpoint_tls_config",
+            "deprecated_property",
+        ]:
+            with self.subTest(t):
+                self.assertFalse(
+                    self.transformer.accepts(
+                        create_property_info("test_property", "test description", t),
+                        None,
+                    )
+                )
+
+    def test_adds_array_field(self):
+        property = PropertyBag()
+        info = create_property_info(
+            "test_property", "test description", "std::vector<int>"
+        )
+        self.transformer.parse(property, info, FilePair("testfile.h", "testfile.cc"))
+
+        self.assertEqual("array", property["type"])
+        self.assertEqual("integer", property["items"]["type"])
+
+
+class NeedsRestartTransformerTestCase(unittest.TestCase):
+    def setUp(self):
+        self.transformer = NeedsRestartTransformer()
+
+    def test_accepts_needs_restart_metadata(self):
+        self.assertTrue(
+            self.transformer.accepts(
+                create_property_info(
+                    "test_property",
+                    "test description",
+                    "bool",
+                    PropertyBag(needs_restart="needs_restart::no"),
+                ),
+                None,
+            )
+        )
+        self.assertTrue(
+            self.transformer.accepts(
+                create_property_info(
+                    "test_property",
+                    "test description",
+                    "bool",
+                    PropertyBag(needs_restart="needs_restart::yes"),
+                ),
+                None,
+            )
+        )
+
+    def test_rejects_without_needs_restart(self):
+        self.assertFalse(
+            self.transformer.accepts(
+                create_property_info(
+                    "test_property", "test description", "bool", PropertyBag()
+                ),
+                None,
+            )
+        )
+
+    def test_adds_needs_restart_true(self):
+        property = PropertyBag()
+        info = create_property_info(
+            "test_property",
+            "test description",
+            "std::vector<int>",
+            PropertyBag(needs_restart="needs_restart::yes"),
+        )
+        self.transformer.parse(property, info, FilePair("testfile.h", "testfile.cc"))
+
+        self.assertTrue(property["needs_restart"])
+
+    def test_adds_needs_restart_false(self):
+        property = PropertyBag()
+        info = create_property_info(
+            "test_property",
+            "test description",
+            "std::vector<int>",
+            PropertyBag(needs_restart="needs_restart::no"),
+        )
+        self.transformer.parse(property, info, FilePair("testfile.h", "testfile.cc"))
+
+        self.assertFalse(property["needs_restart"])
+
+
+class VisibilityTransformerTestCase(unittest.TestCase):
+    def setUp(self):
+        self.transformer = VisibilityTransformer()
+
+    def test_accepts_visibility_metadata(self):
+        self.assertTrue(
+            self.transformer.accepts(
+                create_property_info(
+                    "test_property",
+                    "test description",
+                    "bool",
+                    PropertyBag(visibility="visibility::user"),
+                ),
+                None,
+            )
+        )
+
+    def test_rejects_without_visibility(self):
+        self.assertFalse(
+            self.transformer.accepts(
+                create_property_info(
+                    "test_property", "test description", "bool", PropertyBag()
+                ),
+                None,
+            )
+        )
+
+    def test_adds_visibility(self):
+        for p in [
+            ("user", "visibility::user"),
+            ("tunable", "visibility::tunable"),
+            ("deprecated", "visibility::deprecated"),
+        ]:
+            with self.subTest(p[0]):
+                property = PropertyBag()
+                info = create_property_info(
+                    "test_property",
+                    "test description",
+                    "std::vector<int>",
+                    PropertyBag(visibility=p[1]),
+                )
+                self.transformer.parse(
+                    property, info, FilePair("testfile.h", "testfile.cc")
+                )
+
+                self.assertEqual(p[0], property["visibility"])
+
+
+class TypeTransformerTestCase(unittest.TestCase):
+    def setUp(self):
+        self.transformer = TypeTransformer()
+
+    def test_accepts_everything(self):
+        self.assertTrue(self.transformer.accepts(None, None))
+
+    def test_parse_simple_types(self):
+        types = [
+            ("int", "integer"),
+            ("int32_t", "integer"),
+            ("int16_t", "integer"),
+            ("int64_t", "integer"),
+            ("double", "number"),
+            ("bool", "boolean"),
+            ("std::chrono::milliseconds", "integer"),
+            ("std::chrono::seconds", "integer"),
+            ("std::filesystem::path", "string"),
+        ]
+
+        for t in types:
+            with self.subTest(t[0]):
+                property = PropertyBag()
+                info = create_property_info(
+                    "test_property",
+                    "test description",
+                    "property<" + t[0] + ">" + " test_property;",
+                )
+                self.transformer.parse(
+                    property, info, FilePair("testfile.h", "testfile.cc")
+                )
+
+                self.assertEqual(t[1], property["type"])
+
+    def test_parse_bounded_property(self):
+        property = PropertyBag()
+        info = create_property_info(
+            "test_property",
+            "test description",
+            "bounded_property<double, numeric_bounds> disk_reservation_percent;",
+        )
+        self.transformer.parse(property, info, FilePair("testfile.h", "testfile.cc"))
+
+        self.assertEqual("number", property["type"])
+
+    def test_parse_deprecated_property(self):
+        property = PropertyBag()
+        info = create_property_info(
+            "test_property", "test description", "deprecated_property test_property;"
+        )
+        self.transformer.parse(property, info, FilePair("testfile.h", "testfile.cc"))
+
+        self.assertEqual("deprecated_property", property["type"])
+
+
+class DeprecatedTransformerTestCase(unittest.TestCase):
+    def setUp(self):
+        self.transformer = DeprecatedTransformer()
+
+    def test_accepts_deprecated_visibility(self):
+        self.assertTrue(
+            self.transformer.accepts(
+                create_property_info(
+                    "test_property",
+                    "test description",
+                    "bool",
+                    PropertyBag(visibility="visibility::deprecated"),
+                ),
+                None,
+            )
+        )
+
+    def test_accepts_deprecated_property_type(self):
+        self.assertTrue(
+            self.transformer.accepts(
+                create_property_info(
+                    "test_property",
+                    "test description",
+                    "deprecated_property",
+                    PropertyBag(),
+                ),
+                None,
+            )
+        )
+
+    def test_rejects_without_deprecated_visibility_type(self):
+        self.assertFalse(
+            self.transformer.accepts(
+                create_property_info(
+                    "test_property", "test description", "bool", PropertyBag()
+                ),
+                None,
+            )
+        )
+
+    def test_adds_deprecated_field_from_deprecated_visibility(self):
+        property = PropertyBag()
+        info = create_property_info(
+            "test_property",
+            "test description",
+            "std::vector<int>",
+            PropertyBag(visibility="visibility::deprecated"),
+        )
+        self.transformer.parse(property, info, FilePair("testfile.h", "testfile.cc"))
+
+        self.assertTrue(property["is_deprecated"])
+        self.assertFalse(property["type"])
+
+    def test_adds_deprecated_field_from_deprecated_property_type(self):
+        property = PropertyBag()
+        info = create_property_info(
+            "test_property", "test description", "deprecated_property", PropertyBag()
+        )
+        self.transformer.parse(property, info, FilePair("testfile.h", "testfile.cc"))
+
+        self.assertTrue(property["is_deprecated"])
+        self.assertFalse(property["type"])
+
+
+class IsSecretTransformerTestCase(unittest.TestCase):
+    def setUp(self):
+        self.transformer = IsSecretTransformer()
+
+    def test_accepts_with_secret(self):
+        self.assertTrue(
+            self.transformer.accepts(
+                create_property_info(
+                    "test_property",
+                    "test description",
+                    "bool",
+                    PropertyBag(secret="is_secret::yes"),
+                ),
+                None,
+            )
+        )
+        self.assertTrue(
+            self.transformer.accepts(
+                create_property_info(
+                    "test_property",
+                    "test description",
+                    "bool",
+                    PropertyBag(secret="is_secret::no"),
+                ),
+                None,
+            )
+        )
+
+    def test_rejects_without_secret(self):
+        self.assertFalse(
+            self.transformer.accepts(
+                create_property_info(
+                    "test_property", "test description", "bool", PropertyBag()
+                ),
+                None,
+            )
+        )
+
+    def test_adds_is_secret_true(self):
+        property = PropertyBag()
+        info = create_property_info(
+            "test_property",
+            "test description",
+            "std::vector<int>",
+            PropertyBag(secret="is_secret::yes"),
+        )
+        self.transformer.parse(property, info, FilePair("testfile.h", "testfile.cc"))
+        self.assertTrue(property["is_secret"])
+
+    def test_adds_is_secret_false(self):
+        property = PropertyBag()
+        info = create_property_info(
+            "test_property",
+            "test description",
+            "std::vector<int>",
+            PropertyBag(secret="is_secret::no"),
+        )
+        self.transformer.parse(property, info, FilePair("testfile.h", "testfile.cc"))
+        self.assertFalse(property["is_secret"])
+
+
+if __name__ == "__main__":
+    unittest.main()