@point3/node-rdkafka 3.6.0-1

package/deps/librdkafka/tests/broker_version_tests.py

@@ -0,0 +1,315 @@
+#!/usr/bin/env python3
+#
+#
+# Run librdkafka regression tests on with different SASL parameters
+# and broker verisons.
+#
+# Requires:
+#  trivup python module
+#  gradle in your PATH
+
+from cluster_testing import (
+    LibrdkafkaTestCluster,
+    print_report_summary,
+    read_scenario_conf)
+from LibrdkafkaTestApp import LibrdkafkaTestApp
+
+import subprocess
+import tempfile
+import os
+import sys
+import argparse
+import json
+
+
+def test_it(version, deploy=True, conf={}, rdkconf={}, tests=None,
+            interact=False, debug=False, scenario="default", kraft=False,
+            inherit_env=False):
+    """
+    @brief Create, deploy and start a Kafka cluster using Kafka \\p version
+    Then run librdkafka's regression tests.
+    """
+
+    cluster = LibrdkafkaTestCluster(version, conf,
+                                    num_brokers=int(conf.get('broker_cnt', 3)),
+                                    debug=debug, scenario=scenario,
+                                    kraft=kraft)
+
+    # librdkafka's regression tests, as an App.
+    _rdkconf = conf.copy()  # Base rdkconf on cluster conf + rdkconf
+    _rdkconf.update(rdkconf)
+    rdkafka = LibrdkafkaTestApp(cluster, version, _rdkconf, tests=tests,
+                                scenario=scenario)
+    rdkafka.do_cleanup = False
+
+    if deploy:
+        cluster.deploy()
+
+    cluster.start(timeout=30)
+
+    if conf.get('test_mode', '') == 'bash':
+        rdkafka.finalize_env()
+
+        if inherit_env:
+            env = dict(os.environ, **rdkafka.env)
+        else:
+            env = dict(rdkafka.env)
+        trivup = f'[TRIVUP:{cluster.name}@{version}] '
+        PS1 = ((trivup + env['PS1']) if 'PS1' in env
+               else trivup + '\\u@\\h:\\w$ ')\
+            .translate(str.maketrans({'\'': '\\\''}))
+        cmd = f'bash --rcfile <(cat ~/.bashrc; echo \'PS1="{PS1}"\')'
+        subprocess.call(
+            cmd,
+            env=env,
+            shell=True,
+            executable='/bin/bash')
+        report = None
+
+    else:
+        rdkafka.start()
+        print(
+            '# librdkafka regression tests started, logs in %s' %
+            rdkafka.root_path())
+        rdkafka.wait_stopped(timeout=60 * 30)
+
+        report = rdkafka.report()
+        report['root_path'] = rdkafka.root_path()
+
+        if report.get('tests_failed', 0) > 0 and interact:
+            print(
+                '# Connect to cluster with bootstrap.servers %s' %
+                cluster.bootstrap_servers())
+            print('# Exiting the shell will bring down the cluster. '
+                  'Good luck.')
+            subprocess.call(
+                'bash --rcfile <(cat ~/.bashrc; echo \'PS1="[TRIVUP:%s@%s] \\u@\\h:\\w$ "\')' %  # noqa: E501
+                (cluster.name, version), env=rdkafka.env, shell=True,
+                executable='/bin/bash')
+
+    cluster.stop(force=True)
+
+    cluster.cleanup()
+    return report
+
+
+def handle_report(report, version, suite):
+    """ Parse test report and return tuple (Passed(bool), Reason(str)) """
+    test_cnt = report.get('tests_run', 0)
+
+    if test_cnt == 0:
+        return (False, 'No tests run')
+
+    passed = report.get('tests_passed', 0)
+    failed = report.get('tests_failed', 0)
+    if 'all' in suite.get('expect_fail', []) or version in suite.get(
+            'expect_fail', []):
+        expect_fail = True
+    else:
+        expect_fail = False
+
+    if expect_fail:
+        if failed == test_cnt:
+            return (True, 'All %d/%d tests failed as expected' %
+                    (failed, test_cnt))
+        else:
+            return (False, '%d/%d tests failed: expected all to fail' %
+                    (failed, test_cnt))
+    else:
+        if failed > 0:
+            return (False, '%d/%d tests passed: expected all to pass' %
+                    (passed, test_cnt))
+        else:
+            return (True, 'All %d/%d tests passed as expected' %
+                    (passed, test_cnt))
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(
+        description='Run librdkafka tests on a range of broker versions')
+
+    parser.add_argument('--debug', action='store_true', default=False,
+                        help='Enable trivup debugging')
+    parser.add_argument('--conf', type=str, dest='conf', default=None,
+                        help='trivup JSON config object (not file)')
+    parser.add_argument('--rdkconf', type=str, dest='rdkconf', default=None,
+                        help='trivup JSON config object (not file) '
+                        'for LibrdkafkaTestApp')
+    parser.add_argument('--scenario', type=str, dest='scenario',
+                        default='default',
+                        help='Test scenario (see scenarios/ directory)')
+    parser.add_argument('--tests', type=str, dest='tests', default=None,
+                        help='Test to run (e.g., "0002")')
+    parser.add_argument('--report', type=str, dest='report', default=None,
+                        help='Write test suites report to this filename')
+    parser.add_argument('--interact', action='store_true', dest='interact',
+                        default=False,
+                        help='On test failure start a shell before bringing '
+                        'the cluster down.')
+    parser.add_argument('versions', type=str, nargs='*',
+                        default=['0.8.1.1', '0.8.2.2', '0.9.0.1', '2.3.0'],
+                        help='Broker versions to test')
+    parser.add_argument('--interactive', action='store_true',
+                        dest='interactive',
+                        default=False,
+                        help='Start a shell instead of running tests')
+    parser.add_argument(
+        '--root',
+        type=str,
+        default=os.environ.get(
+            'TRIVUP_ROOT',
+            'tmp'),
+        help='Root working directory')
+    parser.add_argument(
+        '--port',
+        default=None,
+        help='Base TCP port to start allocating from')
+    parser.add_argument(
+        '--kafka-src',
+        dest='kafka_path',
+        type=str,
+        default=None,
+        help='Path to Kafka git repo checkout (used for version=trunk)')
+    parser.add_argument(
+        '--brokers',
+        dest='broker_cnt',
+        type=int,
+        default=3,
+        help='Number of Kafka brokers')
+    parser.add_argument('--ssl', dest='ssl', action='store_true',
+                        default=False,
+                        help='Enable SSL endpoints')
+    parser.add_argument(
+        '--sasl',
+        dest='sasl',
+        type=str,
+        default=None,
+        help='SASL mechanism (PLAIN, GSSAPI)')
+    parser.add_argument(
+        '--kraft',
+        dest='kraft',
+        action='store_true',
+        default=False,
+        help='Run in KRaft mode')
+
+    args = parser.parse_args()
+
+    conf = dict()
+    rdkconf = dict()
+
+    if args.conf is not None:
+        args.conf = json.loads(args.conf)
+    else:
+        args.conf = {}
+
+    if args.port is not None:
+        args.conf['port_base'] = int(args.port)
+    if args.kafka_path is not None:
+        args.conf['kafka_path'] = args.kafka_path
+    if args.ssl:
+        args.conf['security.protocol'] = 'SSL'
+    if args.sasl:
+        if args.sasl == 'PLAIN' and 'sasl_users' not in args.conf:
+            args.conf['sasl_users'] = 'testuser=testpass'
+        args.conf['sasl_mechanisms'] = args.sasl
+        args.conf['sasl_servicename'] = 'kafka'
+    if args.interactive:
+        args.conf['test_mode'] = 'bash'
+    args.conf['broker_cnt'] = args.broker_cnt
+
+    conf.update(args.conf)
+    if args.rdkconf is not None:
+        rdkconf.update(json.loads(args.rdkconf))
+
+    conf.update(read_scenario_conf(args.scenario))
+
+    if args.tests is not None:
+        tests = args.tests.split(',')
+    elif 'tests' in conf:
+        tests = conf.get('tests', '').split(',')
+    else:
+        tests = None
+
+    # Test version + suite matrix
+    if 'versions' in conf:
+        versions = conf.get('versions')
+    else:
+        versions = args.versions
+    suites = [{'name': 'standard'}]
+
+    pass_cnt = 0
+    fail_cnt = 0
+    for version in versions:
+        for suite in suites:
+            _conf = conf.copy()
+            _conf.update(suite.get('conf', {}))
+            _rdkconf = rdkconf.copy()
+            _rdkconf.update(suite.get('rdkconf', {}))
+
+            if 'version' not in suite:
+                suite['version'] = dict()
+
+            # Run tests
+            print('#### Version %s, suite %s, scenario %s: STARTING' %
+                  (version, suite['name'], args.scenario))
+            report = test_it(version, tests=tests, conf=_conf,
+                             rdkconf=_rdkconf,
+                             interact=args.interact, debug=args.debug,
+                             scenario=args.scenario,
+                             kraft=args.kraft)
+
+            if not report:
+                continue
+
+            # Handle test report
+            report['version'] = version
+            passed, reason = handle_report(report, version, suite)
+            report['PASSED'] = passed
+            report['REASON'] = reason
+
+            if passed:
+                print('\033[42m#### Version %s, suite %s: PASSED: %s\033[0m' %
+                      (version, suite['name'], reason))
+                pass_cnt += 1
+            else:
+                print('\033[41m#### Version %s, suite %s: FAILED: %s\033[0m' %
+                      (version, suite['name'], reason))
+                fail_cnt += 1
+
+                # Emit hopefully relevant parts of the log on failure
+                subprocess.call(
+                    "grep --color=always -B100 -A10 FAIL %s" %
+                    (os.path.join(
+                        report['root_path'],
+                        'stderr.log')),
+                    shell=True)
+
+            print('#### Test output: %s/stderr.log' % (report['root_path']))
+
+            suite['version'][version] = report
+
+    # Write test suite report JSON file
+    if args.report is not None:
+        test_suite_report_file = args.report
+        f = open(test_suite_report_file, 'w')
+    else:
+        fd, test_suite_report_file = tempfile.mkstemp(prefix='test_suite_',
+                                                      suffix='.json',
+                                                      dir='.')
+        f = os.fdopen(fd, 'w')
+
+    full_report = {'suites': suites, 'pass_cnt': pass_cnt,
+                   'fail_cnt': fail_cnt, 'total_cnt': pass_cnt + fail_cnt}
+
+    f.write(json.dumps(full_report))
+    f.close()
+
+    print('\n\n\n')
+    print_report_summary(full_report)
+    print('#### Full test suites report in: %s' % test_suite_report_file)
+
+    if pass_cnt == 0 or fail_cnt > 0:
+        sys.exit(1)
+    else:
+        sys.exit(0)

package/deps/librdkafka/tests/buildbox.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Build script for buildbox.io
+# Must be ran from top-level directory.
+
+PFX=tmp_install
+
+[ -d $PFX ] && rm -rf "$PFX"
+
+make clean || true
+./configure --clean
+./configure "--prefix=$PFX" || exit 1
+make || exit 1
+make install || exit 1
+
+
+

package/deps/librdkafka/tests/cleanup-checker-tests.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+#
+# This script runs all tests with valgrind, one by one, forever, to
+# make sure there aren't any memory leaks.
+
+ALL=$(seq 0 15)
+CNT=0
+while true ; do
+    for T in $ALL; do
+	echo "#################### Test $T run #$CNT #################"
+	TESTS=$(printf %04d $T)	./run-test.sh -p valgrind || exit 1
+	CNT=$(expr $CNT + 1)
+    done
+    echo "################## Cleaning up"
+    rm -f *.offset
+    ./delete-test-topics.sh 0
+done
+done
+

package/deps/librdkafka/tests/cluster_testing.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+#
+#
+# Cluster testing helper
+#
+# Requires:
+#  trivup python module
+#  gradle in your PATH
+
+from trivup.trivup import Cluster
+from trivup.apps.ZookeeperApp import ZookeeperApp
+from trivup.apps.KafkaBrokerApp import KafkaBrokerApp
+from trivup.apps.KerberosKdcApp import KerberosKdcApp
+from trivup.apps.SslApp import SslApp
+from trivup.apps.OauthbearerOIDCApp import OauthbearerOIDCApp
+
+import os
+import sys
+import json
+import argparse
+import re
+from jsoncomment import JsonComment
+
+
+def version_as_list(version):
+    if version == 'trunk':
+        return [sys.maxsize]
+    return [int(a) for a in re.findall('\\d+', version)][0:3]
+
+
+def read_scenario_conf(scenario):
+    """ Read scenario configuration from scenarios/<scenario>.json """
+    parser = JsonComment(json)
+    with open(os.path.join('scenarios', scenario + '.json'), 'r') as f:
+        return parser.load(f)
+
+
+class LibrdkafkaTestCluster(Cluster):
+    def __init__(self, version, conf={}, num_brokers=3, debug=False,
+                 scenario="default", kraft=False):
+        """
+        @brief Create, deploy and start a Kafka cluster using Kafka \\p version
+
+        Supported \\p conf keys:
+         * security.protocol - PLAINTEXT, SASL_PLAINTEXT, SASL_SSL
+
+        \\p conf dict is passed to KafkaBrokerApp classes, etc.
+        """
+
+        super(LibrdkafkaTestCluster, self).__init__(
+            self.__class__.__name__,
+            os.environ.get('TRIVUP_ROOT', 'tmp'), debug=debug)
+
+        # Read trivup config from scenario definition.
+        defconf = read_scenario_conf(scenario)
+        defconf.update(conf)
+
+        # Enable SSL if desired
+        if 'SSL' in conf.get('security.protocol', ''):
+            self.ssl = SslApp(self, defconf)
+
+        self.brokers = list()
+
+        if not kraft:
+            # One ZK (from Kafka repo)
+            ZookeeperApp(self)
+
+        # Start Kerberos KDC if GSSAPI (Kerberos) is configured
+        if 'GSSAPI' in defconf.get('sasl_mechanisms', []):
+            kdc = KerberosKdcApp(self, 'MYREALM')
+            # Kerberos needs to be started prior to Kafka so that principals
+            # and keytabs are available at the time of Kafka config generation.
+            kdc.start()
+
+        if 'OAUTHBEARER'.casefold() == \
+            defconf.get('sasl_mechanisms', "").casefold() and \
+                'OIDC'.casefold() == \
+                defconf.get('sasl_oauthbearer_method', "").casefold():
+            self.oidc = OauthbearerOIDCApp(self)
+
+        # Brokers
+        defconf.update({'replication_factor': min(num_brokers, 3),
+                        'version': version,
+                        'security.protocol': 'PLAINTEXT'})
+        self.conf = defconf
+
+        for n in range(0, num_brokers):
+            defconf_curr = dict(defconf)
+            if 'conf' in defconf_curr:
+                defconf_curr['conf'] = list(defconf_curr['conf'])
+            # Configure rack & replica selector if broker supports
+            # fetch-from-follower
+            if version_as_list(version) >= [2, 4, 0]:
+                curr_conf = defconf_curr.get('conf', list())
+                defconf_curr.update(
+                    {
+                        'conf': [
+                            'broker.rack=RACK${appid}',
+                            'replica.selector.class=org.apache.kafka.common.replica.RackAwareReplicaSelector'  # noqa: E501
+                        ] + curr_conf
+                    })  # noqa: E501
+            print('conf broker', str(n), ': ', defconf_curr)
+            self.brokers.append(KafkaBrokerApp(self, defconf_curr))
+
+    def bootstrap_servers(self):
+        """ @return Kafka bootstrap servers based on security.protocol """
+        all_listeners = (
+            ','.join(
+                self.get_all(
+                    'advertised_listeners',
+                    '',
+                    KafkaBrokerApp))).split(',')
+        return ','.join([x for x in all_listeners if x.startswith(
+            self.conf.get('security.protocol'))])
+
+
+def result2color(res):
+    if res == 'PASSED':
+        return '\033[42m'
+    elif res == 'FAILED':
+        return '\033[41m'
+    else:
+        return ''
+
+
+def print_test_report_summary(name, report):
+    """ Print summary for a test run. """
+    passed = report.get('PASSED', False)
+    if passed:
+        resstr = '\033[42mPASSED\033[0m'
+    else:
+        resstr = '\033[41mFAILED\033[0m'
+
+    print('%6s  %-50s: %s' % (resstr, name, report.get('REASON', 'n/a')))
+    if not passed:
+        # Print test details
+        for name, test in report.get('tests', {}).items():
+            testres = test.get('state', '')
+            if testres == 'SKIPPED':
+                continue
+            print('%s   --> %-20s \033[0m' %
+                  ('%s%s\033[0m' %
+                   (result2color(test.get('state', 'n/a')),
+                    test.get('state', 'n/a')),
+                   test.get('name', 'n/a')))
+        print('%8s --> %s/%s' %
+              ('', report.get('root_path', '.'), 'stderr.log'))
+
+
+def print_report_summary(fullreport):
+    """ Print summary from a full report suite """
+    suites = fullreport.get('suites', list())
+    print('#### Full test suite report (%d suite(s))' % len(suites))
+    for suite in suites:
+        for version, report in suite.get('version', {}).items():
+            print_test_report_summary('%s @ %s' %
+                                      (suite.get('name', 'n/a'), version),
+                                      report)
+
+    pass_cnt = fullreport.get('pass_cnt', -1)
+    if pass_cnt == 0:
+        pass_clr = ''
+    else:
+        pass_clr = '\033[42m'
+
+    fail_cnt = fullreport.get('fail_cnt', -1)
+    if fail_cnt == 0:
+        fail_clr = ''
+    else:
+        fail_clr = '\033[41m'
+
+    print('#### %d suites %sPASSED\033[0m, %d suites %sFAILED\033[0m' %
+          (pass_cnt, pass_clr, fail_cnt, fail_clr))
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Show test suite report')
+    parser.add_argument('report', type=str, nargs=1,
+                        help='Show summary from test suites report file')
+
+    args = parser.parse_args()
+
+    passed = False
+    with open(args.report[0], 'r') as f:
+        passed = print_report_summary(json.load(f))
+
+    if passed:
+        sys.exit(0)
+    else:
+        sys.exit(1)

package/deps/librdkafka/tests/delete-test-topics.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+
+set -e
+
+if [[ "$1" == "-n" ]]; then
+    DO_DELETE=0
+    shift
+else
+    DO_DELETE=1
+fi
+
+ZK=$1
+KATOPS=$2
+RE=$3
+
+if [[ -z "$ZK" ]]; then
+    ZK="$ZK_ADDRESS"
+fi
+
+if [[ -z "$KATOPS" ]]; then
+    if [[ -d "$KAFKA_PATH" ]]; then
+        KATOPS="$KAFKA_PATH/bin/kafka-topics.sh"
+    fi
+fi
+
+if [[ -z "$RE" ]]; then
+    RE="^rdkafkatest_"
+fi
+
+if [[ -z "$KATOPS" ]]; then
+    echo "Usage: $0 [-n] <zookeeper-address> <kafka-topics.sh> [<topic-name-regex>]"
+    echo ""
+    echo "Deletes all topics matching regex $RE"
+    echo ""
+    echo "  -n  - Just collect, dont actually delete anything"
+    exit 1
+fi
+
+set -u
+echo -n "Collecting list of matching topics... "
+TOPICS=$($KATOPS --zookeeper $ZK --list 2>/dev/null | grep "$RE") || true
+N_TOPICS=$(echo "$TOPICS" | wc -w)
+echo "$N_TOPICS topics found"
+
+
+for t in $TOPICS; do
+    if [[ $DO_DELETE == 1 ]]; then
+	echo -n "Deleting topic $t... "
+	($KATOPS --zookeeper $ZK --delete --topic "$t" 2>/dev/null && echo "deleted") || echo "failed"
+    else
+	echo "Topic $t"
+    fi
+done
+
+echo "Done"

package/deps/librdkafka/tests/fixtures/oauthbearer/jwt_assertion_template.json

@@ -0,0 +1,10 @@
+{
+  "header": {},
+  "payload": {
+    "sub": "testuser-template",
+    "aud": "https://token_endpoint/token",
+    "iss": "my_service",
+    "target_audience": "kafka-cluster",
+    "jti":"94232d62-8b0c-406b-ba12-f92d7f522253"
+  }
+}

package/deps/librdkafka/tests/fixtures/ssl/Makefile

@@ -0,0 +1,8 @@
+ssl_keys: clear_keys
+	@./create_keys.sh client client2
+
+clear_keys:
+	@rm -f *.key *.crt *.jks \
+	*.csr *.pem *.p12 *.srl extfile
+
+.PHONY: ssl_keys

package/deps/librdkafka/tests/fixtures/ssl/README.md

@@ -0,0 +1,13 @@
+# SSL keys generation for tests
+
+The Makefile in this directory generates a PKCS#12 keystore 
+and corresponding PEM certificate and key for testing 
+SSL keys and keystore usage in librdkafka.
+
+To update those files with a newer OpenSSL version, just run `make`.
+
+# Requirements
+
+* OpenSSL >= 1.1.1
+* Java keytool >= Java 11
+* GNU Make >= 4.2