@platformatic/rdkafka 4.0.0 → 4.1.0-alpha.1

package/deps/librdkafka/tests/fixtures/ssl/create_keys.sh

@@ -1,35 +1,99 @@
 #!/bin/sh
 set -e
 CA_PASSWORD="${CA_PASSWORD:-use_strong_password_ca}"
+CA_INTERMEDIATE_PASSWORD="${CA_INTERMEDIATE_PASSWORD:-use_strong_password_intermediate_ca}"
 KEYSTORE_PASSWORD="${KEYSTORE_PASSWORD:-use_strong_password_keystore}"
 TRUSTSTORE_PASSWORD="${TRUSTSTORE_PASSWORD:-use_strong_password_truststore}"
 OUTPUT_FOLDER=${OUTPUT_FOLDER:-$( dirname "$0" )}
 CNS=${@:-client}
 
 cd ${OUTPUT_FOLDER}
-CA_ROOT_KEY=caroot.key
-CA_ROOT_CRT=caroot.crt
+CA_ROOT_KEY=${CA_ROOT_KEY:-caroot.key}
+CA_ROOT_CRT=${CA_ROOT_CRT:-caroot.crt}
+CA_INTERMEDIATE_KEY=intermediate.key
+CA_INTERMEDIATE_CSR=intermediate.csr
+CA_INTERMEDIATE_CRT=intermediate.crt
 
-echo "# Generate CA"
-openssl req -new -x509 -keyout $CA_ROOT_KEY \
-    -out $CA_ROOT_CRT -days 3650 -subj \
-    '/CN=caroot/OU=/O=/L=/ST=/C=' -passin "pass:${CA_PASSWORD}" \
-    -passout "pass:${CA_PASSWORD}"
+generate_ca_extfile() {
+echo "# $1: Generate extfile"
+cat << EOF > extfile
+[req]
+distinguished_name=dn
+[ dn ]
+CN=$1
+[ ext ]
+basicConstraints=CA:TRUE,pathlen:0
+subjectKeyIdentifier    = hash
+authorityKeyIdentifier  = keyid:always, issuer:always
+keyUsage                = critical, cRLSign, digitalSignature, keyCertSign
+extendedKeyUsage        = clientAuth
+EOF
+}
+
+generate_client_certificate_extfile() {
+local CN=$1
+echo "# $CN: Generate extfile"
+cat << EOF > extfile
+[req]
+distinguished_name = req_distinguished_name
+x509_extensions = v3_req
+prompt = no
+[req_distinguished_name]
+CN = $CN
+[v3_req]
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = $CN
+DNS.2 = localhost
+EOF
+}
+
+if [ ! -f $CA_ROOT_KEY -o ! -f $CA_ROOT_CRT ]; then
+    echo "# Generate CA"
+    generate_ca_extfile caroot
+    openssl req -new -x509 -config extfile -keyout $CA_ROOT_KEY \
+        -out $CA_ROOT_CRT -subj \
+        '/CN=caroot/OU=/O=/L=/ST=/C=' -passin "pass:${CA_PASSWORD}" \
+        -passout "pass:${CA_PASSWORD}"
+fi
+
+echo "# caintermediate: Generate CSR"
+openssl req -new -keyout $CA_INTERMEDIATE_KEY \
+    -out $CA_INTERMEDIATE_CSR -subj \
+    '/CN=caintermediate/OU=/O=/L=/ST=/C=' \
+    -passin "pass:${CA_INTERMEDIATE_PASSWORD}" \
+    -passout "pass:${CA_INTERMEDIATE_PASSWORD}"
+
+generate_ca_extfile caintermediate
+
+echo "# caintermediate: Sign request"
+openssl x509 -req -extfile extfile \
+-passin "pass:${CA_PASSWORD}" \
+-in "${CA_INTERMEDIATE_CSR}" \
+-CA "${CA_ROOT_CRT}" \
+-CAkey "${CA_ROOT_KEY}" \
+-days 3650 \
+-out "${CA_INTERMEDIATE_CRT}"
 
 for CN in $CNS; do
-    KEYSTORE=$CN.keystore.p12
-    TRUSTSTORE=$CN.truststore.p12
-    SIGNED_CRT=$CN-ca-signed.crt
-    CERTIFICATE=$CN.certificate.pem
-    KEY=$CN.key
+for INTERMEDIATE in true false; do
+    INTERMEDIATE_PREFIX=""
+    if [ $INTERMEDIATE = "true" ]; then
+        INTERMEDIATE_PREFIX=".intermediate"
+    fi
+
+    KEYSTORE=${CN}.keystore${INTERMEDIATE_PREFIX}.p12
+    TRUSTSTORE=${CN}.truststore${INTERMEDIATE_PREFIX}.p12
+    CSR=${CN}${INTERMEDIATE_PREFIX}.csr
+    SIGNED_CRT=${CN}-ca-signed${INTERMEDIATE_PREFIX}.crt
+    CERTIFICATE=${CN}.certificate${INTERMEDIATE_PREFIX}.pem
+    KEY=${CN}${INTERMEDIATE_PREFIX}.key
     # Get specific password for this CN
     CN_KEYSTORE_PASSWORD="$(eval echo \$${CN}_KEYSTORE_PASSWORD)"
     if [ -z "$CN_KEYSTORE_PASSWORD" ]; then
         CN_KEYSTORE_PASSWORD=${KEYSTORE_PASSWORD}_$CN
     fi
 
-    echo ${CN_KEYSTORE_PASSWORD}
-
     echo "# $CN: Generate Keystore"
     keytool -genkey -noprompt \
         -alias $CN \
@@ -47,47 +111,58 @@ for CN in $CNS; do
 
     echo "# $CN: Generate CSR"
     keytool -keystore  $KEYSTORE -alias $CN \
-        -certreq -file  $CN.csr -storepass "${CN_KEYSTORE_PASSWORD}" \
+        -certreq -file $CSR -storepass "${CN_KEYSTORE_PASSWORD}" \
         -keypass "${CN_KEYSTORE_PASSWORD}" \
         -ext "SAN=dns:$CN,dns:localhost"
 
-    echo "# $CN: Generate extfile"
-    cat << EOF > extfile
-[req]
-distinguished_name = req_distinguished_name
-x509_extensions = v3_req
-prompt = no
-[req_distinguished_name]
-CN = $CN
-[v3_req]
-subjectAltName = @alt_names
-[alt_names]
-DNS.1 = $CN
-DNS.2 = localhost
-EOF
-
-    echo "# $CN: Sign the certificate with the CA"
-    openssl x509 -req -CA $CA_ROOT_CRT -CAkey $CA_ROOT_KEY \
-        -in $CN.csr \
-        -out $CN-ca-signed.crt -days 9999 \
-        -CAcreateserial -passin "pass:${CA_PASSWORD}" \
-        -extensions v3_req -extfile extfile
+    generate_client_certificate_extfile $CN
 
     echo "# $CN: Import root certificate"
     keytool -noprompt -keystore $KEYSTORE \
         -alias caroot -import -file $CA_ROOT_CRT -storepass "${CN_KEYSTORE_PASSWORD}"
 
+    if [ $INTERMEDIATE = "true" ]; then
+        echo "# $CN: Sign the certificate with the intermediate CA"
+        openssl x509 -req -CA $CA_INTERMEDIATE_CRT -CAkey $CA_INTERMEDIATE_KEY \
+            -in $CSR \
+            -out $SIGNED_CRT -days 9999 \
+            -CAcreateserial -passin "pass:${CA_INTERMEDIATE_PASSWORD}" \
+            -extensions v3_req -extfile extfile
+
+        echo "# $CN: Import intermediate CA certificate"
+        keytool -noprompt -keystore $KEYSTORE \
+            -alias caintermediate -import -file $CA_INTERMEDIATE_CRT \
+            -storepass "${CN_KEYSTORE_PASSWORD}"
+    else
+        echo "# $CN: Sign the certificate with the CA"
+        openssl x509 -req -CA $CA_ROOT_CRT -CAkey $CA_ROOT_KEY \
+            -in $CSR \
+            -out $SIGNED_CRT -days 9999 \
+            -CAcreateserial -passin "pass:${CA_PASSWORD}" \
+            -extensions v3_req -extfile extfile
+    fi
+
     echo "# $CN: Import signed certificate"
     keytool -noprompt -keystore $KEYSTORE -alias $CN \
         -import -file $SIGNED_CRT -storepass "${CN_KEYSTORE_PASSWORD}" \
         -ext "SAN=dns:$CN,dns:localhost"
 
+    # Delete imported certificates as they were only used to import the 
+    # signed certificate.
+    keytool -delete -alias caroot -keystore $KEYSTORE \
+        -storepass "${CN_KEYSTORE_PASSWORD}"
+    if [ $INTERMEDIATE = "true" ]; then
+        keytool -delete -alias caintermediate -keystore $KEYSTORE \
+            -storepass "${CN_KEYSTORE_PASSWORD}"
+    fi
+
     echo "# $CN: Export PEM certificate"
     openssl pkcs12 -in "$KEYSTORE" -out "$CERTIFICATE" \
-        -nodes -passin "pass:${CN_KEYSTORE_PASSWORD}"
+        -nokeys -passin "pass:${CN_KEYSTORE_PASSWORD}"
 
     echo "# $CN: Export PEM key"
     openssl pkcs12 -in "$KEYSTORE" -out "$KEY" \
         -nocerts -passin "pass:${CN_KEYSTORE_PASSWORD}" \
         -passout "pass:${CN_KEYSTORE_PASSWORD}"
 done
+done

package/deps/librdkafka/tests/java/Makefile

@@ -6,7 +6,8 @@ CLASSES=Murmur2Cli.class TransactionProducerCli.class IncrementalRebalanceCli.cl
 all: $(CLASSES)
 
 %.class: %.java
-	javac -classpath $(KAFKA_JARS)/kafka-clients-*.jar $^
+	CLIENTS_JAR=$$(ls $(KAFKA_JARS)/kafka-clients-*.jar); \
+	javac -classpath $$CLIENTS_JAR:$(KAFKA_PATH)/clients/build/classes/java/main $^
 
 clean:
 	rm -f *.class

package/deps/librdkafka/tests/requirements.txt

@@ -1,2 +1,2 @@
-trivup/trivup-0.12.4.tar.gz
+trivup/trivup-0.14.0.tar.gz
 jsoncomment

package/deps/librdkafka/tests/run-all-tests.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+set -e
+
+
+# Helper script to run all tests through the CI
+# Converts the environment variables to the test arguments
+# with the following defaults:
+#
+# - plaintext connection
+# - no authentication
+# - KRaft enabled
+# - all tests
+# - default parallelism
+# - no assertions
+# - following AK and CP versions
+
+export TEST_KAFKA_GIT_REF=${TEST_KAFKA_GIT_REF:-4.0.0}
+export TEST_CP_VERSION=${TEST_CP_VERSION:-8.0.0}
+
+TEST_SSL_ARG=""
+TEST_SSL_INTERMEDIATE_CA_ARG=""
+TEST_SASL_ARG=""
+TEST_KRAFT_ARG="--kraft"
+TEST_LOCAL_ARG=""
+TEST_CONF_ARG=""
+TEST_QUICK_ARG=""
+TEST_ASSERT_ARG=""
+TEST_PARALLEL_ARG=""
+if [ "$TEST_SSL" = "True" ]; then
+    TEST_SSL_ARG="--ssl"
+fi
+if [ "$TEST_SSL" = "True" -a "$TEST_SSL_INTERMEDIATE_CA" = "True" ]; then
+    TEST_SSL_ARG="$TEST_SSL_ARG --ssl-intermediate-ca"
+fi
+if [ ! -z $TEST_SASL ]; then
+    TEST_SASL_ARG="--sasl $TEST_SASL"
+fi
+if [ "$TEST_KRAFT" = "False" ]; then
+    TEST_KRAFT_ARG=""
+fi
+if [ "$TEST_LOCAL" = "Local" ]; then
+    TEST_LOCAL_ARG="-l"
+fi
+if [ "$TEST_LOCAL" = "Non-local" ]; then
+    TEST_LOCAL_ARG="-L"
+fi
+if [ "$TEST_QUICK" = "True" ]; then
+    TEST_QUICK_ARG="-Q"
+fi
+if [ "$TEST_ASSERT" = "True" ]; then
+    TEST_ASSERT_ARG="-a"
+fi
+if [ ! -z $TEST_PARALLEL ]; then
+    TEST_PARALLEL_ARG="-p$TEST_PARALLEL"
+fi
+if [ ! -z $TEST_CONF ]; then
+    TEST_CONF_ARG="--conf '$TEST_CONF'"
+fi
+if [ ! -z $TEST_ENV_VARIABLES ]; then
+    IFS=',' read -ra TEST_ENV_VARIABLES_ARRAY <<< "$TEST_ENV_VARIABLES"
+    for TEST_ENV_VARIABLE in "${TEST_ENV_VARIABLES_ARRAY[@]}"; do
+        export "$TEST_ENV_VARIABLE"
+    done
+    unset TEST_ENV_VARIABLES_ARRAY
+fi
+
+TEST_ARGS="$TEST_PARALLEL_ARG $TEST_ASSERT_ARG $TEST_QUICK_ARG $TEST_LOCAL_ARG $TEST_RUNNER_PARAMETERS $TEST_MODE"
+TEST_CONFIGURATION="$TEST_SSL_ARG $TEST_SASL_ARG $TEST_KRAFT_ARG $TEST_CONF_ARG $TEST_TRIVUP_PARAMETERS"
+
+echo "Running tests with:"
+echo "kafka branch: $TEST_KAFKA_GIT_REF"
+echo "kafka version: $TEST_KAFKA_VERSION"
+echo "CP version: $TEST_CP_VERSION"
+echo "configuration: $TEST_CONFIGURATION"
+echo "arguments: $TEST_ARGS"
+(cd tests && python3 -m trivup.clusters.KafkaCluster $TEST_CONFIGURATION \
+--version "$TEST_KAFKA_GIT_REF" \
+--cpversion "$TEST_CP_VERSION" \
+--cmd "python run-test-batches.py $TEST_ARGS")

package/deps/librdkafka/tests/run-test-batches.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+import os
+import sys
+import re
+import subprocess
+import signal
+curr_dir = os.path.dirname(os.path.realpath(__file__))
+os.chdir(curr_dir)
+
+# This test script accepts the same arguments as run-test.sh, and
+# the same environment variables as well. Additionally, it reads the following
+# environment variables and can run tests in batches and for a certain
+# number of iterations, especially useful when running tests with valgrind or
+# to debug infrequent errors.
+#
+# TESTS: A comma-separated list of test numbers or test names. In case
+#        they're numeric there's no need to pad them with zeros. There can be
+#        ranges of tests, open ranges and they're sorted and deduplicated.
+#        If not set, it defaults to all tests in the current directory.
+#        Example: TESTS=1,2,3-5,2-,11
+#
+# TESTS_BATCH_SIZE: The number of tests to run in each batch. If not set, it
+#                   defaults to the number of tests in TESTS.
+#
+# TEST_ITERATIONS: The number of iterations to run the tests. If not set, it
+#                  defaults to 1. It continues iterating only if there are no
+#                  errors. When "-a" is set it stops after the first error
+#                  in any batch, when it isn't set it stops after the
+#                  first error in any iteration.
+
+# Max number of tests
+max_automatic_tests = 1000
+# Higher test number
+max_test = 0
+tests = set()
+for f in os.listdir(curr_dir):
+    m = re.match(r'^(\d{4}).*\.c$', f)
+    if m:
+        test_num = int(m.group(1))
+        if test_num < max_automatic_tests:
+            tests.add(test_num)
+max_test = max(tests)
+# Tests batch size, defaults to len(TESTS).
+batch_size = os.environ.get('TESTS_BATCH_SIZE', None)
+if batch_size and batch_size.isnumeric():
+    batch_size = int(batch_size)
+
+# Number of iterations for each batch, it continues iterating only
+# if there are no errors, even when "-a" isn't set
+test_iterations = os.environ.get('TEST_ITERATIONS', "1")
+test_iterations = test_iterations.isnumeric() and int(test_iterations) or 1
+
+
+def valid_test_number(test_number):
+    return test_number >= 0 and test_number <= max_test
+
+
+def test_number_to_str(test_number):
+    return str(test_number).rjust(4, '0')
+
+
+def test_numbers_to_str(test_numbers):
+    return list(map(test_number_to_str, test_numbers))
+
+
+def extract_tests(tests):
+    test_ranges = [test_range for test_range in tests.strip().split(',')
+                   if len(test_range.strip()) > 0]
+    all_tests = []
+    for test_range in test_ranges:
+
+        if '-' in test_range:
+            # 13-15 or 13- or -13
+            start, end = test_range.split('-')
+            start, end = start.strip(), end.strip()
+
+            if start.isnumeric() and end.isnumeric():
+                start, end = int(start), int(end)
+                if valid_test_number(start) and valid_test_number(
+                        end) and start <= end:
+                    all_tests.extend(
+                        test_numbers_to_str(range(start, end + 1)))
+
+            elif start.isnumeric() and end == '':
+                start = int(start)
+                if valid_test_number(start):
+                    all_tests.extend(
+                        test_numbers_to_str(range(start, max_test + 1)))
+            elif start == '' and end.isnumeric():
+                end = int(end)
+                if valid_test_number(end):
+                    all_tests.extend(
+                        test_numbers_to_str(range(0, end + 1)))
+
+        elif test_range.isnumeric():
+            # 13 or 0013
+            all_tests.append(test_number_to_str(test_range))
+        else:
+            # A specific test like 0045_subscribe_update_racks_mock
+            all_tests.append(test_range)
+
+    return list(sorted(set(all_tests)))
+
+
+if 'TESTS' in os.environ:
+    TESTS = extract_tests(os.environ['TESTS'])
+else:
+    TESTS = test_numbers_to_str(range(0, max_test + 1))
+
+if batch_size is None:
+    batch_size = len(TESTS)
+
+args = sys.argv[1:]
+
+exit_on_first_error = '-a' in args
+
+
+def run_tests():
+    first_error = None
+    for i in range(test_iterations):
+        for j in range(0, len(TESTS), batch_size):
+            interrupted = False
+            TESTS_BATCH = ','.join(TESTS[j:j + batch_size])
+            print(f"Running tests: {TESTS_BATCH}, iteration: {i + 1}",
+                  file=sys.stderr)
+            p = subprocess.Popen(['./run-test.sh', '-D'] + args,
+                                 env={**os.environ, 'TESTS': TESTS_BATCH},
+                                 start_new_session=True)
+            try:
+                p.communicate()
+                if p.returncode != 0 and first_error is None:
+                    first_error = p.returncode
+                    if exit_on_first_error:
+                        return first_error
+            except BaseException:
+                first_error = 1
+                interrupted = True
+                return first_error
+            finally:
+                if interrupted:
+                    print('Terminating process group...', file=sys.stderr)
+                    os.killpg(p.pid, signal.SIGTERM)
+                    try:
+                        p.wait(10)
+                    except subprocess.TimeoutExpired:
+                        os.killpg(p.pid, signal.SIGKILL)
+                        p.wait(10)
+        if first_error is not None:
+            # Stop running more iterations if there was an error
+            return first_error
+    return first_error
+
+
+first_error = run_tests()
+print('End of run-test-batches', file=sys.stderr)
+if first_error:
+    sys.exit(first_error)