@point3/node-rdkafka 3.6.0-1

package/deps/librdkafka/tests/performance_plot.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+#
+
+import sys
+import json
+import numpy as np
+import matplotlib.pyplot as plt
+
+from collections import defaultdict
+
+
+def semver2int(semver):
+    if semver == 'trunk':
+        semver = '0.10.0.0'
+    vi = 0
+    i = 0
+    for v in reversed(semver.split('.')):
+        vi += int(v) * (i * 10)
+        i += 1
+    return vi
+
+
+def get_perf_data(perfname, stats):
+    """ Return [labels,x,y,errs] for perfname 'mb_per_sec' as a numpy arrays
+        labels: broker versions
+        x: list with identical value (to plot on same x point)
+        y: perfname counter (average)
+        errs: errors
+    """
+    ver = defaultdict(list)
+
+    # Per version:
+    #  * accumulate values
+    #  * calculate average
+    #  * calculate error
+
+    # Accumulate values per version
+    for x in stats:
+        v = str(x[0])
+        ver[v].append(x[1][perfname])
+    print('%s is %s' % (perfname, ver))
+
+    labels0 = sorted(ver.keys(), key=semver2int)
+    y0 = list()
+    errs0 = list()
+
+    # Maintain order by using labels0
+    for v in labels0:
+        # Calculate average
+        avg = sum(ver[v]) / float(len(ver[v]))
+        y0.append(avg)
+        # Calculate error
+        errs0.append(max(ver[v]) - avg)
+
+    labels = np.array(labels0)
+    y1 = np.array(y0)
+    x1 = np.array(range(0, len(labels)))
+    errs = np.array(errs0)
+    return [labels, x1, y1, errs]
+
+
+def plot(description, name, stats, perfname, outfile=None):
+    labels, x, y, errs = get_perf_data(perfname, stats)
+    plt.title('%s: %s %s' % (description, name, perfname))
+    plt.xlabel('Kafka version')
+    plt.ylabel(perfname)
+    plt.errorbar(x, y, yerr=errs, alpha=0.5)
+    plt.xticks(x, labels, rotation='vertical')
+    plt.margins(0.2)
+    plt.subplots_adjust(bottom=0.2)
+    if outfile is None:
+        plt.show()
+    else:
+        plt.savefig(outfile, bbox_inches='tight')
+    return
+
+
+if __name__ == '__main__':
+
+    outfile = sys.argv[1]
+
+    reports = []
+    for rf in sys.argv[2:]:
+        with open(rf) as f:
+            reports.append(json.load(f))
+
+    stats = defaultdict(list)
+
+    # Extract performance test data
+    for rep in reports:
+        perfs = rep.get(
+            'tests',
+            dict()).get(
+            '0038_performance',
+            list).get(
+            'report',
+            None)
+        if perfs is None:
+            continue
+
+        for perf in perfs:
+            for n in ['producer', 'consumer']:
+                o = perf.get(n, None)
+                if o is None:
+                    print('no %s in %s' % (n, perf))
+                    continue
+
+                stats[n].append((rep.get('broker_version', 'unknown'), o))
+
+    for t in ['producer', 'consumer']:
+        for perfname in ['mb_per_sec', 'records_per_sec']:
+            plot('librdkafka 0038_performance test: %s (%d samples)' %
+                 (outfile, len(reports)),
+                 t, stats[t], perfname, outfile='%s_%s_%s.png' % (
+                     outfile, t, perfname))

package/deps/librdkafka/tests/plugin_test/Makefile

@@ -0,0 +1,19 @@
+LIBNAME=	plugin_test
+LIBVER=		1
+
+-include ../../Makefile.config
+
+SRCS=		plugin_test.c
+
+OBJS=		$(SRCS:.c=.o)
+
+# For rdkafka.h
+CPPFLAGS+=-I../../src
+
+all: lib
+
+include ../../mklove/Makefile.base
+
+clean: lib-clean
+
+-include $(DEPS)

package/deps/librdkafka/tests/plugin_test/plugin_test.c

@@ -0,0 +1,58 @@
+/*
+ * librdkafka - The Apache Kafka C/C++ library
+ *
+ * Copyright (c) 2017-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+/**
+ * @brief Plugin test library
+ */
+
+#include <stdio.h>
+#include <assert.h>
+
+/* typical include path outside tests is <librdkafka/rdkafka.h> */
+#include "rdkafka.h"
+
+
+
+static void *my_opaque = (void *)0x5678;
+/*
+ * Common methods
+ */
+rd_kafka_resp_err_t conf_init(rd_kafka_conf_t *conf,
+                              void **plug_opaquep,
+                              char *errstr,
+                              size_t errstr_size) {
+        printf("plugin conf_init called!\n");
+        *plug_opaquep = my_opaque;
+        return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+void conf_destroy(const rd_kafka_conf_t *conf, void *plug_opaque) {
+        assert(plug_opaque == plug_opaque);
+        printf("plugin destroy called\n");
+}

package/deps/librdkafka/tests/requirements.txt

@@ -0,0 +1,2 @@
+trivup/trivup-0.14.0.tar.gz
+jsoncomment

package/deps/librdkafka/tests/run-all-tests.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+set -e
+
+
+# Helper script to run all tests through the CI
+# Converts the environment variables to the test arguments
+# with the following defaults:
+#
+# - plaintext connection
+# - no authentication
+# - KRaft enabled
+# - all tests
+# - default parallelism
+# - no assertions
+# - following AK and CP versions
+
+export TEST_KAFKA_GIT_REF=${TEST_KAFKA_GIT_REF:-4.0.0}
+export TEST_CP_VERSION=${TEST_CP_VERSION:-8.0.0}
+
+TEST_SSL_ARG=""
+TEST_SSL_INTERMEDIATE_CA_ARG=""
+TEST_SASL_ARG=""
+TEST_KRAFT_ARG="--kraft"
+TEST_LOCAL_ARG=""
+TEST_CONF_ARG=""
+TEST_QUICK_ARG=""
+TEST_ASSERT_ARG=""
+TEST_PARALLEL_ARG=""
+if [ "$TEST_SSL" = "True" ]; then
+    TEST_SSL_ARG="--ssl"
+fi
+if [ "$TEST_SSL" = "True" -a "$TEST_SSL_INTERMEDIATE_CA" = "True" ]; then
+    TEST_SSL_ARG="$TEST_SSL_ARG --ssl-intermediate-ca"
+fi
+if [ ! -z $TEST_SASL ]; then
+    TEST_SASL_ARG="--sasl $TEST_SASL"
+fi
+if [ "$TEST_KRAFT" = "False" ]; then
+    TEST_KRAFT_ARG=""
+fi
+if [ "$TEST_LOCAL" = "Local" ]; then
+    TEST_LOCAL_ARG="-l"
+fi
+if [ "$TEST_LOCAL" = "Non-local" ]; then
+    TEST_LOCAL_ARG="-L"
+fi
+if [ "$TEST_QUICK" = "True" ]; then
+    TEST_QUICK_ARG="-Q"
+fi
+if [ "$TEST_ASSERT" = "True" ]; then
+    TEST_ASSERT_ARG="-a"
+fi
+if [ ! -z $TEST_PARALLEL ]; then
+    TEST_PARALLEL_ARG="-p$TEST_PARALLEL"
+fi
+if [ ! -z $TEST_CONF ]; then
+    TEST_CONF_ARG="--conf '$TEST_CONF'"
+fi
+if [ ! -z $TEST_ENV_VARIABLES ]; then
+    IFS=',' read -ra TEST_ENV_VARIABLES_ARRAY <<< "$TEST_ENV_VARIABLES"
+    for TEST_ENV_VARIABLE in "${TEST_ENV_VARIABLES_ARRAY[@]}"; do
+        export "$TEST_ENV_VARIABLE"
+    done
+    unset TEST_ENV_VARIABLES_ARRAY
+fi
+
+TEST_ARGS="$TEST_PARALLEL_ARG $TEST_ASSERT_ARG $TEST_QUICK_ARG $TEST_LOCAL_ARG $TEST_RUNNER_PARAMETERS $TEST_MODE"
+TEST_CONFIGURATION="$TEST_SSL_ARG $TEST_SASL_ARG $TEST_KRAFT_ARG $TEST_CONF_ARG $TEST_TRIVUP_PARAMETERS"
+
+echo "Running tests with:"
+echo "kafka branch: $TEST_KAFKA_GIT_REF"
+echo "kafka version: $TEST_KAFKA_VERSION"
+echo "CP version: $TEST_CP_VERSION"
+echo "configuration: $TEST_CONFIGURATION"
+echo "arguments: $TEST_ARGS"
+(cd tests && python3 -m trivup.clusters.KafkaCluster $TEST_CONFIGURATION \
+--version "$TEST_KAFKA_GIT_REF" \
+--cpversion "$TEST_CP_VERSION" \
+--cmd "python run-test-batches.py $TEST_ARGS")

package/deps/librdkafka/tests/run-consumer-tests.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+#
+# Run all tests that employ a consumer.
+#
+
+set -e
+
+TESTS=$(for t in $(grep -l '[Cc]onsume' 0*.{c,cpp}); do \
+            echo $t | sed -e 's/^\([0-9][0-9][0-9][0-9]\)-.*/\1/g' ; \
+        done)
+
+export TESTS
+echo "# Running consumer tests: $TESTS"
+
+./run-test.sh $*

package/deps/librdkafka/tests/run-producer-tests.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+#
+# Run all tests that employ a producer.
+#
+
+set -e
+
+TESTS=$(for t in $(grep -l '[pp]roduce' 0*.{c,cpp}); do \
+            echo $t | sed -e 's/^\([0-9][0-9][0-9][0-9]\)-.*/\1/g' ; \
+        done)
+
+export TESTS
+echo "# Running producer tests: $TESTS"
+
+./run-test.sh $*

package/deps/librdkafka/tests/run-test-batches.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+import os
+import sys
+import re
+import subprocess
+import signal
+curr_dir = os.path.dirname(os.path.realpath(__file__))
+os.chdir(curr_dir)
+
+# This test script accepts the same arguments as run-test.sh, and
+# the same environment variables as well. Additionally, it reads the following
+# environment variables and can run tests in batches and for a certain
+# number of iterations, especially useful when running tests with valgrind or
+# to debug infrequent errors.
+#
+# TESTS: A comma-separated list of test numbers or test names. In case
+#        they're numeric there's no need to pad them with zeros. There can be
+#        ranges of tests, open ranges and they're sorted and deduplicated.
+#        If not set, it defaults to all tests in the current directory.
+#        Example: TESTS=1,2,3-5,2-,11
+#
+# TESTS_BATCH_SIZE: The number of tests to run in each batch. If not set, it
+#                   defaults to the number of tests in TESTS.
+#
+# TEST_ITERATIONS: The number of iterations to run the tests. If not set, it
+#                  defaults to 1. It continues iterating only if there are no
+#                  errors. When "-a" is set it stops after the first error
+#                  in any batch, when it isn't set it stops after the
+#                  first error in any iteration.
+
+# Max number of tests
+max_automatic_tests = 1000
+# Higher test number
+max_test = 0
+tests = set()
+for f in os.listdir(curr_dir):
+    m = re.match(r'^(\d{4}).*\.c$', f)
+    if m:
+        test_num = int(m.group(1))
+        if test_num < max_automatic_tests:
+            tests.add(test_num)
+max_test = max(tests)
+# Tests batch size, defaults to len(TESTS).
+batch_size = os.environ.get('TESTS_BATCH_SIZE', None)
+if batch_size and batch_size.isnumeric():
+    batch_size = int(batch_size)
+
+# Number of iterations for each batch, it continues iterating only
+# if there are no errors, even when "-a" isn't set
+test_iterations = os.environ.get('TEST_ITERATIONS', "1")
+test_iterations = test_iterations.isnumeric() and int(test_iterations) or 1
+
+
+def valid_test_number(test_number):
+    return test_number >= 0 and test_number <= max_test
+
+
+def test_number_to_str(test_number):
+    return str(test_number).rjust(4, '0')
+
+
+def test_numbers_to_str(test_numbers):
+    return list(map(test_number_to_str, test_numbers))
+
+
+def extract_tests(tests):
+    test_ranges = [test_range for test_range in tests.strip().split(',')
+                   if len(test_range.strip()) > 0]
+    all_tests = []
+    for test_range in test_ranges:
+
+        if '-' in test_range:
+            # 13-15 or 13- or -13
+            start, end = test_range.split('-')
+            start, end = start.strip(), end.strip()
+
+            if start.isnumeric() and end.isnumeric():
+                start, end = int(start), int(end)
+                if valid_test_number(start) and valid_test_number(
+                        end) and start <= end:
+                    all_tests.extend(
+                        test_numbers_to_str(range(start, end + 1)))
+
+            elif start.isnumeric() and end == '':
+                start = int(start)
+                if valid_test_number(start):
+                    all_tests.extend(
+                        test_numbers_to_str(range(start, max_test + 1)))
+            elif start == '' and end.isnumeric():
+                end = int(end)
+                if valid_test_number(end):
+                    all_tests.extend(
+                        test_numbers_to_str(range(0, end + 1)))
+
+        elif test_range.isnumeric():
+            # 13 or 0013
+            all_tests.append(test_number_to_str(test_range))
+        else:
+            # A specific test like 0045_subscribe_update_racks_mock
+            all_tests.append(test_range)
+
+    return list(sorted(set(all_tests)))
+
+
+if 'TESTS' in os.environ:
+    TESTS = extract_tests(os.environ['TESTS'])
+else:
+    TESTS = test_numbers_to_str(range(0, max_test + 1))
+
+if batch_size is None:
+    batch_size = len(TESTS)
+
+args = sys.argv[1:]
+
+exit_on_first_error = '-a' in args
+
+
+def run_tests():
+    first_error = None
+    for i in range(test_iterations):
+        for j in range(0, len(TESTS), batch_size):
+            interrupted = False
+            TESTS_BATCH = ','.join(TESTS[j:j + batch_size])
+            print(f"Running tests: {TESTS_BATCH}, iteration: {i + 1}",
+                  file=sys.stderr)
+            p = subprocess.Popen(['./run-test.sh', '-D'] + args,
+                                 env={**os.environ, 'TESTS': TESTS_BATCH},
+                                 start_new_session=True)
+            try:
+                p.communicate()
+                if p.returncode != 0 and first_error is None:
+                    first_error = p.returncode
+                    if exit_on_first_error:
+                        return first_error
+            except BaseException:
+                first_error = 1
+                interrupted = True
+                return first_error
+            finally:
+                if interrupted:
+                    print('Terminating process group...', file=sys.stderr)
+                    os.killpg(p.pid, signal.SIGTERM)
+                    try:
+                        p.wait(10)
+                    except subprocess.TimeoutExpired:
+                        os.killpg(p.pid, signal.SIGKILL)
+                        p.wait(10)
+        if first_error is not None:
+            # Stop running more iterations if there was an error
+            return first_error
+    return first_error
+
+
+first_error = run_tests()
+print('End of run-test-batches', file=sys.stderr)
+if first_error:
+    sys.exit(first_error)

package/deps/librdkafka/tests/run-test.sh

@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+#
+
+RED='\033[31m'
+GREEN='\033[32m'
+CYAN='\033[36m'
+CCLR='\033[0m'
+
+if [[ $1 == -h ]]; then
+    echo "Usage: $0 [-..] [modes..]"
+    echo ""
+    echo "  Modes: bare valgrind helgrind cachegrind drd gdb lldb bash"
+    echo "  Options:"
+    echo "   -..    - test-runner command arguments (pass thru)"
+    exit 0
+fi
+
+ARGS=
+
+while [[ $1 == -* ]]; do
+    ARGS="$ARGS $1"
+    shift
+done
+
+TEST=./test-runner
+
+if [ ! -z "$1" ]; then
+    MODES=$1
+else
+    MODES="bare"
+    # Enable valgrind:
+    #MODES="bare valgrind"
+fi
+
+FAILED=0
+
+export RDKAFKA_GITVER="$(git rev-parse --short HEAD)@$(git symbolic-ref -q --short HEAD)"
+
+# Enable valgrind suppressions for false positives
+SUPP="--suppressions=librdkafka.suppressions"
+
+# Uncomment to generate valgrind suppressions
+#GEN_SUPP="--gen-suppressions=yes"
+
+# Common valgrind arguments
+VALGRIND_ARGS="--error-exitcode=3"
+
+# Enable vgdb on valgrind errors.
+#VALGRIND_ARGS="$VALGRIND_ARGS --vgdb-error=1"
+
+# Exit valgrind on first error
+VALGRIND_ARGS="$VALGRIND_ARGS --exit-on-first-error=yes"
+
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../src:../src-cpp
+export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:../src:../src-cpp
+
+echo -e "${CYAN}############## $TEST ################${CCLR}"
+
+for mode in $MODES; do
+    echo -e "${CYAN}### Running test $TEST in $mode mode ###${CCLR}"
+    export TEST_MODE=$mode
+    case "$mode" in
+	valgrind)
+	    valgrind $VALGRIND_ARGS --leak-check=full --show-leak-kinds=all \
+		     --errors-for-leak-kinds=all \
+		     --track-origins=yes \
+                     --track-fds=yes \
+		     $SUPP $GEN_SUPP \
+		$TEST $ARGS
+	    RET=$?
+	    ;;
+	helgrind)
+	    valgrind $VALGRIND_ARGS --tool=helgrind \
+                     --sim-hints=no-nptl-pthread-stackcache \
+                     $SUPP $GEN_SUPP \
+		$TEST	$ARGS
+	    RET=$?
+	    ;;
+	cachegrind|callgrind)
+	    valgrind $VALGRIND_ARGS --tool=$mode \
+		     $SUPP $GEN_SUPP \
+		$TEST $ARGS
+	    RET=$?
+	    ;;
+	drd)
+	    valgrind $VALGRIND_ARGS --tool=drd $SUPP $GEN_SUPP \
+		$TEST	$ARGS
+	    RET=$?
+	    ;;
+        callgrind)
+	    valgrind $VALGRIND_ARGS --tool=callgrind $SUPP $GEN_SUPP \
+		$TEST	$ARGS
+	    RET=$?
+	    ;;
+        gdb)
+            grun=$(mktemp gdbrunXXXXXX)
+            cat >$grun <<EOF
+set \$_exitcode = -999
+run $ARGS
+if \$_exitcode != -999
+ quit
+end
+EOF
+            export ASAN_OPTIONS="$ASAN_OPTIONS:abort_on_error=1"
+            gdb -x $grun $TEST
+            RET=$?
+            rm $grun
+            ;;
+	bare)
+	    $TEST $ARGS
+	    RET=$?
+	    ;;
+        lldb)
+            lldb -b -o "process launch --environment DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH" -- $TEST $ARGS
+            RET=$?
+            ;;
+	bash)
+	    PS1="[run-test.sh] $PS1" bash
+	    RET=$?
+	    ;;
+	*)
+	    echo -e "${RED}### Unknown mode $mode for $TEST ###${CCLR}"
+	    RET=1
+	    ;;
+    esac
+
+    if [ $RET -gt 0 ]; then
+	echo -e "${RED}###"
+	echo -e "### Test $TEST in $mode mode FAILED! (return code $RET) ###"
+	echo -e "###${CCLR}"
+	FAILED=1
+    else
+	echo -e "${GREEN}###"
+	echo -e "### $Test $TEST in $mode mode PASSED! ###"
+	echo -e "###${CCLR}"
+    fi
+done
+
+exit $FAILED
+