@platformatic/rdkafka 4.0.0

package/deps/librdkafka/tests/lz4_manual_test.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+
+#
+# Manual test (verification) of LZ4
+# See README for details
+#
+
+set -e
+# Debug what commands are being executed:
+#set -x
+
+TOPIC=lz4
+
+if [[ $TEST_KAFKA_VERSION == "trunk" ]]; then
+    RDK_ARGS="$RDK_ARGS -X api.version.request=true"
+else
+    if [[ $TEST_KAFKA_VERSION == 0.8.* ]]; then
+	BROKERS=$(echo $BROKERS | sed -e 's/PLAINTEXT:\/\///g')
+    fi
+    RDK_ARGS="$RDK_ARGS -X broker.version.fallback=$TEST_KAFKA_VERSION"
+fi
+
+# Create topic
+${KAFKA_PATH}/bin/kafka-topics.sh --zookeeper $ZK_ADDRESS --create \
+	     --topic $TOPIC --partitions 1 --replication-factor 1
+
+# Produce messages with rdkafka
+echo "### Producing with librdkafka: ids 1000-1010"
+seq 1000 1010 | ../examples/rdkafka_example -P -b $BROKERS -t $TOPIC \
+					    -z lz4 $RDK_ARGS
+
+# Produce with Kafka
+echo "### Producing with Kafka: ids 2000-2010"
+seq 2000 2010 | ${KAFKA_PATH}/bin/kafka-console-producer.sh \
+			     --broker-list $BROKERS --compression-codec lz4 \
+			     --topic $TOPIC
+
+# Consume with rdkafka
+echo "### Consuming with librdkafka: expect 1000-1010 and 2000-2010"
+../examples/rdkafka_example -C -b $BROKERS -t $TOPIC -p 0 -o beginning -e -q -A \
+			    $RDK_ARGS
+
+# Consume with Kafka
+echo "### Consuming with Kafka: expect 1000-1010 and 2000-2010"
+if [[ $TEST_KAFKA_VERSION == "trunk" ]]; then
+    ${KAFKA_PATH}/bin/kafka-console-consumer.sh -new-consumer \
+		 --bootstrap-server $BROKERS --from-beginning --topic $TOPIC \
+		 --timeout-ms 1000
+else
+    ${KAFKA_PATH}/bin/kafka-console-consumer.sh \
+		 --zookeeper $ZK_ADDRESS --from-beginning --topic $TOPIC \
+		 --max-messages 22
+fi
+
+
+echo ""
+echo "### $TEST_KAFKA_VERSION: Did you see messages 1000-1010 and 2000-2010 from both consumers?"
+

package/deps/librdkafka/tests/multi-broker-version-test.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+
+set -e
+
+# Test current librdkafka with multiple broker versions.
+
+if [[ ! -z $TEST_KAFKA_VERSION ]]; then
+    echo "Must not be run from within a trivup session"
+    exit 1
+fi
+
+
+VERSIONS="$*"
+if [[ -z $VERSIONS ]]; then
+    VERSIONS="0.8.2.1 0.9.0.1 0.10.0.1 0.10.1.1 0.10.2.1 0.11.0.0"
+fi
+
+FAILED_VERSIONS=""
+PASSED_VERSIONS=""
+for VERSION in $VERSIONS ; do
+    echo "Testing broker version $VERSION"
+    if [[ $VERSION == "trunk" ]]; then
+        extra_args="--kafka-src ~/src/kafka --no-deploy"
+    else
+        extra_args=""
+    fi
+    ./interactive_broker_version.py \
+        --root ~/old/kafka -c "make run_seq" $extra_args "$VERSION"
+
+    if [[ $? == 0 ]] ; then
+        echo "#### broker $VERSION passed ####"
+        PASSED_VERSIONS="${PASSED_VERSIONS}${VERSION} "
+    else
+        echo "#### broker $VERSION FAILED ####"
+        FAILED_VERSIONS="${FAILED_VERSIONS}${VERSION} "
+    fi
+done
+
+
+echo "broker versions PASSED: ${PASSED_VERSIONS}"
+echo "broker versions FAILED: ${FAILED_VERSIONS}"
+
+if [[ ! -z $FAILED_VERSIONS ]]; then
+    exit 1
+else
+    exit 0
+fi
+
+

package/deps/librdkafka/tests/parse-refcnt.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+#
+
+set -e
+
+# Parse a log with --enable-refcnt output enabled.
+
+log="$1"
+
+if [[ ! -f $log ]]; then
+    echo "Usage: $0 <log-file>"
+    exit 1
+fi
+
+
+# Create a file with all refcnt creations
+cfile=$(mktemp)
+grep 'REFCNT.* 0 +1:' $log | awk '{print $6}' | sort > $cfile
+
+# .. and one file with all refcnt destructions
+dfile=$(mktemp)
+grep 'REFCNT.* 1 -1:' $log | awk '{print $6}' | sort > $dfile
+
+# For each refcnt that was never destructed (never reached 0), find it
+# in the input log.
+
+seen=
+for p in $(grep -v -f $dfile $cfile) ; do
+    echo "=== REFCNT $p never reached 0 ==="
+    grep -nH "$p" $log
+    echo ""
+    seen=yes
+done
+
+rm -f "$cfile" "$dfile"
+
+if [[ -z $seen ]]; then
+    echo "No refcount leaks found"
+    exit 0
+fi
+
+exit 2

package/deps/librdkafka/tests/performance_plot.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+#
+
+import sys
+import json
+import numpy as np
+import matplotlib.pyplot as plt
+
+from collections import defaultdict
+
+
+def semver2int(semver):
+    if semver == 'trunk':
+        semver = '0.10.0.0'
+    vi = 0
+    i = 0
+    for v in reversed(semver.split('.')):
+        vi += int(v) * (i * 10)
+        i += 1
+    return vi
+
+
+def get_perf_data(perfname, stats):
+    """ Return [labels,x,y,errs] for perfname 'mb_per_sec' as a numpy arrays
+        labels: broker versions
+        x: list with identical value (to plot on same x point)
+        y: perfname counter (average)
+        errs: errors
+    """
+    ver = defaultdict(list)
+
+    # Per version:
+    #  * accumulate values
+    #  * calculate average
+    #  * calculate error
+
+    # Accumulate values per version
+    for x in stats:
+        v = str(x[0])
+        ver[v].append(x[1][perfname])
+    print('%s is %s' % (perfname, ver))
+
+    labels0 = sorted(ver.keys(), key=semver2int)
+    y0 = list()
+    errs0 = list()
+
+    # Maintain order by using labels0
+    for v in labels0:
+        # Calculate average
+        avg = sum(ver[v]) / float(len(ver[v]))
+        y0.append(avg)
+        # Calculate error
+        errs0.append(max(ver[v]) - avg)
+
+    labels = np.array(labels0)
+    y1 = np.array(y0)
+    x1 = np.array(range(0, len(labels)))
+    errs = np.array(errs0)
+    return [labels, x1, y1, errs]
+
+
+def plot(description, name, stats, perfname, outfile=None):
+    labels, x, y, errs = get_perf_data(perfname, stats)
+    plt.title('%s: %s %s' % (description, name, perfname))
+    plt.xlabel('Kafka version')
+    plt.ylabel(perfname)
+    plt.errorbar(x, y, yerr=errs, alpha=0.5)
+    plt.xticks(x, labels, rotation='vertical')
+    plt.margins(0.2)
+    plt.subplots_adjust(bottom=0.2)
+    if outfile is None:
+        plt.show()
+    else:
+        plt.savefig(outfile, bbox_inches='tight')
+    return
+
+
+if __name__ == '__main__':
+
+    outfile = sys.argv[1]
+
+    reports = []
+    for rf in sys.argv[2:]:
+        with open(rf) as f:
+            reports.append(json.load(f))
+
+    stats = defaultdict(list)
+
+    # Extract performance test data
+    for rep in reports:
+        perfs = rep.get(
+            'tests',
+            dict()).get(
+            '0038_performance',
+            list).get(
+            'report',
+            None)
+        if perfs is None:
+            continue
+
+        for perf in perfs:
+            for n in ['producer', 'consumer']:
+                o = perf.get(n, None)
+                if o is None:
+                    print('no %s in %s' % (n, perf))
+                    continue
+
+                stats[n].append((rep.get('broker_version', 'unknown'), o))
+
+    for t in ['producer', 'consumer']:
+        for perfname in ['mb_per_sec', 'records_per_sec']:
+            plot('librdkafka 0038_performance test: %s (%d samples)' %
+                 (outfile, len(reports)),
+                 t, stats[t], perfname, outfile='%s_%s_%s.png' % (
+                     outfile, t, perfname))

package/deps/librdkafka/tests/plugin_test/Makefile

@@ -0,0 +1,19 @@
+LIBNAME=	plugin_test
+LIBVER=		1
+
+-include ../../Makefile.config
+
+SRCS=		plugin_test.c
+
+OBJS=		$(SRCS:.c=.o)
+
+# For rdkafka.h
+CPPFLAGS+=-I../../src
+
+all: lib
+
+include ../../mklove/Makefile.base
+
+clean: lib-clean
+
+-include $(DEPS)

package/deps/librdkafka/tests/plugin_test/plugin_test.c

@@ -0,0 +1,58 @@
+/*
+ * librdkafka - The Apache Kafka C/C++ library
+ *
+ * Copyright (c) 2017-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+/**
+ * @brief Plugin test library
+ */
+
+#include <stdio.h>
+#include <assert.h>
+
+/* typical include path outside tests is <librdkafka/rdkafka.h> */
+#include "rdkafka.h"
+
+
+
+static void *my_opaque = (void *)0x5678;
+/*
+ * Common methods
+ */
+rd_kafka_resp_err_t conf_init(rd_kafka_conf_t *conf,
+                              void **plug_opaquep,
+                              char *errstr,
+                              size_t errstr_size) {
+        printf("plugin conf_init called!\n");
+        *plug_opaquep = my_opaque;
+        return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+void conf_destroy(const rd_kafka_conf_t *conf, void *plug_opaque) {
+        assert(plug_opaque == plug_opaque);
+        printf("plugin destroy called\n");
+}

package/deps/librdkafka/tests/requirements.txt

@@ -0,0 +1,2 @@
+trivup/trivup-0.12.4.tar.gz
+jsoncomment

package/deps/librdkafka/tests/run-consumer-tests.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+#
+# Run all tests that employ a consumer.
+#
+
+set -e
+
+TESTS=$(for t in $(grep -l '[Cc]onsume' 0*.{c,cpp}); do \
+            echo $t | sed -e 's/^\([0-9][0-9][0-9][0-9]\)-.*/\1/g' ; \
+        done)
+
+export TESTS
+echo "# Running consumer tests: $TESTS"
+
+./run-test.sh $*

package/deps/librdkafka/tests/run-producer-tests.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+#
+# Run all tests that employ a producer.
+#
+
+set -e
+
+TESTS=$(for t in $(grep -l '[pp]roduce' 0*.{c,cpp}); do \
+            echo $t | sed -e 's/^\([0-9][0-9][0-9][0-9]\)-.*/\1/g' ; \
+        done)
+
+export TESTS
+echo "# Running producer tests: $TESTS"
+
+./run-test.sh $*

package/deps/librdkafka/tests/run-test.sh

@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+#
+
+RED='\033[31m'
+GREEN='\033[32m'
+CYAN='\033[36m'
+CCLR='\033[0m'
+
+if [[ $1 == -h ]]; then
+    echo "Usage: $0 [-..] [modes..]"
+    echo ""
+    echo "  Modes: bare valgrind helgrind cachegrind drd gdb lldb bash"
+    echo "  Options:"
+    echo "   -..    - test-runner command arguments (pass thru)"
+    exit 0
+fi
+
+ARGS=
+
+while [[ $1 == -* ]]; do
+    ARGS="$ARGS $1"
+    shift
+done
+
+TEST=./test-runner
+
+if [ ! -z "$1" ]; then
+    MODES=$1
+else
+    MODES="bare"
+    # Enable valgrind:
+    #MODES="bare valgrind"
+fi
+
+FAILED=0
+
+export RDKAFKA_GITVER="$(git rev-parse --short HEAD)@$(git symbolic-ref -q --short HEAD)"
+
+# Enable valgrind suppressions for false positives
+SUPP="--suppressions=librdkafka.suppressions"
+
+# Uncomment to generate valgrind suppressions
+#GEN_SUPP="--gen-suppressions=yes"
+
+# Common valgrind arguments
+VALGRIND_ARGS="--error-exitcode=3"
+
+# Enable vgdb on valgrind errors.
+#VALGRIND_ARGS="$VALGRIND_ARGS --vgdb-error=1"
+
+# Exit valgrind on first error
+VALGRIND_ARGS="$VALGRIND_ARGS --exit-on-first-error=yes"
+
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../src:../src-cpp
+export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:../src:../src-cpp
+
+echo -e "${CYAN}############## $TEST ################${CCLR}"
+
+for mode in $MODES; do
+    echo -e "${CYAN}### Running test $TEST in $mode mode ###${CCLR}"
+    export TEST_MODE=$mode
+    case "$mode" in
+	valgrind)
+	    valgrind $VALGRIND_ARGS --leak-check=full --show-leak-kinds=all \
+		     --errors-for-leak-kinds=all \
+		     --track-origins=yes \
+                     --track-fds=yes \
+		     $SUPP $GEN_SUPP \
+		$TEST $ARGS
+	    RET=$?
+	    ;;
+	helgrind)
+	    valgrind $VALGRIND_ARGS --tool=helgrind \
+                     --sim-hints=no-nptl-pthread-stackcache \
+                     $SUPP $GEN_SUPP \
+		$TEST	$ARGS
+	    RET=$?
+	    ;;
+	cachegrind|callgrind)
+	    valgrind $VALGRIND_ARGS --tool=$mode \
+		     $SUPP $GEN_SUPP \
+		$TEST $ARGS
+	    RET=$?
+	    ;;
+	drd)
+	    valgrind $VALGRIND_ARGS --tool=drd $SUPP $GEN_SUPP \
+		$TEST	$ARGS
+	    RET=$?
+	    ;;
+        callgrind)
+	    valgrind $VALGRIND_ARGS --tool=callgrind $SUPP $GEN_SUPP \
+		$TEST	$ARGS
+	    RET=$?
+	    ;;
+        gdb)
+            grun=$(mktemp gdbrunXXXXXX)
+            cat >$grun <<EOF
+set \$_exitcode = -999
+run $ARGS
+if \$_exitcode != -999
+ quit
+end
+EOF
+            export ASAN_OPTIONS="$ASAN_OPTIONS:abort_on_error=1"
+            gdb -x $grun $TEST
+            RET=$?
+            rm $grun
+            ;;
+	bare)
+	    $TEST $ARGS
+	    RET=$?
+	    ;;
+        lldb)
+            lldb -b -o "process launch --environment DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH" -- $TEST $ARGS
+            RET=$?
+            ;;
+	bash)
+	    PS1="[run-test.sh] $PS1" bash
+	    RET=$?
+	    ;;
+	*)
+	    echo -e "${RED}### Unknown mode $mode for $TEST ###${CCLR}"
+	    RET=1
+	    ;;
+    esac
+
+    if [ $RET -gt 0 ]; then
+	echo -e "${RED}###"
+	echo -e "### Test $TEST in $mode mode FAILED! (return code $RET) ###"
+	echo -e "###${CCLR}"
+	FAILED=1
+    else
+	echo -e "${GREEN}###"
+	echo -e "### $Test $TEST in $mode mode PASSED! ###"
+	echo -e "###${CCLR}"
+    fi
+done
+
+exit $FAILED
+