@platformatic/rdkafka 4.0.0

package/deps/librdkafka/packaging/rpm/mock-on-docker.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+#
+#
+#
+# Run mock in docker to create RPM packages of librdkafka.
+#
+# Usage:
+#   packaging/rpm/mock-on-docker.sh [<mock configs ..>]
+#
+
+set -ex
+
+_DOCKER_IMAGE=rockylinux:9
+_MOCK_CONFIGS="rocky+epel-8-x86_64 rocky+epel-9-x86_64"
+
+if [[ $1 == "--build" ]]; then
+    on_builder=1
+    shift
+else
+    on_builder=0
+fi
+
+
+if [[ -n $* ]]; then
+    _MOCK_CONFIGS="$*"
+fi
+
+
+if [[ $on_builder == 0 ]]; then
+    #
+    # Running on host, fire up a docker container and run the latter
+    # part of this script in docker.
+    #
+
+    if [[ ! -f configure.self ]]; then
+        echo "$0 must be run from librdkafka top directory"
+        exit 1
+    fi
+
+    mkdir -p ${PWD}/packaging/rpm/cache/mock
+
+    docker run \
+           --privileged \
+           -t \
+           -v ${PWD}/packaging/rpm/cache/mock:/var/cache/mock \
+           -v ${PWD}:/io \
+           $_DOCKER_IMAGE \
+           /io/packaging/rpm/mock-on-docker.sh --build $_MOCK_CONFIGS
+
+    mkdir -p artifacts
+    for MOCK_CONFIG in $_MOCK_CONFIGS ; do
+        cp -vr --no-preserve=ownership packaging/rpm/arts-${MOCK_CONFIG}/*rpm artifacts/
+    done
+
+    echo "All Done"
+
+else
+    #
+    # Running in docker container.
+    #
+
+    dnf install -y -q epel-release make git
+    dnf install -y -q mock mock-core-configs
+
+    echo "%_netsharedpath /sys:/proc" >> /etc/rpm/macros.netshared
+
+    pushd /io/packaging/rpm
+
+    for MOCK_CONFIG in $_MOCK_CONFIGS ; do
+        cfg_file=/etc/mock/${MOCK_CONFIG}.cfg
+        if [[ ! -f $cfg_file ]]; then
+            echo "Error: Mock config $cfg_file does not exist"
+            exit 1
+        fi
+
+        echo "config_opts['plugin_conf']['bind_mount_enable'] = False" >> $cfg_file
+        echo "config_opts['docker_unshare_warning'] = False" >> $cfg_file
+        echo "Building $MOCK_CONFIG in $PWD"
+        cat $cfg_file
+
+        echo "Setting git safe.directory"
+        git config --global --add safe.directory /io
+
+        export MOCK_CONFIG=$MOCK_CONFIG
+        make all
+
+        echo "Done building $MOCK_CONFIG: copying artifacts"
+        artdir="arts-$MOCK_CONFIG"
+        mkdir -p "$artdir"
+        make ARTIFACTS_DIR="$artdir" copy-artifacts
+
+    done
+
+    popd
+    echo "Done"
+fi

package/deps/librdkafka/packaging/rpm/tests/Makefile

@@ -0,0 +1,25 @@
+
+PROGS?=test test-static testcpp testcpp-static
+
+all: $(PROGS)
+
+test: test.c
+	$(CC) -O2 -Werror -Wall $^ -o $@ $$(pkg-config --libs rdkafka)
+
+test-static: test.c
+	$(CC) -O2 -Werror -Wall $^ -o $@ $$(pkg-config --libs rdkafka-static)
+
+testcpp: test.cpp
+	$(CXX) -O2 -Werror -Wall $^ -o $@ $$(pkg-config --libs rdkafka++)
+
+testcpp-static: test.cpp
+	$(CXX) -O2 -Werror -Wall $^ -o $@ $$(pkg-config --libs rdkafka++-static)
+
+run:
+	@(for p in $(PROGS); do \
+		echo "# Running $$p" ; \
+		./$$p || (echo $$p failed ; exit 1) ; \
+	done)
+
+clean:
+	rm -f $(PROGS)

package/deps/librdkafka/packaging/rpm/tests/README.md

@@ -0,0 +1,8 @@
+# Test librdkafka RPMs using docker
+
+After building the RPMs (see README.md in parent directory) test
+the RPMs on the supported CentOS/RHEL versions using:
+
+    $ packaging/rpm/tests/test-on-docker.sh
+
+

package/deps/librdkafka/packaging/rpm/tests/run-test.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# This script runs in the docker container, performing:
+# * install build toolchain
+# * install librdkafka rpms
+# * builds test apps
+# * runs test apps
+#
+# Usage: $0 <docker-image-name>
+
+set -ex
+
+pushd /v
+
+_IMG=$1
+
+echo "Testing on $_IMG"
+
+if [[ $_IMG == "rockylinux:8" ]]; then
+    _EL=8
+    _INST="dnf install -y -q"
+else
+    _EL=9
+    _INST="dnf install -y -q"
+fi
+
+$_INST gcc gcc-c++ make pkg-config
+
+if [[ -n $_UPG ]]; then
+    $_UPG
+fi
+
+$_INST /rpms/librdkafka1-*el${_EL}.x86_64.rpm /rpms/librdkafka-devel-*el${_EL}.x86_64.rpm
+
+make clean all
+
+make run
+
+make clean
+
+echo "$_IMG is all good!"
+

package/deps/librdkafka/packaging/rpm/tests/test-on-docker.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+#
+# Test librdkafka packages in <rpmdirectory> using docker.
+# Must be executed from the librdkafka top-level directory.
+#
+# Usage:
+#   packaging/rpm/test-on-docker.sh [<rpm-dir>]
+
+set -ex
+
+if [[ ! -f configure.self ]]; then
+    echo "Must be executed from the librdkafka top-level directory"
+    exit 1
+fi
+
+_DOCKER_IMAGES="rockylinux:8 rockylinux:9"
+_RPMDIR=artifacts
+
+if [[ -n $1 ]]; then
+    _RPMDIR="$1"
+fi
+
+_RPMDIR=$(readlink -f $_RPMDIR)
+
+if [[ ! -d $_RPMDIR ]]; then
+    echo "$_RPMDIR does not exist"
+    exit 1
+fi
+
+
+fails=""
+for _IMG in $_DOCKER_IMAGES ; do
+    if ! docker run \
+         -t \
+         -v $_RPMDIR:/rpms \
+         -v $(readlink -f packaging/rpm/tests):/v \
+         $_IMG \
+         /v/run-test.sh $_IMG ; then
+        echo "ERROR: $_IMG FAILED"
+        fails="${fails}$_IMG "
+    fi
+done
+
+if [[ -n $fails ]]; then
+    echo "##################################################"
+    echo "# Package verification failed for:"
+    echo "# $fails"
+    echo "# See previous errors"
+    echo "##################################################"
+    exit 1
+fi
+
+exit 0
+
+

package/deps/librdkafka/packaging/rpm/tests/test.c

@@ -0,0 +1,77 @@
+#include <stdio.h>
+#include <string.h>
+#include <librdkafka/rdkafka.h>
+
+int main(int argc, char **argv) {
+        rd_kafka_conf_t *conf;
+        rd_kafka_t *rk;
+        char features[256];
+        size_t fsize = sizeof(features);
+        char errstr[512];
+        const char *exp_features[] = {
+            "gzip", "snappy",           "ssl",        "sasl",       "regex",
+            "lz4",  "sasl_gssapi",      "sasl_plain", "sasl_scram", "plugins",
+            "zstd", "sasl_oauthbearer", NULL,
+        };
+        const char **exp;
+        int missing = 0;
+
+
+        printf("librdkafka %s\n", rd_kafka_version_str());
+
+        conf = rd_kafka_conf_new();
+        if (rd_kafka_conf_get(conf, "builtin.features", features, &fsize) !=
+            RD_KAFKA_CONF_OK) {
+                fprintf(stderr, "conf_get failed\n");
+                return 1;
+        }
+
+        printf("builtin.features %s\n", features);
+
+        /* Verify that expected features are enabled. */
+        for (exp = exp_features; *exp; exp++) {
+                const char *t = features;
+                size_t elen   = strlen(*exp);
+                int match     = 0;
+
+                while ((t = strstr(t, *exp))) {
+                        if (t[elen] == ',' || t[elen] == '\0') {
+                                match = 1;
+                                break;
+                        }
+                        t += elen;
+                }
+
+                if (match)
+                        continue;
+
+                fprintf(stderr, "ERROR: feature %s not found\n", *exp);
+                missing++;
+        }
+
+        if (rd_kafka_conf_set(conf, "security.protocol", "SASL_SSL", errstr,
+                              sizeof(errstr)) ||
+            rd_kafka_conf_set(conf, "sasl.mechanism", "PLAIN", errstr,
+                              sizeof(errstr)) ||
+            rd_kafka_conf_set(conf, "sasl.username", "username", errstr,
+                              sizeof(errstr)) ||
+            rd_kafka_conf_set(conf, "sasl.password", "password", errstr,
+                              sizeof(errstr)) ||
+            rd_kafka_conf_set(conf, "debug", "security", errstr,
+                              sizeof(errstr))) {
+                fprintf(stderr, "conf_set failed: %s\n", errstr);
+                return 1;
+        }
+
+        rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr));
+        if (!rk) {
+                fprintf(stderr, "rd_kafka_new failed: %s\n", errstr);
+                return 1;
+        }
+
+        printf("client name %s\n", rd_kafka_name(rk));
+
+        rd_kafka_destroy(rk);
+
+        return missing ? 1 : 0;
+}

package/deps/librdkafka/packaging/rpm/tests/test.cpp

@@ -0,0 +1,34 @@
+#include <iostream>
+#include <librdkafka/rdkafkacpp.h>
+
+
+int main() {
+  std::cout << "librdkafka++ " << RdKafka::version_str() << std::endl;
+
+  RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
+
+  std::string features;
+
+  if (conf->get("builtin.features", features) != RdKafka::Conf::CONF_OK) {
+    std::cerr << "conf_get failed" << std::endl;
+    return 1;
+  }
+
+  std::cout << "builtin.features " << features << std::endl;
+
+  std::string errstr;
+  RdKafka::Producer *producer = RdKafka::Producer::create(conf, errstr);
+  if (!producer) {
+    std::cerr << "Producer::create failed: " << errstr << std::endl;
+    return 1;
+  }
+
+  delete conf;
+
+  std::cout << "client name " << producer->name() << std::endl;
+
+
+  delete producer;
+
+  return 0;
+}

package/deps/librdkafka/packaging/tools/build-configurations-checks.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+# Disable all flags to make sure it
+# compiles correctly in all cases
+./configure --install-deps --disable-ssl --disable-gssapi \
+--disable-curl --disable-zlib \
+--disable-zstd --disable-lz4-ext --disable-regex-ext \
+--disable-c11threads --disable-syslog
+make -j
+make -j -C tests run_local_quick

package/deps/librdkafka/packaging/tools/build-deb-package.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+#
+# Build librdkafka Debian package on a bare-bone Debian host, such as ubuntu:16.04 (docker).
+#
+# Usage (from top-level librdkafka dir):
+#   docker run -it -v $PWD:/v ubuntu:16.04 /v/packaging/tools/build-deb-package.sh 1.0.0 master
+#
+
+set -exu
+
+if [[ $# -ne 2 ]]; then
+    echo "Usage: $0 <package-version> <librdkafka-branch-or-tag>"
+    exit 1
+fi
+
+export VERSION=$1
+LRK_BRANCH=$2
+
+apt-get update
+
+# Install debian packaging tools and librdkafka build dependencies
+apt-get install -y git-buildpackage debhelper \
+        zlib1g-dev libssl-dev libsasl2-dev liblz4-dev
+
+
+# Clone the librdkafka git repo to a new location to avoid messing
+# up the librdkafka working directory.
+
+
+BUILD_DIR=$(mktemp -d)
+
+pushd $BUILD_DIR
+
+git clone /v librdkafka
+
+pushd librdkafka
+
+export DEBEMAIL="librdkafka packaging <cloud-support@confluent.io>"
+git config user.email "cloud-support@confluent.io"
+git config user.name "librdkafka packaging"
+
+DEB_BRANCH=origin/confluent-debian
+TMP_BRANCH=tmp-debian
+git checkout -b $TMP_BRANCH $LRK_BRANCH
+git merge --no-edit $DEB_BRANCH
+
+dch --newversion ${VERSION/-/\~}-1 "Release version $VERSION" --urgency low && dch --release --distribution unstable ""
+
+git commit -a -m "Tag Debian release $VERSION."
+
+make archive
+mkdir -p ../tarballs || true
+mv librdkafka-${VERSION}.tar.gz ../tarballs/librdkafka_${VERSION}.orig.tar.gz
+
+gbp buildpackage -us -uc --git-debian-branch=$TMP_BRANCH \
+    --git-upstream-tree=$LRK_BRANCH \
+    --git-verbose \
+    --git-builder="debuild --set-envvar=VERSION=$VERSION --set-envvar=SKIP_TESTS=y -i -I"
+
+
+popd # librdkafka
+
+popd # $BUILD_DIR
+

package/deps/librdkafka/packaging/tools/build-debian.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# Build librdkafka on a bare-bone Debian host, such as the
+# mcr.microsoft.com/dotnet/sdk Docker image.
+#
+# Statically linked
+# WITH openssl 1.0, zlib
+# WITHOUT libsasl2, lz4(ext, using builtin instead)
+#
+# Usage (from top-level librdkafka dir):
+#   docker run -it -v $PWD:/v mcr.microsoft.com/dotnet/sdk /v/packaging/tools/build-debian.sh /v /v/librdkafka-debian9.tgz
+#
+
+
+set -ex
+
+LRK_DIR=$1
+shift
+OUT_TGZ=$1
+shift
+CONFIG_ARGS=$*
+
+if [[ ! -f $LRK_DIR/configure.self || -z $OUT_TGZ ]]; then
+    echo "Usage: $0 <librdkafka-root-direcotry> <output-tgz> [<configure-args..>]"
+    exit 1
+fi
+
+set -u
+
+apt-get update
+apt-get install -y gcc g++ zlib1g-dev python3 git-core make patch
+
+
+# Copy the librdkafka git archive to a new location to avoid messing
+# up the librdkafka working directory.
+
+BUILD_DIR=$(mktemp -d)
+
+pushd $BUILD_DIR
+
+DEST_DIR=$PWD/dest
+mkdir -p $DEST_DIR
+
+# Workaround for newer Git not allowing clone directory to be owned by
+# another user (which is a questionable limitation for the read-only archive
+# command..)
+git config --global --add safe.directory /v
+
+(cd $LRK_DIR ; git archive --format tar HEAD) | tar xf -
+
+./configure --install-deps --disable-gssapi --disable-lz4-ext --enable-static --prefix=$DEST_DIR $CONFIG_ARGS
+make -j
+examples/rdkafka_example -X builtin.features
+CI=true make -C tests run_local_quick
+make install
+
+# Tar up the output directory
+pushd $DEST_DIR
+ldd lib/*.so.1
+tar cvzf $OUT_TGZ .
+popd # $DEST_DIR
+
+popd # $BUILD_DIR
+
+rm -rf "$BUILD_DIR"

package/deps/librdkafka/packaging/tools/build-manylinux.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+#
+# Build on a manylinux (https://github.com/pypa/manylinux) docker container.
+#
+# This will provide a self-contained librdkafka shared library that works
+# on most glibc-based Linuxes.
+#
+# Statically linked
+# WITH openssl 1.1.1, zlib, lz4(bundled)
+# WITHOUT libsasl2
+#
+#
+# Run:
+#  docker run -t -v "$PWD:/v quay.io/pypa/manylinux2010_x86_64 /v/packaging/tools/build-manylinux.sh /v /v/artifacts/librdkafka-manylinux2010_x86_64.tgz $config_args"
+
+set -ex
+
+LRK_DIR=$1
+shift
+OUT_TGZ=$1
+shift
+CONFIG_ARGS=$*
+
+if [[ ! -f $LRK_DIR/configure.self || -z $OUT_TGZ ]]; then
+    echo "Usage: $0 <librdkafka-root-direcotry> <output-tgz> [<configure-args..>]"
+    exit 1
+fi
+
+set -u
+
+yum install -y libstdc++-devel gcc gcc-c++ python34
+
+# Copy the librdkafka git archive to a new location to avoid messing
+# up the librdkafka working directory.
+
+BUILD_DIR=$(mktemp -d)
+
+pushd $BUILD_DIR
+
+DEST_DIR=$PWD/dest
+mkdir -p $DEST_DIR
+
+# Workaround for newer Git not allowing clone directory to be owned by
+# another user (which is a questionable limitation for the read-only archive
+# command..)
+git config --global --add safe.directory /v
+
+(cd $LRK_DIR ; git archive --format tar HEAD) | tar xf -
+
+./configure --install-deps --source-deps-only --disable-gssapi --disable-lz4-ext --enable-static --prefix=$DEST_DIR $CONFIG_ARGS
+
+make -j
+
+examples/rdkafka_example -X builtin.features
+
+CI=true make -C tests run_local_quick
+
+make install
+
+# Tar up the output directory
+pushd $DEST_DIR
+ldd lib/*.so.1
+tar cvzf $OUT_TGZ .
+popd # $DEST_DIR
+
+popd # $BUILD_DIR
+
+rm -rf "$BUILD_DIR"

package/deps/librdkafka/packaging/tools/build-release-artifacts.sh

@@ -0,0 +1,139 @@
+#!/bin/sh
+#
+# ^ NOTE: This needs to be sh, not bash, for alpine compatibility.
+#
+#
+# Build dynamic and statically linked librdkafka libraries useful for
+# release artifacts in high-level clients.
+#
+# Requires docker.
+# Supported docker images:
+#   alpine:3.16
+#   quay.io/pypa/manylinux_2_28_aarch64 (centos8)
+#   quay.io/pypa/manylinux_2_28_x86_64  (centos8)
+#
+# Usage:
+# packaging/tools/build-release-artifacts.sh [--disable-gssapi] <docker-image> <relative-output-tarball-path.tgz>
+#
+# The output path must be a relative path and inside the librdkafka directory
+# structure.
+#
+
+set -e
+
+docker_image=""
+extra_pkgs_rpm=""
+extra_pkgs_apk=""
+extra_config_args=""
+expected_features="gzip snappy ssl sasl regex lz4 sasl_plain sasl_scram plugins zstd sasl_oauthbearer http oidc"
+
+# Since cyrus-sasl is the only non-statically-linkable dependency,
+# we provide a --disable-gssapi option so that two different libraries
+# can be built: one with GSSAPI/Kerberos support, and one without, depending
+# on this option.
+if [ "$1" = "--disable-gssapi" ]; then
+    extra_config_args="${extra_config_args} --disable-gssapi"
+    disable_gssapi="$1"
+    shift
+else
+    extra_pkgs_rpm="${extra_pkgs_rpm} cyrus-sasl cyrus-sasl-devel"
+    extra_pkgs_apk="${extra_pkgs_apk} cyrus-sasl cyrus-sasl-dev"
+    expected_features="${expected_features} sasl_gssapi"
+    disable_gssapi=""
+fi
+
+# Check if we're running on the host or the (docker) build target.
+if [ "$1" = "--in-docker" -a $# -eq 2 ]; then
+    output="$2"
+elif [ $# -eq 2 ]; then
+    docker_image="$1"
+    output="$2"
+else
+    echo "Usage: $0 [--disable-gssapi] <manylinux-docker-image> <output-path.tgz>"
+    exit 1
+fi
+
+if [ -n "$docker_image" ]; then
+    # Running on the host, spin up the docker builder.
+    exec docker run -v "$PWD:/v" $docker_image /v/packaging/tools/build-release-artifacts.sh $disable_gssapi --in-docker "/v/$output"
+    # Only reached on exec error
+    exit $?
+fi
+
+
+########################################################################
+# Running in the docker instance, this is where we perform the build.  #
+########################################################################
+
+
+# Packages required for building librdkafka (perl is for openssl).
+
+if grep -q alpine /etc/os-release 2>/dev/null ; then
+    # Alpine
+    apk add \
+        bash curl gcc g++ make musl-dev linux-headers bsd-compat-headers git \
+        python3 perl patch $extra_pkgs_apk
+
+else
+    # CentOS
+    yum install -y libstdc++-devel gcc gcc-c++ python3 git perl-IPC-Cmd perl-Pod-Html $extra_pkgs_rpm
+fi
+
+
+# Clone the repo so other builds are unaffected of what we're doing
+# and we get a pristine build tree.
+git config --system --add safe.directory '/v/.git'
+git config --system --add safe.directory '/librdkafka/.git'
+git clone /v /librdkafka
+
+cd /librdkafka
+
+# Build librdkafka
+./configure \
+    --install-deps --source-deps-only --disable-lz4-ext \
+    --enable-static --enable-strip $extra_config_args
+
+make -j
+
+# Show library linkage (for troubleshooting) and checksums (for verification)
+for lib in src/librdkafka.so.1 src-cpp/librdkafka++.so.1; do
+    echo "$0: LINKAGE ${lib}:"
+    ldd src/librdkafka.so.1
+    echo "$0: SHA256 ${lib}:"
+    sha256sum "$lib"
+done
+
+# Verify that expected features are indeed built.
+features=$(examples/rdkafka_example -X builtin.features)
+echo "$0: FEATURES: $features"
+
+missing=""
+for f in $expected_features; do
+    if ! echo "$features" | grep -q "$f" ; then
+        echo "$0: BUILD IS MISSING FEATURE $f"
+        missing="${missing} $f"
+    fi
+done
+
+if [ -n "$missing" ]; then
+    exit 1
+fi
+
+
+# Run quick test suite, mark it as CI to avoid time/resource sensitive
+# tests to fail in case the worker is under-powered.
+CI=true make -C tests run_local_quick
+
+
+# Install librdkafka and then make a tar ball of the installed files.
+mkdir -p /destdir
+
+DESTDIR=/destdir make install
+
+cd /destdir
+tar cvzf "$output" .
+
+# Emit output hash so that build logs can be used to verify artifacts later.
+echo "$0: SHA256 $output:"
+sha256sum "$output"
+