@point3/node-rdkafka 3.6.0-1

package/deps/librdkafka/packaging/tools/build-release-artifacts.sh

@@ -0,0 +1,139 @@
+#!/bin/sh
+#
+# ^ NOTE: This needs to be sh, not bash, for alpine compatibility.
+#
+#
+# Build dynamic and statically linked librdkafka libraries useful for
+# release artifacts in high-level clients.
+#
+# Requires docker.
+# Supported docker images:
+#   alpine:3.16
+#   quay.io/pypa/manylinux_2_28_aarch64 (centos8)
+#   quay.io/pypa/manylinux_2_28_x86_64  (centos8)
+#
+# Usage:
+# packaging/tools/build-release-artifacts.sh [--disable-gssapi] <docker-image> <relative-output-tarball-path.tgz>
+#
+# The output path must be a relative path and inside the librdkafka directory
+# structure.
+#
+
+set -e
+
+docker_image=""
+extra_pkgs_rpm=""
+extra_pkgs_apk=""
+extra_config_args=""
+expected_features="gzip snappy ssl sasl regex lz4 sasl_plain sasl_scram plugins zstd sasl_oauthbearer http oidc"
+
+# Since cyrus-sasl is the only non-statically-linkable dependency,
+# we provide a --disable-gssapi option so that two different libraries
+# can be built: one with GSSAPI/Kerberos support, and one without, depending
+# on this option.
+if [ "$1" = "--disable-gssapi" ]; then
+    extra_config_args="${extra_config_args} --disable-gssapi"
+    disable_gssapi="$1"
+    shift
+else
+    extra_pkgs_rpm="${extra_pkgs_rpm} cyrus-sasl cyrus-sasl-devel"
+    extra_pkgs_apk="${extra_pkgs_apk} cyrus-sasl cyrus-sasl-dev"
+    expected_features="${expected_features} sasl_gssapi"
+    disable_gssapi=""
+fi
+
+# Check if we're running on the host or the (docker) build target.
+if [ "$1" = "--in-docker" -a $# -eq 2 ]; then
+    output="$2"
+elif [ $# -eq 2 ]; then
+    docker_image="$1"
+    output="$2"
+else
+    echo "Usage: $0 [--disable-gssapi] <manylinux-docker-image> <output-path.tgz>"
+    exit 1
+fi
+
+if [ -n "$docker_image" ]; then
+    # Running on the host, spin up the docker builder.
+    exec docker run -v "$PWD:/v" $docker_image /v/packaging/tools/build-release-artifacts.sh $disable_gssapi --in-docker "/v/$output"
+    # Only reached on exec error
+    exit $?
+fi
+
+
+########################################################################
+# Running in the docker instance, this is where we perform the build.  #
+########################################################################
+
+
+# Packages required for building librdkafka (perl is for openssl).
+
+if grep -q alpine /etc/os-release 2>/dev/null ; then
+    # Alpine
+    apk add \
+        bash curl gcc g++ make musl-dev linux-headers bsd-compat-headers git \
+        python3 perl patch $extra_pkgs_apk
+
+else
+    # CentOS
+    yum install -y libstdc++-devel gcc gcc-c++ python3 git perl-IPC-Cmd perl-Pod-Html $extra_pkgs_rpm
+fi
+
+
+# Clone the repo so other builds are unaffected of what we're doing
+# and we get a pristine build tree.
+git config --system --add safe.directory '/v/.git'
+git config --system --add safe.directory '/librdkafka/.git'
+git clone /v /librdkafka
+
+cd /librdkafka
+
+# Build librdkafka
+./configure \
+    --install-deps --source-deps-only --disable-lz4-ext \
+    --enable-static --enable-strip $extra_config_args
+
+make -j
+
+# Show library linkage (for troubleshooting) and checksums (for verification)
+for lib in src/librdkafka.so.1 src-cpp/librdkafka++.so.1; do
+    echo "$0: LINKAGE ${lib}:"
+    ldd src/librdkafka.so.1
+    echo "$0: SHA256 ${lib}:"
+    sha256sum "$lib"
+done
+
+# Verify that expected features are indeed built.
+features=$(examples/rdkafka_example -X builtin.features)
+echo "$0: FEATURES: $features"
+
+missing=""
+for f in $expected_features; do
+    if ! echo "$features" | grep -q "$f" ; then
+        echo "$0: BUILD IS MISSING FEATURE $f"
+        missing="${missing} $f"
+    fi
+done
+
+if [ -n "$missing" ]; then
+    exit 1
+fi
+
+
+# Run quick test suite, mark it as CI to avoid time/resource sensitive
+# tests to fail in case the worker is under-powered.
+CI=true make -C tests run_local_quick
+
+
+# Install librdkafka and then make a tar ball of the installed files.
+mkdir -p /destdir
+
+DESTDIR=/destdir make install
+
+cd /destdir
+tar cvzf "$output" .
+
+# Emit output hash so that build logs can be used to verify artifacts later.
+echo "$0: SHA256 $output:"
+sha256sum "$output"
+

package/deps/librdkafka/packaging/tools/distro-build.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# Build librdkafka for different distros to produce distro-specific artifacts.
+# Requires docker.
+#
+
+set -e
+
+distro=$1
+shift
+config_args=$*
+
+case $distro in
+    manylinux*)
+        # Any pypa/manylinux docker image build.
+        docker run -t -v "$PWD:/v" quay.io/pypa/$distro /v/packaging/tools/build-manylinux.sh /v /v/artifacts/librdkafka-${distro}.tgz $config_args
+        ;;
+    centos)
+        if [[ -n $config_args ]]; then
+            echo "Warning: configure arguments ignored for centos RPM build"
+        fi
+        packaging/rpm/mock-on-docker.sh
+        packaging/rpm/tests/test-on-docker.sh
+        ;;
+    debian)
+        docker run -it -v "$PWD:/v" mcr.microsoft.com/dotnet/sdk:3.1 /v/packaging/tools/build-debian.sh /v /v/artifacts/librdkafka-debian9.tgz $config_args
+        ;;
+    alpine)
+        packaging/alpine/build-alpine.sh $config_args
+        ;;
+    alpine-static)
+        packaging/alpine/build-alpine.sh --enable-static --source-deps-only $config_args
+        ;;
+    *)
+        echo "Usage: $0 <centos|debian|alpine|alpine-static>"
+        exit 1
+        ;;
+esac

package/deps/librdkafka/packaging/tools/gh-release-checksums.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+#
+# Calculate checksums for GitHub release artifacts/assets.
+#
+# Use the direct links rather than getting the tarball URLs from
+# the GitHub API since the latter uses the git-sha1 rather than the tag
+# in its zipped up content, causing checksum mismatches.
+#
+
+import sys
+import requests
+import hashlib
+
+
+if __name__ == '__main__':
+
+    if len(sys.argv) != 2:
+        print("Usage: {} <tag>".format(sys.argv[0]))
+        sys.exit(1)
+
+    tag = sys.argv[1]
+
+    print("## Checksums")
+    print("Release asset checksums:")
+
+    for ftype in ["zip", "tar.gz"]:
+        url = "https://github.com/confluentinc/" + \
+            "librdkafka/archive/{}.{}".format(tag, ftype)
+
+        h = hashlib.sha256()
+
+        r = requests.get(url, stream=True)
+        while True:
+            buf = r.raw.read(100 * 1000)
+            if len(buf) == 0:
+                break
+            h.update(buf)
+
+        print(" * {}.{} SHA256 `{}`".format(tag, ftype, h.hexdigest()))

package/deps/librdkafka/packaging/tools/rdutcoverage.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Verify that code coverage numbers are not reused in multiple places.
+#
+
+set -e
+
+echo "Checking for duplicate coverage numbers:"
+cnt=0
+for d in $(egrep -Rsoh 'RD_UT_COVERAGE\([[:digit:]]+\)' src \
+               | sort | uniq -c | \
+               egrep -v '^[[:space:]]*1 ' | awk '{print $2}'); do
+    grep -RsnF "$d" src
+    cnt=$(expr $cnt + 1)
+done
+
+echo ""
+
+if [[ $cnt -gt 0 ]]; then
+    echo "$cnt duplicates found: please use unique numbers"
+    exit 1
+else
+    echo "No duplicate(s) found"
+    exit 0
+fi

package/deps/librdkafka/packaging/tools/requirements.txt

@@ -0,0 +1,2 @@
+flake8
+autopep8

package/deps/librdkafka/packaging/tools/run-in-docker.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+if [ "$#" -lt 2 ]; then
+    echo "Usage: $0 <docker-image> [<args>...]"
+    exit 1
+fi
+
+IMAGE=$1
+SCRIPT_DIR=$(dirname "$0")
+ENTRYPOINT=${2}
+REST=${@:3}
+
+if [ $(which cache) ]; then
+    cache restore ${IMAGE}.tar
+fi
+if [ ! -f ./${IMAGE}.tar ]; then
+    docker build -f $SCRIPT_DIR/Dockerfile -t $IMAGE --build-arg UID=$UID .
+    docker save $IMAGE -o ./${IMAGE}.tar
+
+    if [ $(which cache) ]; then
+        cache store ${IMAGE}.tar ./${IMAGE}.tar
+    fi
+else
+    docker load -i ./${IMAGE}.tar
+fi
+
+docker run --rm --entrypoint $ENTRYPOINT \
+    -v .:/librdkafka -w /librdkafka -e CI -u $UID:$UID ${IMAGE} ${REST}

package/deps/librdkafka/packaging/tools/run-integration-tests.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -e
+
+if [ "$#" -lt 2 ]; then
+    echo "Usage: $0 <kafka-version> <cp-version> [<consumer_group_protocol>]"
+    echo "Example: $0 3.9.0 7.9.0"
+    exit 1
+fi
+
+KAFKA_VERSION=$1
+CP_VERSION=$2
+if [ -n "$3" ]; then
+    export TEST_CONSUMER_GROUP_PROTOCOL=$3
+fi
+
+source /home/user/venv/bin/activate
+./configure --install-deps --enable-werror --enable-devel
+./packaging/tools/rdutcoverage.sh
+make copyright-check
+make -j all examples check
+echo "Verifying that CONFIGURATION.md does not have manual changes"
+git diff --exit-code CONFIGURATION.md
+examples/rdkafka_example -X builtin.features
+ldd src/librdkafka.so.1
+ldd src-cpp/librdkafka++.so.1
+make -j -C tests build
+make -C tests run_local_quick
+DESTDIR="$PWD/dest" make install
+(cd tests && python3 -m trivup.clusters.KafkaCluster --kraft \
+ --version ${KAFKA_VERSION} \
+ --cpversion ${CP_VERSION} --cmd 'make quick')

package/deps/librdkafka/packaging/tools/run-style-check.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+source /home/user/venv/bin/activate
+make style-check

package/deps/librdkafka/packaging/tools/style-format.sh

@@ -0,0 +1,149 @@
+#!/bin/bash
+#
+# Check or apply/fix the project coding style to all files passed as arguments.
+# Uses clang-format for C/C++ and flake8 for Python.
+#
+
+
+
+# Requires clang-format version 18  (apt install clang-format-18).
+CLANG_FORMAT=${CLANG_FORMAT:-clang-format-18}
+CLANG_FORMAT_REQUIRED_VERSION=18
+
+set -e
+
+ret=0
+
+if [[ -z $1 ]]; then
+    echo "Usage: $0 [--fix] srcfile1.c srcfile2.h srcfile3.c ..."
+    echo ""
+    exit 0
+fi
+
+if [[ $1 == "--fix" ]]; then
+    fix=1
+    shift
+else
+    fix=0
+fi
+
+clang_format_version=$(${CLANG_FORMAT} --version | sed -Ee 's/.*version ([[:digit:]]+)\.[[:digit:]]+\.[[:digit:]]+.*/\1/')
+if [[ $clang_format_version != "$CLANG_FORMAT_REQUIRED_VERSION" ]] ; then
+    echo "$0: clang-format version ${CLANG_FORMAT_REQUIRED_VERSION}, '$clang_format_version' detected"
+    exit 1
+fi
+
+# Get list of files from .formatignore to ignore formatting for.
+ignore_files=( $(grep '^[^#]..' .formatignore) )
+
+function ignore {
+    local file=$1
+
+    local f
+    for f in "${ignore_files[@]}" ; do
+        [[ $file == $f ]] && return 0
+    done
+
+    return 1
+}
+
+# Read the C++ style from src-cpp/.clang-format and store it
+# in a json-like string which is passed to --style.
+# (It would be great if clang-format could take a file path for the
+#  format file..).
+cpp_style="{ $(grep -v '^...$' .clang-format-cpp | grep -v '^$' | tr '\n' ',' | sed -e 's/,$//') }"
+if [[ -z $cpp_style ]]; then
+    echo "$0: Unable to read .clang-format-cpp"
+    exit 1
+fi
+
+extra_info=""
+
+for f in $*; do
+
+    if ignore $f ; then
+        echo "$f is ignored by .formatignore" 1>&2
+        continue
+    fi
+
+    lang="c"
+    if [[ $f == *.cpp ]]; then
+        style="$cpp_style"
+        stylename="C++"
+    elif [[ $f == *.h && $(basename $f) == *cpp* ]]; then
+        style="$cpp_style"
+        stylename="C++ (header)"
+    elif [[ $f == *.py ]]; then
+        lang="py"
+        style="pep8"
+        stylename="pep8"
+    else
+        style="file"  # Use .clang-format
+        stylename="C"
+    fi
+
+    check=0
+
+    if [[ $fix == 1 ]]; then
+        # Convert tabs to 8 spaces first.
+        if grep -ql $'\t' "$f"; then
+            sed -i -e 's/\t/        /g' "$f"
+            echo "$f: tabs converted to spaces"
+        fi
+
+        if [[ $lang == c ]]; then
+            # Run clang-format to reformat the file
+            ${CLANG_FORMAT} --style="$style" "$f" > _styletmp
+
+        else
+            # Run autopep8 to reformat the file.
+            python3 -m autopep8 -a "$f" > _styletmp
+            # autopep8 can't fix all errors, so we also perform a flake8 check.
+            check=1
+        fi
+
+        if ! cmp -s "$f" _styletmp; then
+            echo "$f: style fixed ($stylename)"
+            # Use cp to preserve target file mode/attrs.
+            cp _styletmp "$f"
+            rm _styletmp
+        fi
+    fi
+
+    if [[ $fix == 0 || $check == 1 ]]; then
+        # Check for tabs
+        if grep -q $'\t' "$f" ; then
+            echo "$f: contains tabs: convert to 8 spaces instead"
+            ret=1
+        fi
+
+        # Check style
+        if [[ $lang == c ]]; then
+            if ! ${CLANG_FORMAT} --style="$style" --Werror --dry-run "$f" ; then
+                echo "$f: had style errors ($stylename): see clang-format output above"
+                ret=1
+            fi
+        elif [[ $lang == py ]]; then
+            if ! python3 -m flake8 "$f"; then
+                echo "$f: had style errors ($stylename): see flake8 output above"
+                if [[ $fix == 1 ]]; then
+                    # autopep8 couldn't fix all errors. Let the user know.
+                    extra_info="Error: autopep8 could not fix all errors, fix the flake8 errors manually and run again."
+                fi
+                ret=1
+            fi
+        fi
+    fi
+
+done
+
+rm -f _styletmp
+
+if [[ $ret != 0 ]]; then
+    echo ""
+    echo "You can run the following command to automatically fix the style:"
+    echo "  $ make style-fix"
+    [[ -n $extra_info ]] && echo "$extra_info"
+fi
+
+exit $ret

package/deps/librdkafka/packaging/tools/update_rpcs_max_versions.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+import json
+import sys
+import re
+
+# This script updates the Apache Kafka RPCs max versions.
+# It reads the input from `input_file`, which should be a table
+# looking like the table in `INTRODUCTION.md`.
+# Should look like this (without the first space after the comment):
+# | 0       | Produce                       | 12         | 10             |
+# | 1       | Fetch                         | 17         | 16             |
+# | 2       | ListOffsets                   | 10         | 7              |
+# | 3       | Metadata                      | 13         | 12             |
+# | 8       | OffsetCommit                  | 9          | 9              |
+# | 9       | OffsetFetch                   | 9          | 9              |
+# | 10      | FindCoordinator               | 6          | 2              |
+# | 11      | JoinGroup                     | 9          | 5              |
+# | 12      | Heartbeat                     | 4          | 3              |
+# | 13      | LeaveGroup                    | 5          | 1              |
+# | 14      | SyncGroup                     | 5          | 3              |
+# | 15      | DescribeGroups                | 6          | 4              |
+# | 16      | ListGroups                    | 5          | 4              |
+# | 17      | SaslHandshake                 | 1          | 1              |
+# | 18      | ApiVersions                   | 4          | 3              |
+# | 19      | CreateTopics                  | 7          | 4              |
+# | 20      | DeleteTopics                  | 6          | 1              |
+# | 21      | DeleteRecords                 | 2          | 1              |
+# | 22      | InitProducerId                | 5          | 4              |
+# | 23      | OffsetForLeaderEpoch          | 4          | 2              |
+# | 24      | AddPartitionsToTxn            | 5          | 0              |
+# | 25      | AddOffsetsToTxn               | 4          | 0              |
+# | 26      | EndTxn                        | 5          | 1              |
+# | 28      | TxnOffsetCommit               | 5          | 3              |
+# | 29      | DescribeAcls                  | 3          | 1              |
+# | 30      | CreateAcls                    | 3          | 1              |
+# | 31      | DeleteAcls                    | 3          | 1              |
+# | 32      | DescribeConfigs               | 4          | 1              |
+# | 33      | AlterConfigs                  | 2          | 2              |
+# | 36      | SaslAuthenticate              | 2          | 1              |
+# | 37      | CreatePartitions              | 3          | 0              |
+# | 42      | DeleteGroups                  | 2          | 1              |
+# | 43      | ElectLeaders                  | 2          | 2              |
+# | 44      | IncrementalAlterConfigs       | 1          | 1              |
+# | 47      | OffsetDelete                  | 0          | 0              |
+# | 50      | DescribeUserScramCredentials  | 0          | 0              |
+# | 51      | AlterUserScramCredentials     | 0          | 0              |
+# | 68      | ConsumerGroupHeartbeat        | 1          | 1              |
+# | 69      | ConsumerGroupDescribe         | 1          | 0              |
+# | 71      | GetTelemetrySubscriptions     | 0          | 0              |
+# | 72      | PushTelemetry                 | 0          | 0              |
+#
+# Output will be the same with max versions updated
+# Should pass Apache Kafka root folder as first argument and the input file
+# as second argument.
+ak_folder = sys.argv[1]
+input_file = sys.argv[2]
+
+if len(sys.argv) != 3:
+    print("Usage: python3 update_rpcs_max_versions.py <kafka_folder> "
+          "<input_file>")
+    sys.exit(1)
+
+with open(input_file, 'r') as input:
+    lines = input.readlines()
+    max_first_column = 0
+    max_second_column = 0
+    max_third_column = 0
+    apis = []
+    for line in lines:
+        line = re.sub('^\\s*\\|\\s*', '', line)
+        pipe_char = line.find('|')
+        max_first_column = max(max_first_column, pipe_char)
+        api_num = int(line[0:pipe_char])
+        line = line[pipe_char + 1:]
+        line = re.sub('^\\s*', '', line)
+        pipe_char = line.find('|')
+        max_second_column = max(max_second_column, pipe_char)
+        api = line[0:pipe_char].strip()
+        line = line[pipe_char + 1:].lstrip()
+        pipe_char = line.find('|')
+        max_third_column = max(max_third_column, pipe_char)
+        rest = line[pipe_char + 1:].strip()
+        apis.append((api_num, api, rest))
+
+    for api_num, api, rest in apis:
+        with open(f'{ak_folder}/clients/src/main/resources/common/message/'
+                  f'{api}Request.json',
+                  'r') as f:
+            text = f.readlines()
+            text = "".join([line for line in text
+                            if '#' not in line and '//' not in line])
+            json_object = json.loads(text)
+            max_version = json_object["validVersions"].split("-")[-1]
+            print('| ', end='')
+            print(str(api_num).ljust(max_first_column), end='')
+            print('| ', end='')
+            print(api.ljust(max_second_column), end='')
+            print('| ', end='')
+            print(str(max_version).ljust(max_third_column) + '|', end='')
+            print(f' {rest}')