@platformatic/rdkafka 4.0.0

package/deps/librdkafka/packaging/tools/distro-build.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# Build librdkafka for different distros to produce distro-specific artifacts.
+# Requires docker.
+#
+
+set -e
+
+distro=$1
+shift
+config_args=$*
+
+case $distro in
+    manylinux*)
+        # Any pypa/manylinux docker image build.
+        docker run -t -v "$PWD:/v" quay.io/pypa/$distro /v/packaging/tools/build-manylinux.sh /v /v/artifacts/librdkafka-${distro}.tgz $config_args
+        ;;
+    centos)
+        if [[ -n $config_args ]]; then
+            echo "Warning: configure arguments ignored for centos RPM build"
+        fi
+        packaging/rpm/mock-on-docker.sh
+        packaging/rpm/tests/test-on-docker.sh
+        ;;
+    debian)
+        docker run -it -v "$PWD:/v" mcr.microsoft.com/dotnet/sdk:3.1 /v/packaging/tools/build-debian.sh /v /v/artifacts/librdkafka-debian9.tgz $config_args
+        ;;
+    alpine)
+        packaging/alpine/build-alpine.sh $config_args
+        ;;
+    alpine-static)
+        packaging/alpine/build-alpine.sh --enable-static --source-deps-only $config_args
+        ;;
+    *)
+        echo "Usage: $0 <centos|debian|alpine|alpine-static>"
+        exit 1
+        ;;
+esac

package/deps/librdkafka/packaging/tools/gh-release-checksums.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+#
+# Calculate checksums for GitHub release artifacts/assets.
+#
+# Use the direct links rather than getting the tarball URLs from
+# the GitHub API since the latter uses the git-sha1 rather than the tag
+# in its zipped up content, causing checksum mismatches.
+#
+
+import sys
+import requests
+import hashlib
+
+
+if __name__ == '__main__':
+
+    if len(sys.argv) != 2:
+        print("Usage: {} <tag>".format(sys.argv[0]))
+        sys.exit(1)
+
+    tag = sys.argv[1]
+
+    print("## Checksums")
+    print("Release asset checksums:")
+
+    for ftype in ["zip", "tar.gz"]:
+        url = "https://github.com/confluentinc/" + \
+            "librdkafka/archive/{}.{}".format(tag, ftype)
+
+        h = hashlib.sha256()
+
+        r = requests.get(url, stream=True)
+        while True:
+            buf = r.raw.read(100 * 1000)
+            if len(buf) == 0:
+                break
+            h.update(buf)
+
+        print(" * {}.{} SHA256 `{}`".format(tag, ftype, h.hexdigest()))

package/deps/librdkafka/packaging/tools/rdutcoverage.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Verify that code coverage numbers are not reused in multiple places.
+#
+
+set -e
+
+echo "Checking for duplicate coverage numbers:"
+cnt=0
+for d in $(egrep -Rsoh 'RD_UT_COVERAGE\([[:digit:]]+\)' src \
+               | sort | uniq -c | \
+               egrep -v '^[[:space:]]*1 ' | awk '{print $2}'); do
+    grep -RsnF "$d" src
+    cnt=$(expr $cnt + 1)
+done
+
+echo ""
+
+if [[ $cnt -gt 0 ]]; then
+    echo "$cnt duplicates found: please use unique numbers"
+    exit 1
+else
+    echo "No duplicate(s) found"
+    exit 0
+fi

package/deps/librdkafka/packaging/tools/requirements.txt

@@ -0,0 +1,2 @@
+flake8
+autopep8

package/deps/librdkafka/packaging/tools/style-format.sh

@@ -0,0 +1,148 @@
+#!/bin/bash
+#
+# Check or apply/fix the project coding style to all files passed as arguments.
+# Uses clang-format for C/C++ and flake8 for Python.
+#
+# Requires clang-format version 10  (apt install clang-format-10).
+#
+
+
+CLANG_FORMAT=${CLANG_FORMAT:-clang-format}
+
+set -e
+
+ret=0
+
+if [[ -z $1 ]]; then
+    echo "Usage: $0 [--fix] srcfile1.c srcfile2.h srcfile3.c ..."
+    echo ""
+    exit 0
+fi
+
+if [[ $1 == "--fix" ]]; then
+    fix=1
+    shift
+else
+    fix=0
+fi
+
+clang_format_version=$(${CLANG_FORMAT} --version | sed -Ee 's/.*version ([[:digit:]]+)\.[[:digit:]]+\.[[:digit:]]+.*/\1/')
+if [[ $clang_format_version != "10" ]] ; then
+    echo "$0: clang-format version 10, '$clang_format_version' detected"
+    exit 1
+fi
+
+# Get list of files from .formatignore to ignore formatting for.
+ignore_files=( $(grep '^[^#]..' .formatignore) )
+
+function ignore {
+    local file=$1
+
+    local f
+    for f in "${ignore_files[@]}" ; do
+        [[ $file == $f ]] && return 0
+    done
+
+    return 1
+}
+
+# Read the C++ style from src-cpp/.clang-format and store it
+# in a json-like string which is passed to --style.
+# (It would be great if clang-format could take a file path for the
+#  format file..).
+cpp_style="{ $(grep -v '^...$' .clang-format-cpp | grep -v '^$' | tr '\n' ',' | sed -e 's/,$//') }"
+if [[ -z $cpp_style ]]; then
+    echo "$0: Unable to read .clang-format-cpp"
+    exit 1
+fi
+
+extra_info=""
+
+for f in $*; do
+
+    if ignore $f ; then
+        echo "$f is ignored by .formatignore" 1>&2
+        continue
+    fi
+
+    lang="c"
+    if [[ $f == *.cpp ]]; then
+        style="$cpp_style"
+        stylename="C++"
+    elif [[ $f == *.h && $(basename $f) == *cpp* ]]; then
+        style="$cpp_style"
+        stylename="C++ (header)"
+    elif [[ $f == *.py ]]; then
+        lang="py"
+        style="pep8"
+        stylename="pep8"
+    else
+        style="file"  # Use .clang-format
+        stylename="C"
+    fi
+
+    check=0
+
+    if [[ $fix == 1 ]]; then
+        # Convert tabs to 8 spaces first.
+        if grep -ql $'\t' "$f"; then
+            sed -i -e 's/\t/        /g' "$f"
+            echo "$f: tabs converted to spaces"
+        fi
+
+        if [[ $lang == c ]]; then
+            # Run clang-format to reformat the file
+            ${CLANG_FORMAT} --style="$style" "$f" > _styletmp
+
+        else
+            # Run autopep8 to reformat the file.
+            python3 -m autopep8 -a "$f" > _styletmp
+            # autopep8 can't fix all errors, so we also perform a flake8 check.
+            check=1
+        fi
+
+        if ! cmp -s "$f" _styletmp; then
+            echo "$f: style fixed ($stylename)"
+            # Use cp to preserve target file mode/attrs.
+            cp _styletmp "$f"
+            rm _styletmp
+        fi
+    fi
+
+    if [[ $fix == 0 || $check == 1 ]]; then
+        # Check for tabs
+        if grep -q $'\t' "$f" ; then
+            echo "$f: contains tabs: convert to 8 spaces instead"
+            ret=1
+        fi
+
+        # Check style
+        if [[ $lang == c ]]; then
+            if ! ${CLANG_FORMAT} --style="$style" --Werror --dry-run "$f" ; then
+                echo "$f: had style errors ($stylename): see clang-format output above"
+                ret=1
+            fi
+        elif [[ $lang == py ]]; then
+            if ! python3 -m flake8 "$f"; then
+                echo "$f: had style errors ($stylename): see flake8 output above"
+                if [[ $fix == 1 ]]; then
+                    # autopep8 couldn't fix all errors. Let the user know.
+                    extra_info="Error: autopep8 could not fix all errors, fix the flake8 errors manually and run again."
+                fi
+                ret=1
+            fi
+        fi
+    fi
+
+done
+
+rm -f _styletmp
+
+if [[ $ret != 0 ]]; then
+    echo ""
+    echo "You can run the following command to automatically fix the style:"
+    echo "  $ make style-fix"
+    [[ -n $extra_info ]] && echo "$extra_info"
+fi
+
+exit $ret

package/deps/librdkafka/service.yml

@@ -0,0 +1,18 @@
+name: librdkafka
+lang: unknown
+lang_version: unknown
+git:
+  enable: true
+github:
+  enable: true
+semaphore:
+  enable: true
+  pipeline_enable: false
+  triggers:
+  - tags
+  - branches
+  branches:
+  - master
+  - /semaphore.*/
+  - /dev_.*/
+  - /feature\/.*/

package/deps/librdkafka/src/CMakeLists.txt

@@ -0,0 +1,374 @@
+set(LIBVER 1)
+
+set(
+    sources
+    crc32c.c
+    rdaddr.c
+    rdavl.c
+    rdbuf.c
+    rdcrc32.c
+    rdfnv1a.c
+    rdbase64.c
+    rdkafka.c
+    rdkafka_assignor.c
+    rdkafka_broker.c
+    rdkafka_buf.c
+    rdkafka_cgrp.c
+    rdkafka_conf.c
+    rdkafka_event.c
+    rdkafka_feature.c
+    rdkafka_lz4.c
+    rdkafka_metadata.c
+    rdkafka_metadata_cache.c
+    rdkafka_msg.c
+    rdkafka_msgset_reader.c
+    rdkafka_msgset_writer.c
+    rdkafka_offset.c
+    rdkafka_op.c
+    rdkafka_partition.c
+    rdkafka_pattern.c
+    rdkafka_queue.c
+    rdkafka_range_assignor.c
+    rdkafka_request.c
+    rdkafka_roundrobin_assignor.c
+    rdkafka_sasl.c
+    rdkafka_sasl_plain.c
+    rdkafka_sticky_assignor.c
+    rdkafka_subscription.c
+    rdkafka_assignment.c
+    rdkafka_timer.c
+    rdkafka_topic.c
+    rdkafka_transport.c
+    rdkafka_interceptor.c
+    rdkafka_header.c
+    rdkafka_admin.c
+    rdkafka_aux.c
+    rdkafka_background.c
+    rdkafka_idempotence.c
+    rdkafka_txnmgr.c
+    rdkafka_cert.c
+    rdkafka_coord.c
+    rdkafka_mock.c
+    rdkafka_mock_handlers.c
+    rdkafka_mock_cgrp.c
+    rdkafka_error.c
+    rdkafka_fetcher.c
+    rdkafka_telemetry.c
+    rdkafka_telemetry_decode.c
+    rdkafka_telemetry_encode.c
+    nanopb/pb_encode.c
+    nanopb/pb_decode.c
+    nanopb/pb_common.c
+    opentelemetry/metrics.pb.c
+    opentelemetry/common.pb.c
+    opentelemetry/resource.pb.c
+    rdlist.c
+    rdlog.c
+    rdmurmur2.c
+    rdports.c
+    rdrand.c
+    rdregex.c
+    rdstring.c
+    rdunittest.c
+    rdvarint.c
+    rdmap.c
+    snappy.c
+    tinycthread.c
+    tinycthread_extra.c
+    rdxxhash.c
+    cJSON.c
+)
+
+if(WITH_SSL)
+  list(APPEND sources rdkafka_ssl.c)
+endif()
+
+if(WITH_CURL)
+ list(APPEND sources rdhttp.c)
+endif()
+
+if(WITH_HDRHISTOGRAM)
+  list(APPEND sources rdhdrhistogram.c)
+endif()
+
+if(WITH_LIBDL OR WIN32)
+  list(APPEND sources rddl.c)
+endif()
+
+if(WITH_PLUGINS)
+  list(APPEND sources rdkafka_plugin.c)
+endif()
+
+if(WIN32)
+  list(APPEND sources rdkafka_sasl_win32.c)
+elseif(WITH_SASL_CYRUS)
+  list(APPEND sources rdkafka_sasl_cyrus.c)
+endif()
+
+if(WITH_SASL_SCRAM)
+  list(APPEND sources rdkafka_sasl_scram.c)
+endif()
+
+if(WITH_SASL_OAUTHBEARER)
+  list(APPEND sources rdkafka_sasl_oauthbearer.c)
+endif()
+
+if(WITH_OAUTHBEARER_OIDC)
+  list(APPEND sources rdkafka_sasl_oauthbearer_oidc.c)
+endif()
+
+if(WITH_ZLIB)
+  list(APPEND sources rdgz.c)
+endif()
+
+if(WITH_ZSTD)
+  list(APPEND sources rdkafka_zstd.c)
+endif()
+
+if(NOT WITH_LZ4_EXT)
+  list(APPEND sources lz4.c lz4frame.c lz4hc.c)
+endif()
+
+if(NOT HAVE_REGEX)
+  list(APPEND sources regexp.c)
+endif()
+
+# Define flags with cmake instead of by defining them on win32_config.h
+if(WITHOUT_WIN32_CONFIG)
+  list(APPEND rdkafka_compile_definitions WITHOUT_WIN32_CONFIG)
+  if(WITH_SSL)
+    list(APPEND rdkafka_compile_definitions WITH_SSL=1)
+  else()
+    list(APPEND rdkafka_compile_definitions WITH_SSL=0)
+  endif(WITH_SSL)
+  if(WITH_ZLIB)
+    list(APPEND rdkafka_compile_definitions WITH_ZLIB=1)
+  else()
+    list(APPEND rdkafka_compile_definitions WITH_ZLIB=0)
+  endif(WITH_ZLIB)
+  if(WITH_SNAPPY)
+    list(APPEND rdkafka_compile_definitions WITH_SNAPPY=1)
+  else()
+    list(APPEND rdkafka_compile_definitions WITH_SNAPPY=0)
+  endif(WITH_SNAPPY)
+  if(WITH_ZSTD)
+    list(APPEND rdkafka_compile_definitions WITH_ZSTD=1)
+  else()
+    list(APPEND rdkafka_compile_definitions WITH_ZSTD=0)
+  endif(WITH_ZSTD)
+  if(WITH_SASL_SCRAM)
+    list(APPEND rdkafka_compile_definitions WITH_SASL_SCRAM=1)
+  else()
+    list(APPEND rdkafka_compile_definitions WITH_SASL_SCRAM=0)
+  endif(WITH_SASL_SCRAM)
+  if(WITH_SASL_OAUTHBEARER)
+    list(APPEND rdkafka_compile_definitions WITH_SASL_OAUTHBEARER=1)
+  else()
+    list(APPEND rdkafka_compile_definitions WITH_SASL_OAUTHBEARER=0)
+  endif(WITH_SASL_OAUTHBEARER)
+  if(ENABLE_DEVEL)
+    list(APPEND rdkafka_compile_definitions ENABLE_DEVEL=1)
+  else()
+    list(APPEND rdkafka_compile_definitions ENABLE_DEVEL=0)
+  endif(ENABLE_DEVEL)
+  if(WITH_PLUGINS)
+    list(APPEND rdkafka_compile_definitions WITH_PLUGINS=1)
+  else()
+    list(APPEND rdkafka_compile_definitions WITH_PLUGINS=0)
+  endif(WITH_PLUGINS)
+endif()
+
+if(RDKAFKA_BUILD_STATIC)
+  set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+  set(RDKAFKA_BUILD_MODE STATIC)
+else()
+  set(RDKAFKA_BUILD_MODE SHARED)
+endif()
+
+add_library(rdkafka ${RDKAFKA_BUILD_MODE} ${sources})
+if(NOT RDKAFKA_BUILD_STATIC)
+  set_property(TARGET rdkafka PROPERTY SOVERSION ${LIBVER})
+endif()
+
+if(MINGW)
+  # Target Windows 8.1 to match the VS projects (MinGW defaults to an older WinAPI version)
+  list(APPEND rdkafka_compile_definitions WINVER=0x0603 _WIN32_WINNT=0x0603 UNICODE)
+endif(MINGW)
+
+# Support '#include <rdkafka.h>'
+target_include_directories(rdkafka PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}> $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
+target_compile_definitions(rdkafka PUBLIC ${rdkafka_compile_definitions})
+if(RDKAFKA_BUILD_STATIC)
+  target_compile_definitions(rdkafka PUBLIC LIBRDKAFKA_STATICLIB)
+endif()
+
+# We need 'dummy' directory to support `#include "../config.h"` path
+set(dummy "${GENERATED_DIR}/dummy")
+file(MAKE_DIRECTORY "${dummy}")
+target_include_directories(rdkafka PUBLIC "$<BUILD_INTERFACE:${dummy}>")
+
+if(WITH_CURL)
+  find_package(CURL REQUIRED)
+  target_include_directories(rdkafka PRIVATE ${CURL_INCLUDE_DIRS})
+  target_link_libraries(rdkafka PUBLIC CURL::libcurl)
+endif()
+
+if(WITH_HDRHISTOGRAM)
+  target_link_libraries(rdkafka PUBLIC m)
+endif()
+
+if(WITH_ZLIB)
+  find_package(ZLIB REQUIRED)
+  target_include_directories(rdkafka PRIVATE ${ZLIB_INCLUDE_DIRS})
+  target_link_libraries(rdkafka PUBLIC ZLIB::ZLIB)
+endif()
+
+if(WITH_ZSTD)
+  target_link_libraries(rdkafka PRIVATE ${ZSTD_LIBRARY})
+  target_include_directories(rdkafka PRIVATE ${ZSTD_INCLUDE_DIR})
+  message(STATUS "Found ZSTD: ${ZSTD_LIBRARY}")
+endif()
+
+if(WITH_SSL)
+  if(WITH_BUNDLED_SSL) # option from 'h2o' parent project
+    if(NOT TARGET bundled-ssl)
+      message(FATAL_ERROR "bundled-ssl target not exist")
+    endif()
+    target_include_directories(rdkafka BEFORE PRIVATE ${BUNDLED_SSL_INCLUDE_DIR})
+    target_link_libraries(rdkafka PUBLIC ${BUNDLED_SSL_LIBRARIES})
+    add_dependencies(rdkafka bundled-ssl)
+  else()
+    find_package(OpenSSL REQUIRED)
+    target_include_directories(rdkafka PRIVATE ${OPENSSL_INCLUDE_DIR})
+    target_link_libraries(rdkafka PUBLIC OpenSSL::SSL OpenSSL::Crypto)
+    get_target_property(OPENSSL_TARGET_TYPE OpenSSL::SSL TYPE)
+    if(OPENSSL_CRYPTO_LIBRARY MATCHES "\\.a$")
+      target_compile_definitions(rdkafka PUBLIC WITH_STATIC_LIB_libcrypto)
+    endif()
+  endif()
+endif()
+
+if(LINK_ATOMIC)
+  target_link_libraries(rdkafka PUBLIC "-latomic")
+endif()
+
+find_package(Threads REQUIRED)
+target_link_libraries(rdkafka PUBLIC Threads::Threads)
+
+if(WITH_SASL_CYRUS)
+  target_include_directories(rdkafka PRIVATE ${SASL_INCLUDE_DIRS})
+  target_link_libraries(rdkafka PUBLIC ${SASL_LIBRARIES})
+endif()
+
+if(WITH_LIBDL)
+  target_link_libraries(rdkafka PUBLIC ${CMAKE_DL_LIBS})
+endif()
+
+if(WITH_LZ4_EXT)
+  target_include_directories(rdkafka PRIVATE ${LZ4_INCLUDE_DIRS})
+  target_link_libraries(rdkafka PUBLIC LZ4::LZ4)
+endif()
+
+if(WIN32)
+  if(WITH_SSL)
+    target_link_libraries(rdkafka PUBLIC crypt32)
+  endif()
+
+  target_link_libraries(rdkafka PUBLIC ws2_32 secur32)
+  if(NOT RDKAFKA_BUILD_STATIC)
+    target_compile_definitions(rdkafka PRIVATE LIBRDKAFKA_EXPORTS)
+  endif()
+endif()
+
+# Generate pkg-config file
+set(PKG_CONFIG_VERSION "${PROJECT_VERSION}")
+set(PKG_CONFIG_REQUIRES_PRIVATE "")
+if (WIN32)
+  set(PKG_CONFIG_LIBS_PRIVATE "-lws2_32 -lsecur32 -lcrypt32")
+else()
+  set(PKG_CONFIG_LIBS_PRIVATE "-lpthread")
+  find_library(RT_LIBRARY rt)
+  if(RT_LIBRARY)
+    string(APPEND PKG_CONFIG_LIBS_PRIVATE " -lrt")
+  endif()
+
+  if(WITH_LIBDL)
+    string(APPEND PKG_CONFIG_LIBS_PRIVATE " -ldl")
+  endif()
+
+  if(WITH_HDRHISTOGRAM)
+    string(APPEND PKG_CONFIG_LIBS_PRIVATE " -lm")
+  endif()
+endif()
+
+if(NOT RDKAFKA_BUILD_STATIC)
+  set(PKG_CONFIG_NAME "librdkafka")
+  set(PKG_CONFIG_DESCRIPTION "The Apache Kafka C/C++ library")
+
+  if(WITH_CURL)
+    string(APPEND PKG_CONFIG_REQUIRES_PRIVATE "libcurl ")
+  endif()
+
+  if(WITH_ZLIB)
+    string(APPEND PKG_CONFIG_REQUIRES_PRIVATE "zlib ")
+  endif()
+
+  if(WITH_SSL)
+    string(APPEND PKG_CONFIG_REQUIRES_PRIVATE "libcrypto libssl ")
+  endif()
+
+  if(WITH_SASL_CYRUS)
+    string(APPEND PKG_CONFIG_REQUIRES_PRIVATE "libsasl2 ")
+  endif()
+
+  if(WITH_ZSTD)
+    string(APPEND PKG_CONFIG_REQUIRES_PRIVATE "libzstd ")
+  endif()
+
+  if(WITH_LZ4_EXT)
+    string(APPEND PKG_CONFIG_REQUIRES_PRIVATE "liblz4 ")
+  endif()
+
+  set(PKG_CONFIG_CFLAGS "-I\${includedir}")
+  set(PKG_CONFIG_LIBS "-L\${libdir} -lrdkafka")
+
+  configure_file(
+    "../packaging/cmake/rdkafka.pc.in"
+    "${GENERATED_DIR}/rdkafka.pc"
+    @ONLY
+  )
+  install(
+    FILES ${GENERATED_DIR}/rdkafka.pc
+    DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig"
+  )
+else()
+  set(PKG_CONFIG_NAME "librdkafka-static")
+  set(PKG_CONFIG_DESCRIPTION "The Apache Kafka C/C++ library (static)")
+  set(PKG_CONFIG_CFLAGS "-I\${includedir} -DLIBRDKAFKA_STATICLIB")
+  set(PKG_CONFIG_LIBS "-L\${libdir} \${libdir}/librdkafka.a")
+  string(APPEND PKG_CONFIG_LIBS " ${PKG_CONFIG_LIBS_PRIVATE}")
+  set(PKG_CONFIG_LIBS_PRIVATE "")
+  configure_file(
+    "../packaging/cmake/rdkafka.pc.in"
+    "${GENERATED_DIR}/rdkafka-static.pc"
+    @ONLY
+  )
+  install(
+    FILES ${GENERATED_DIR}/rdkafka-static.pc
+    DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig"
+  )
+endif()
+
+install(
+    TARGETS rdkafka
+    EXPORT "${targets_export_name}"
+    LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
+    ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
+    RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
+    INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
+)
+
+install(
+    FILES "rdkafka.h" "rdkafka_mock.h"
+    DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/librdkafka"
+)