@platformatic/rdkafka 4.0.0 → 4.1.0-alpha.1

package/deps/librdkafka/packaging/mingw-w64/semaphoreci-build.sh

@@ -17,7 +17,7 @@ fi
 
 archive="${PWD}/$1"
 
-source ./packaging/mingw-w64/travis-before-install.sh
+source ./packaging/mingw-w64/export-variables.sh
 
 if [[ $linkage == "static" ]]; then
     ./packaging/mingw-w64/configure-build-msys2-mingw-static.sh

package/deps/librdkafka/packaging/nuget/requirements.txt

@@ -1,3 +1,3 @@
-boto3==1.18.45
-rpmfile==1.0.8
-filemagic==1.6
+boto3~=1.38
+rpmfile~=1.1
+filemagic~=1.6

package/deps/librdkafka/packaging/tools/build-configurations-checks.sh

@@ -1,10 +1,12 @@
 #!/bin/bash
 set -e
+
 # Disable all flags to make sure it
 # compiles correctly in all cases
 ./configure --install-deps --disable-ssl --disable-gssapi \
 --disable-curl --disable-zlib \
 --disable-zstd --disable-lz4-ext --disable-regex-ext \
---disable-c11threads --disable-syslog
+--disable-c11threads --disable-syslog \
+--enable-werror --enable-devel
 make -j
 make -j -C tests run_local_quick

package/deps/librdkafka/packaging/tools/run-in-docker.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+if [ "$#" -lt 2 ]; then
+    echo "Usage: $0 <docker-image> [<args>...]"
+    exit 1
+fi
+
+IMAGE=$1
+SCRIPT_DIR=$(dirname "$0")
+ENTRYPOINT=${2}
+REST=${@:3}
+
+if [ $(which cache) ]; then
+    cache restore ${IMAGE}.tar
+fi
+if [ ! -f ./${IMAGE}.tar ]; then
+    docker build -f $SCRIPT_DIR/Dockerfile -t $IMAGE --build-arg UID=$UID .
+    docker save $IMAGE -o ./${IMAGE}.tar
+
+    if [ $(which cache) ]; then
+        cache store ${IMAGE}.tar ./${IMAGE}.tar
+    fi
+else
+    docker load -i ./${IMAGE}.tar
+fi
+
+docker run --rm --entrypoint $ENTRYPOINT \
+    -v .:/librdkafka -w /librdkafka -e CI -u $UID:$UID ${IMAGE} ${REST}

package/deps/librdkafka/packaging/tools/run-integration-tests.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -e
+
+if [ "$#" -lt 2 ]; then
+    echo "Usage: $0 <kafka-version> <cp-version> [<consumer_group_protocol>]"
+    echo "Example: $0 3.9.0 7.9.0"
+    exit 1
+fi
+
+KAFKA_VERSION=$1
+CP_VERSION=$2
+if [ -n "$3" ]; then
+    export TEST_CONSUMER_GROUP_PROTOCOL=$3
+fi
+
+source /home/user/venv/bin/activate
+./configure --install-deps --enable-werror --enable-devel
+./packaging/tools/rdutcoverage.sh
+make copyright-check
+make -j all examples check
+echo "Verifying that CONFIGURATION.md does not have manual changes"
+git diff --exit-code CONFIGURATION.md
+examples/rdkafka_example -X builtin.features
+ldd src/librdkafka.so.1
+ldd src-cpp/librdkafka++.so.1
+make -j -C tests build
+make -C tests run_local_quick
+DESTDIR="$PWD/dest" make install
+(cd tests && python3 -m trivup.clusters.KafkaCluster --kraft \
+ --version ${KAFKA_VERSION} \
+ --cpversion ${CP_VERSION} --cmd 'make quick')

package/deps/librdkafka/packaging/tools/run-style-check.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+source /home/user/venv/bin/activate
+make style-check

package/deps/librdkafka/packaging/tools/style-format.sh

@@ -3,11 +3,12 @@
 # Check or apply/fix the project coding style to all files passed as arguments.
 # Uses clang-format for C/C++ and flake8 for Python.
 #
-# Requires clang-format version 10  (apt install clang-format-10).
-#
 
 
-CLANG_FORMAT=${CLANG_FORMAT:-clang-format}
+
+# Requires clang-format version 18  (apt install clang-format-18).
+CLANG_FORMAT=${CLANG_FORMAT:-clang-format-18}
+CLANG_FORMAT_REQUIRED_VERSION=18
 
 set -e
 
@@ -27,8 +28,8 @@ else
 fi
 
 clang_format_version=$(${CLANG_FORMAT} --version | sed -Ee 's/.*version ([[:digit:]]+)\.[[:digit:]]+\.[[:digit:]]+.*/\1/')
-if [[ $clang_format_version != "10" ]] ; then
-    echo "$0: clang-format version 10, '$clang_format_version' detected"
+if [[ $clang_format_version != "$CLANG_FORMAT_REQUIRED_VERSION" ]] ; then
+    echo "$0: clang-format version ${CLANG_FORMAT_REQUIRED_VERSION}, '$clang_format_version' detected"
     exit 1
 fi
 

package/deps/librdkafka/packaging/tools/update_rpcs_max_versions.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+import json
+import sys
+import re
+
+# This script updates the Apache Kafka RPCs max versions.
+# It reads the input from `input_file`, which should be a table
+# looking like the table in `INTRODUCTION.md`.
+# Should look like this (without the first space after the comment):
+# | 0       | Produce                       | 12         | 10             |
+# | 1       | Fetch                         | 17         | 16             |
+# | 2       | ListOffsets                   | 10         | 7              |
+# | 3       | Metadata                      | 13         | 12             |
+# | 8       | OffsetCommit                  | 9          | 9              |
+# | 9       | OffsetFetch                   | 9          | 9              |
+# | 10      | FindCoordinator               | 6          | 2              |
+# | 11      | JoinGroup                     | 9          | 5              |
+# | 12      | Heartbeat                     | 4          | 3              |
+# | 13      | LeaveGroup                    | 5          | 1              |
+# | 14      | SyncGroup                     | 5          | 3              |
+# | 15      | DescribeGroups                | 6          | 4              |
+# | 16      | ListGroups                    | 5          | 4              |
+# | 17      | SaslHandshake                 | 1          | 1              |
+# | 18      | ApiVersions                   | 4          | 3              |
+# | 19      | CreateTopics                  | 7          | 4              |
+# | 20      | DeleteTopics                  | 6          | 1              |
+# | 21      | DeleteRecords                 | 2          | 1              |
+# | 22      | InitProducerId                | 5          | 4              |
+# | 23      | OffsetForLeaderEpoch          | 4          | 2              |
+# | 24      | AddPartitionsToTxn            | 5          | 0              |
+# | 25      | AddOffsetsToTxn               | 4          | 0              |
+# | 26      | EndTxn                        | 5          | 1              |
+# | 28      | TxnOffsetCommit               | 5          | 3              |
+# | 29      | DescribeAcls                  | 3          | 1              |
+# | 30      | CreateAcls                    | 3          | 1              |
+# | 31      | DeleteAcls                    | 3          | 1              |
+# | 32      | DescribeConfigs               | 4          | 1              |
+# | 33      | AlterConfigs                  | 2          | 2              |
+# | 36      | SaslAuthenticate              | 2          | 1              |
+# | 37      | CreatePartitions              | 3          | 0              |
+# | 42      | DeleteGroups                  | 2          | 1              |
+# | 43      | ElectLeaders                  | 2          | 2              |
+# | 44      | IncrementalAlterConfigs       | 1          | 1              |
+# | 47      | OffsetDelete                  | 0          | 0              |
+# | 50      | DescribeUserScramCredentials  | 0          | 0              |
+# | 51      | AlterUserScramCredentials     | 0          | 0              |
+# | 68      | ConsumerGroupHeartbeat        | 1          | 1              |
+# | 69      | ConsumerGroupDescribe         | 1          | 0              |
+# | 71      | GetTelemetrySubscriptions     | 0          | 0              |
+# | 72      | PushTelemetry                 | 0          | 0              |
+#
+# Output will be the same with max versions updated
+# Should pass Apache Kafka root folder as first argument and the input file
+# as second argument.
+ak_folder = sys.argv[1]
+input_file = sys.argv[2]
+
+if len(sys.argv) != 3:
+    print("Usage: python3 update_rpcs_max_versions.py <kafka_folder> "
+          "<input_file>")
+    sys.exit(1)
+
+with open(input_file, 'r') as input:
+    lines = input.readlines()
+    max_first_column = 0
+    max_second_column = 0
+    max_third_column = 0
+    apis = []
+    for line in lines:
+        line = re.sub('^\\s*\\|\\s*', '', line)
+        pipe_char = line.find('|')
+        max_first_column = max(max_first_column, pipe_char)
+        api_num = int(line[0:pipe_char])
+        line = line[pipe_char + 1:]
+        line = re.sub('^\\s*', '', line)
+        pipe_char = line.find('|')
+        max_second_column = max(max_second_column, pipe_char)
+        api = line[0:pipe_char].strip()
+        line = line[pipe_char + 1:].lstrip()
+        pipe_char = line.find('|')
+        max_third_column = max(max_third_column, pipe_char)
+        rest = line[pipe_char + 1:].strip()
+        apis.append((api_num, api, rest))
+
+    for api_num, api, rest in apis:
+        with open(f'{ak_folder}/clients/src/main/resources/common/message/'
+                  f'{api}Request.json',
+                  'r') as f:
+            text = f.readlines()
+            text = "".join([line for line in text
+                            if '#' not in line and '//' not in line])
+            json_object = json.loads(text)
+            max_version = json_object["validVersions"].split("-")[-1]
+            print('| ', end='')
+            print(str(api_num).ljust(max_first_column), end='')
+            print('| ', end='')
+            print(api.ljust(max_second_column), end='')
+            print('| ', end='')
+            print(str(max_version).ljust(max_third_column) + '|', end='')
+            print(f' {rest}')

package/deps/librdkafka/service.yml

@@ -16,3 +16,157 @@ semaphore:
   - /semaphore.*/
   - /dev_.*/
   - /feature\/.*/
+  tasks:
+  - name: verify-linux-packages
+    branch: master
+    scheduled: false
+    pipeline_file: .semaphore/verify-linux-packages.yml
+    parameters:
+      - name: LIBRDKAFKA_VERSION
+        required: true
+        description: librdkafka version to verify
+  - name: run-all-tests
+    branch: master
+    scheduled: false
+    pipeline_file: .semaphore/run-all-tests.yml
+    parameters:
+      - name: GIT_REF
+        required: false
+        description: "librdkafka git ref to test."
+
+      - name: TEST_KAFKA_GIT_REF
+        required: true
+        default_value: ""
+        description: "Kafka version to test against. When a valid version, the corresponding 
+          package is downloaded. [<username>/]<git_ref>[@<commit_hash>] is also supported 
+          to clone from GitHub and build from source."
+
+      - name: TEST_CP_VERSION
+        required: false
+        description: "CP version to test against"
+
+      - name: TEST_KAFKA_VERSION
+        required: false
+        description: "Kafka version to assume in this branch. If empty (default) 
+          uses TEST_KAFKA_GIT_REF when a valid version, otherwise 9.9.9."
+
+      - name: TEST_KRAFT
+        required: true
+        default_value: "True"
+        description: "Run a KRaft cluster."
+        options:
+          - "True"
+          - "False"
+
+      - name: TEST_TYPE
+        required: true
+        default_value: "plaintext,ssl"
+        description: "Run PLAINTEXT or SSL tests or both. Defaults to both."
+        options:
+          - "plaintext,ssl"
+          - "plaintext"
+          - "ssl"
+
+      - name: TEST_SSL_INTERMEDIATE_CA
+        required: false
+        description: "Run SSL tests with an intermediate CA certificate."
+        options:
+          - "True"
+          - "False"
+
+      - name: TEST_SASL
+        required: false
+        description: "SASL mechanism to use."
+        options:
+          - ""
+          - "PLAIN"
+          - "SCRAM-SHA-256"
+          - "SCRAM-SHA-512"
+          - "GSSAPI"
+          - "OAUTHBEARER"
+
+      - name: TEST_QUICK
+        required: true
+        default_value: "False"
+        description: "Run only quick tests."
+        options:
+          - "True"
+          - "False"
+
+      - name: TEST_LOCAL
+        required: true
+        default_value: "All"
+        description: "Run only local, only non-local or all tests."
+        options:
+          - "All"
+          - "Local"
+          - "Non-local"
+
+      - name: TEST_PARALLEL
+        required: false
+        description: "Number of parallel tests to run. Empty means default value."
+
+      - name: TEST_MODE
+        required: false
+        description: "Test mode to use. Empty means default value."
+
+      - name: TESTS
+        required: false
+        description: "Run specified tests only. Empty means run all tests. 
+          Can be a comma-separated list of test numbers or ranges, 
+          even overlapping. Example: 
+          1,3,9-11,10-15,17-,-10"
+
+      - name: TESTS_BATCH_SIZE
+        required: false
+        description: "Run tests in batches of this size. 
+          Empty means no batches, run all tests at once."
+
+      - name: TEST_ITERATIONS
+        required: false
+        description: "Run specified tests this many times. 
+          Empty means run tests once."
+
+      - name: TEST_ASSERT
+        required: true
+        default_value: "False"
+        description: "Assert on failures"
+        options:
+          - "True"
+          - "False"
+
+      - name: TEST_DEBUG
+        required: false
+        description: "Enable debug output for tests."
+
+      - name: TEST_LEVEL
+        required: false
+        description: "Test verbosity level. Default is 2. Increase to 3 or 4 for more output."
+
+      - name: TEST_CONF
+        required: false
+        description: "Additional broker configuration as a JSON array. Example: 
+          [group.coordinator.rebalance.protocols=classic,consumer]"
+
+      - name: TEST_TRIVUP_PARAMETERS
+        required: false
+        description: "Additional trivup parameters."
+
+      - name: TEST_RUNNER_PARAMETERS
+        required: false
+        description: "Additional test runner parameters."
+
+      - name: TEST_ENV_VARIABLES
+        required: false
+        description: "Additional environment variables to set for tests. 
+          Example: A=1,B=2,C=3"
+
+      - name: TEST_ARCHES
+        required: true
+        default_value: "x86_64,aarch64"
+        description: "Test on these architectures: x86_64, aarch64 or both"
+        options:
+          - "x86_64,aarch64"
+          - "x86_64"
+          - "aarch64"
+      

package/deps/librdkafka/src/rd.h

@@ -53,6 +53,7 @@
 #include <time.h>
 #include <assert.h>
 #include <limits.h>
+#include <sys/stat.h>
 
 #include "tinycthread.h"
 #include "rdsysqueue.h"
@@ -220,12 +221,18 @@ static RD_INLINE RD_UNUSED char *rd_strndup(const char *s, size_t len) {
 
 
 /* Round/align X upwards to STRIDE, which must be power of 2. */
-#define RD_ROUNDUP(X, STRIDE) (((X) + ((STRIDE)-1)) & ~(STRIDE - 1))
+#define RD_ROUNDUP(X, STRIDE) (((X) + ((STRIDE) - 1)) & ~(STRIDE - 1))
 
 #define RD_ARRAY_SIZE(A)          (sizeof((A)) / sizeof(*(A)))
 #define RD_ARRAYSIZE(A)           RD_ARRAY_SIZE(A)
 #define RD_SIZEOF(TYPE, MEMBER)   sizeof(((TYPE *)NULL)->MEMBER)
 #define RD_OFFSETOF(TYPE, MEMBER) ((size_t) & (((TYPE *)NULL)->MEMBER))
+/** Array foreach */
+#define RD_ARRAY_FOREACH_INDEX(ELEM, ARRAY, INDEX)                             \
+        for ((INDEX = 0, (ELEM) = (ARRAY)[INDEX]);                             \
+             INDEX < RD_ARRAY_SIZE(ARRAY);                                     \
+             (ELEM) =                                                          \
+                 (++INDEX < RD_ARRAY_SIZE(ARRAY) ? (ARRAY)[INDEX] : (ELEM)))
 
 /**
  * Returns the 'I'th array element from static sized array 'A'
@@ -427,7 +434,7 @@ static RD_INLINE RD_UNUSED int rd_refcnt_get(rd_refcnt_t *R) {
 
 #define RD_INTERFACE_CALL(i, name, ...) (i->name(i->opaque, __VA_ARGS__))
 
-#define RD_CEIL_INTEGER_DIVISION(X, DEN) (((X) + ((DEN)-1)) / (DEN))
+#define RD_CEIL_INTEGER_DIVISION(X, DEN) (((X) + ((DEN) - 1)) / (DEN))
 
 /**
  * @brief Utility types to hold memory,size tuple.
@@ -438,4 +445,140 @@ typedef struct rd_chariov_s {
         size_t size;
 } rd_chariov_t;
 
+/**
+ * @brief Read the file at \p file_path in binary mode and return its contents.
+ *        The returned buffer is NULL-terminated,
+ *        the size parameter will contain the actual file size.
+ *
+ * @param file_path Path to the file to read.
+ * @param size Pointer to store the file size (optional).
+ * @param max_size Maximum file size to read (0 for no limit) (optional).
+ *
+ * @returns Newly allocated buffer containing the file contents.
+ *          NULL on error (file not found, too large, etc).
+ *
+ * @remark The returned pointer ownership is transferred to the caller.
+ *
+ * @locality Any thread
+ */
+static RD_INLINE RD_UNUSED char *
+rd_file_read(const char *file_path, size_t *size, size_t max_size) {
+        FILE *file;
+        char *buf = NULL;
+        size_t file_size;
+        size_t read_size;
+        if (!size)
+                size = &read_size;
+
+#ifndef _WIN32
+        file = fopen(file_path, "rb");
+#else
+        file  = NULL;
+        errno = fopen_s(&file, file_path, "rb");
+#endif
+        if (!file)
+                return NULL;
+
+        if (fseek(file, 0, SEEK_END) != 0)
+                goto err;
+
+        file_size = (size_t)ftell(file);
+        if (file_size < 0)
+                goto err;
+
+        if (fseek(file, 0, SEEK_SET) != 0)
+                goto err;
+
+        /* Check if file is too large */
+        if (max_size > 0 && file_size > max_size)
+                goto err;
+
+        /* Allocate buffer with extra byte for NULL terminator */
+        buf       = (char *)rd_malloc(file_size + 1);
+        read_size = fread(buf, 1, file_size, file);
+
+        if (read_size != file_size)
+                goto err;
+
+        /* NULL terminate the buffer */
+        buf[file_size] = '\0';
+        *size          = file_size;
+        fclose(file);
+        return buf;
+err:
+        fclose(file);
+        if (buf)
+                rd_free(buf);
+        return NULL;
+}
+
+static RD_INLINE RD_UNUSED FILE *
+rd_file_mkstemp(const char *prefix,
+                const char *mode,
+                char *tempfile_path_out,
+                size_t tempfile_path_out_size) {
+        FILE *tempfile;
+
+#ifdef _WIN32
+        char tempfolder_path[MAX_PATH];
+        char tempfile_path[MAX_PATH];
+        if (!GetTempPathA(MAX_PATH, tempfolder_path))
+                return NULL; /* Failed to get temp folder path */
+
+
+        if (!GetTempFileNameA(tempfolder_path, "TMP", 1, tempfile_path))
+                return NULL; /* Failed to create temp file name */
+
+        tempfile = fopen(tempfile_path, mode);
+#else
+        int tempfile_fd;
+        char tempfile_path[512];
+        rd_snprintf(tempfile_path, sizeof(tempfile_path), "/tmp/%sXXXXXX",
+                    prefix);
+        tempfile_fd = mkstemp(tempfile_path);
+        if (tempfile_fd < 0)
+                return NULL;
+
+        tempfile = fdopen(tempfile_fd, mode);
+#endif
+
+        if (!tempfile)
+                return NULL;
+
+        if (tempfile_path_out)
+                rd_snprintf(tempfile_path_out, tempfile_path_out_size, "%s",
+                            tempfile_path);
+        return tempfile;
+}
+
+/**
+ * @brief Retrive stat for a \p path .
+ *
+ * @param path Path to the file or directory.
+ * @param is_dir Pointer to store if the \p path is a directory (optional).
+ *
+ * @return `rd_true` if the path exists.
+ */
+static RD_INLINE RD_UNUSED rd_bool_t rd_file_stat(const char *path,
+                                                  rd_bool_t *is_dir) {
+#ifdef _WIN32
+        struct _stat st;
+        if (_stat(path, &st) == 0) {
+                if (is_dir)
+                        *is_dir = st.st_mode & S_IFDIR;
+                return rd_true;
+        }
+#else
+        struct stat st;
+        if (stat(path, &st) == 0) {
+                if (is_dir)
+                        *is_dir = S_ISDIR(st.st_mode);
+                return rd_true;
+        }
+#endif
+        if (is_dir)
+                *is_dir = rd_false;
+        return rd_false;
+}
+
 #endif /* _RD_H_ */

package/deps/librdkafka/src/rdaddr.h

@@ -54,14 +54,13 @@ typedef union {
 #define sinx_family in.sin_family
 #define sinx_addr   in.sin_addr
 #define RD_SOCKADDR_INX_LEN(sinx)                                              \
-        ((sinx)->sinx_family == AF_INET                                        \
-             ? sizeof(struct sockaddr_in)                                      \
-             : (sinx)->sinx_family == AF_INET6 ? sizeof(struct sockaddr_in6)   \
-                                               : sizeof(rd_sockaddr_inx_t))
+        ((sinx)->sinx_family == AF_INET    ? sizeof(struct sockaddr_in)        \
+         : (sinx)->sinx_family == AF_INET6 ? sizeof(struct sockaddr_in6)       \
+                                           : sizeof(rd_sockaddr_inx_t))
 #define RD_SOCKADDR_INX_PORT(sinx)                                             \
-        ((sinx)->sinx_family == AF_INET                                        \
-             ? (sinx)->in.sin_port                                             \
-             : (sinx)->sinx_family == AF_INET6 ? (sinx)->in6.sin6_port : 0)
+        ((sinx)->sinx_family == AF_INET    ? (sinx)->in.sin_port               \
+         : (sinx)->sinx_family == AF_INET6 ? (sinx)->in6.sin6_port             \
+                                           : 0)
 
 #define RD_SOCKADDR_INX_PORT_SET(sinx, port)                                   \
         do {                                                                   \