@platformatic/rdkafka 4.0.0

package/deps/librdkafka/src/rdkafka_feature.c

@@ -0,0 +1,461 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2016-2022, Magnus Edenhill
+ *               2023, Confluent Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include "rdkafka_int.h"
+#include "rdkafka_feature.h"
+
+#include <stdlib.h>
+
+static const char *rd_kafka_feature_names[] = {"MsgVer1",
+                                               "ApiVersion",
+                                               "BrokerBalancedConsumer",
+                                               "ThrottleTime",
+                                               "Sasl",
+                                               "SaslHandshake",
+                                               "BrokerGroupCoordinator",
+                                               "LZ4",
+                                               "OffsetTime",
+                                               "MsgVer2",
+                                               "IdempotentProducer",
+                                               "ZSTD",
+                                               "SaslAuthReq",
+                                               "UnitTest",
+                                               NULL};
+
+
+static const struct rd_kafka_feature_map {
+        /* RD_KAFKA_FEATURE_... */
+        int feature;
+
+        /* Depends on the following ApiVersions overlapping with
+         * what the broker supports: */
+        struct rd_kafka_ApiVersion depends[RD_KAFKAP__NUM];
+
+} rd_kafka_feature_map[] = {
+    /**
+     * @brief List of features and the ApiVersions they depend on.
+     *
+     * The dependency list consists of the ApiKey followed by this
+     * client's supported minimum and maximum API versions.
+     * As long as this list and its versions overlaps with the
+     * broker supported API versions the feature will be enabled.
+     */
+    {
+
+        /* @brief >=0.10.0: Message.MagicByte version 1:
+         * Relative offsets (KIP-31) and message timestamps (KIP-32). */
+        .feature = RD_KAFKA_FEATURE_MSGVER1,
+        .depends =
+            {
+                {RD_KAFKAP_Produce, 2, 2},
+                {RD_KAFKAP_Fetch, 2, 2},
+                {-1},
+            },
+    },
+    {
+        /* @brief >=0.11.0: Message.MagicByte version 2 */
+        .feature = RD_KAFKA_FEATURE_MSGVER2,
+        .depends =
+            {
+                {RD_KAFKAP_Produce, 3, 3},
+                {RD_KAFKAP_Fetch, 4, 4},
+                {-1},
+            },
+    },
+    {
+        /* @brief >=0.10.0: ApiVersionQuery support.
+         * @remark This is a bit of chicken-and-egg problem but needs to be
+         *         set by feature_check() to avoid the feature being cleared
+         *         even when broker supports it. */
+        .feature = RD_KAFKA_FEATURE_APIVERSION,
+        .depends =
+            {
+                {RD_KAFKAP_ApiVersion, 0, 0},
+                {-1},
+            },
+    },
+    {
+        /* @brief >=0.8.2.0: Broker-based Group coordinator */
+        .feature = RD_KAFKA_FEATURE_BROKER_GROUP_COORD,
+        .depends =
+            {
+                {RD_KAFKAP_FindCoordinator, 0, 0},
+                {-1},
+            },
+    },
+    {
+        /* @brief >=0.9.0: Broker-based balanced consumer groups. */
+        .feature = RD_KAFKA_FEATURE_BROKER_BALANCED_CONSUMER,
+        .depends =
+            {
+                {RD_KAFKAP_FindCoordinator, 0, 0},
+                {RD_KAFKAP_OffsetCommit, 1, 2},
+                {RD_KAFKAP_OffsetFetch, 1, 1},
+                {RD_KAFKAP_JoinGroup, 0, 0},
+                {RD_KAFKAP_SyncGroup, 0, 0},
+                {RD_KAFKAP_Heartbeat, 0, 0},
+                {RD_KAFKAP_LeaveGroup, 0, 0},
+                {-1},
+            },
+    },
+    {
+        /* @brief >=0.9.0: ThrottleTime */
+        .feature = RD_KAFKA_FEATURE_THROTTLETIME,
+        .depends =
+            {
+                {RD_KAFKAP_Produce, 1, 2},
+                {RD_KAFKAP_Fetch, 1, 2},
+                {-1},
+            },
+
+    },
+    {
+        /* @brief >=0.9.0: SASL (GSSAPI) authentication.
+         * Since SASL is not using the Kafka protocol
+         * we must use something else to map us to the
+         * proper broker version support:
+         * JoinGroup was released along with SASL in 0.9.0. */
+        .feature = RD_KAFKA_FEATURE_SASL_GSSAPI,
+        .depends =
+            {
+                {RD_KAFKAP_JoinGroup, 0, 0},
+                {-1},
+            },
+    },
+    {
+        /* @brief >=0.10.0: SASL mechanism handshake (KIP-43)
+         *                  to automatically support other mechanisms
+         *                  than GSSAPI, such as PLAIN. */
+        .feature = RD_KAFKA_FEATURE_SASL_HANDSHAKE,
+        .depends =
+            {
+                {RD_KAFKAP_SaslHandshake, 0, 0},
+                {-1},
+            },
+    },
+    {
+        /* @brief >=0.8.2: LZ4 compression.
+         * Since LZ4 initially did not rely on a specific API
+         * type or version (it does in >=0.10.0)
+         * we must use something else to map us to the
+         * proper broker version support:
+         * GrooupCoordinator was released in 0.8.2 */
+        .feature = RD_KAFKA_FEATURE_LZ4,
+        .depends =
+            {
+                {RD_KAFKAP_FindCoordinator, 0, 0},
+                {-1},
+            },
+    },
+    {/* @brief >=0.10.1.0: Offset v1 (KIP-79)
+      * Time-based offset requests */
+     .feature = RD_KAFKA_FEATURE_OFFSET_TIME,
+     .depends =
+         {
+             {RD_KAFKAP_ListOffsets, 1, 1},
+             {-1},
+         }},
+    {/* @brief >=0.11.0.0: Idempotent Producer*/
+     .feature = RD_KAFKA_FEATURE_IDEMPOTENT_PRODUCER,
+     .depends =
+         {
+             {RD_KAFKAP_InitProducerId, 0, 0},
+             {-1},
+         }},
+    {
+        /* @brief >=2.1.0-IV2: Support ZStandard Compression Codec (KIP-110) */
+        .feature = RD_KAFKA_FEATURE_ZSTD,
+        .depends =
+            {
+                {RD_KAFKAP_Produce, 7, 7},
+                {RD_KAFKAP_Fetch, 10, 10},
+                {-1},
+            },
+    },
+    {
+        /* @brief >=1.0.0: SaslAuthenticateRequest */
+        .feature = RD_KAFKA_FEATURE_SASL_AUTH_REQ,
+        .depends =
+            {
+                {RD_KAFKAP_SaslHandshake, 1, 1},
+                {RD_KAFKAP_SaslAuthenticate, 0, 1},
+                {-1},
+            },
+    },
+    {.feature = 0}, /* sentinel */
+};
+
+
+
+/**
+ * @brief In absence of KIP-35 support in earlier broker versions we provide
+ * hardcoded lists that corresponds to older broker versions.
+ */
+
+/* >= 0.10.0.0: dummy for all future versions that support ApiVersionRequest */
+static struct rd_kafka_ApiVersion rd_kafka_ApiVersion_Queryable[] = {
+    {RD_KAFKAP_ApiVersion, 0, 0}};
+
+
+/* =~ 0.9.0 */
+static struct rd_kafka_ApiVersion rd_kafka_ApiVersion_0_9_0[] = {
+    {RD_KAFKAP_Produce, 0, 1},         {RD_KAFKAP_Fetch, 0, 1},
+    {RD_KAFKAP_ListOffsets, 0, 0},     {RD_KAFKAP_Metadata, 0, 0},
+    {RD_KAFKAP_OffsetCommit, 0, 2},    {RD_KAFKAP_OffsetFetch, 0, 1},
+    {RD_KAFKAP_FindCoordinator, 0, 0}, {RD_KAFKAP_JoinGroup, 0, 0},
+    {RD_KAFKAP_Heartbeat, 0, 0},       {RD_KAFKAP_LeaveGroup, 0, 0},
+    {RD_KAFKAP_SyncGroup, 0, 0},       {RD_KAFKAP_DescribeGroups, 0, 0},
+    {RD_KAFKAP_ListGroups, 0, 0}};
+
+/* =~ 0.8.2 */
+static struct rd_kafka_ApiVersion rd_kafka_ApiVersion_0_8_2[] = {
+    {RD_KAFKAP_Produce, 0, 0},        {RD_KAFKAP_Fetch, 0, 0},
+    {RD_KAFKAP_ListOffsets, 0, 0},    {RD_KAFKAP_Metadata, 0, 0},
+    {RD_KAFKAP_OffsetCommit, 0, 1},   {RD_KAFKAP_OffsetFetch, 0, 1},
+    {RD_KAFKAP_FindCoordinator, 0, 0}};
+
+/* =~ 0.8.1 */
+static struct rd_kafka_ApiVersion rd_kafka_ApiVersion_0_8_1[] = {
+    {RD_KAFKAP_Produce, 0, 0},      {RD_KAFKAP_Fetch, 0, 0},
+    {RD_KAFKAP_ListOffsets, 0, 0},  {RD_KAFKAP_Metadata, 0, 0},
+    {RD_KAFKAP_OffsetCommit, 0, 1}, {RD_KAFKAP_OffsetFetch, 0, 0}};
+
+/* =~ 0.8.0 */
+static struct rd_kafka_ApiVersion rd_kafka_ApiVersion_0_8_0[] = {
+    {RD_KAFKAP_Produce, 0, 0},
+    {RD_KAFKAP_Fetch, 0, 0},
+    {RD_KAFKAP_ListOffsets, 0, 0},
+    {RD_KAFKAP_Metadata, 0, 0}};
+
+
+/**
+ * @brief Returns the ApiVersion list for legacy broker versions that do not
+ *        support the ApiVersionQuery request. E.g., brokers <0.10.0.
+ *
+ * @param broker_version Broker version to match (longest prefix matching).
+ * @param use_default If no match is found return the default APIs (but return
+ * 0).
+ *
+ * @returns 1 if \p broker_version was recognized: \p *apisp will point to
+ *          the ApiVersion list and *api_cntp will be set to its element count.
+ *          0 if \p broker_version was not recognized: \p *apisp remains
+ * unchanged.
+ *
+ */
+int rd_kafka_get_legacy_ApiVersions(const char *broker_version,
+                                    struct rd_kafka_ApiVersion **apisp,
+                                    size_t *api_cntp,
+                                    const char *fallback) {
+        static const struct {
+                const char *pfx;
+                struct rd_kafka_ApiVersion *apis;
+                size_t api_cnt;
+        } vermap[] = {
+#define _VERMAP(PFX, APIS) {PFX, APIS, RD_ARRAYSIZE(APIS)}
+            _VERMAP("0.9.0", rd_kafka_ApiVersion_0_9_0),
+            _VERMAP("0.8.2", rd_kafka_ApiVersion_0_8_2),
+            _VERMAP("0.8.1", rd_kafka_ApiVersion_0_8_1),
+            _VERMAP("0.8.0", rd_kafka_ApiVersion_0_8_0),
+            {"0.7.", NULL}, /* Unsupported */
+            {"0.6.", NULL}, /* Unsupported */
+            _VERMAP("", rd_kafka_ApiVersion_Queryable),
+            {NULL}};
+        int i;
+        int fallback_i = -1;
+        int ret        = 0;
+
+        *apisp    = NULL;
+        *api_cntp = 0;
+
+        for (i = 0; vermap[i].pfx; i++) {
+                if (!strncmp(vermap[i].pfx, broker_version,
+                             strlen(vermap[i].pfx))) {
+                        if (!vermap[i].apis)
+                                return 0;
+                        *apisp    = vermap[i].apis;
+                        *api_cntp = vermap[i].api_cnt;
+                        ret       = 1;
+                        break;
+                } else if (fallback && !strcmp(vermap[i].pfx, fallback))
+                        fallback_i = i;
+        }
+
+        if (!*apisp && fallback) {
+                rd_kafka_assert(NULL, fallback_i != -1);
+                *apisp    = vermap[fallback_i].apis;
+                *api_cntp = vermap[fallback_i].api_cnt;
+        }
+
+        return ret;
+}
+
+
+/**
+ * @returns 1 if the provided broker version (probably)
+ *          supports api.version.request.
+ */
+int rd_kafka_ApiVersion_is_queryable(const char *broker_version) {
+        struct rd_kafka_ApiVersion *apis;
+        size_t api_cnt;
+
+
+        if (!rd_kafka_get_legacy_ApiVersions(broker_version, &apis, &api_cnt,
+                                             0))
+                return 0;
+
+        return apis == rd_kafka_ApiVersion_Queryable;
+}
+
+
+
+/**
+ * @brief Check if match's versions overlaps with \p apis.
+ *
+ * @returns 1 if true, else 0.
+ * @remark \p apis must be sorted using rd_kafka_ApiVersion_key_cmp()
+ */
+static RD_INLINE int
+rd_kafka_ApiVersion_check(const struct rd_kafka_ApiVersion *apis,
+                          size_t api_cnt,
+                          const struct rd_kafka_ApiVersion *match) {
+        const struct rd_kafka_ApiVersion *api;
+
+        api = bsearch(match, apis, api_cnt, sizeof(*apis),
+                      rd_kafka_ApiVersion_key_cmp);
+        if (unlikely(!api))
+                return 0;
+
+        return match->MinVer <= api->MaxVer && api->MinVer <= match->MaxVer;
+}
+
+
+/**
+ * @brief Compare broker's supported API versions to our feature request map
+ *        and enable/disable features accordingly.
+ *
+ * @param broker_apis Broker's supported APIs. If NULL the
+ *        \p broker.version.fallback configuration property will specify a
+ *        default legacy version to use.
+ * @param broker_api_cnt Number of elements in \p broker_apis
+ *
+ * @returns the supported features (bitmask) to enable.
+ */
+int rd_kafka_features_check(rd_kafka_broker_t *rkb,
+                            struct rd_kafka_ApiVersion *broker_apis,
+                            size_t broker_api_cnt) {
+        int features = 0;
+        int i;
+
+        /* Scan through features. */
+        for (i = 0; rd_kafka_feature_map[i].feature != 0; i++) {
+                const struct rd_kafka_ApiVersion *match;
+                int fails = 0;
+
+                /* For each feature check that all its API dependencies
+                 * can be fullfilled. */
+
+                for (match = &rd_kafka_feature_map[i].depends[0];
+                     match->ApiKey != -1; match++) {
+                        int r;
+
+                        r = rd_kafka_ApiVersion_check(broker_apis,
+                                                      broker_api_cnt, match);
+
+                        rd_rkb_dbg(rkb, FEATURE, "APIVERSION",
+                                   " Feature %s: %s (%hd..%hd) "
+                                   "%ssupported by broker",
+                                   rd_kafka_features2str(
+                                       rd_kafka_feature_map[i].feature),
+                                   rd_kafka_ApiKey2str(match->ApiKey),
+                                   match->MinVer, match->MaxVer,
+                                   r ? "" : "NOT ");
+
+                        fails += !r;
+                }
+
+                rd_rkb_dbg(
+                    rkb, FEATURE, "APIVERSION", "%s feature %s",
+                    fails ? "Disabling" : "Enabling",
+                    rd_kafka_features2str(rd_kafka_feature_map[i].feature));
+
+
+                if (!fails)
+                        features |= rd_kafka_feature_map[i].feature;
+        }
+
+        return features;
+}
+
+
+
+/**
+ * @brief Make an allocated and sorted copy of \p src.
+ */
+void rd_kafka_ApiVersions_copy(const struct rd_kafka_ApiVersion *src,
+                               size_t src_cnt,
+                               struct rd_kafka_ApiVersion **dstp,
+                               size_t *dst_cntp) {
+        *dstp     = rd_memdup(src, sizeof(*src) * src_cnt);
+        *dst_cntp = src_cnt;
+        qsort(*dstp, *dst_cntp, sizeof(**dstp), rd_kafka_ApiVersion_key_cmp);
+}
+
+
+
+/**
+ * @returns a human-readable feature flag string.
+ */
+const char *rd_kafka_features2str(int features) {
+        static RD_TLS char ret[4][256];
+        size_t of              = 0;
+        static RD_TLS int reti = 0;
+        int i;
+
+        reti = (reti + 1) % 4;
+
+        *ret[reti] = '\0';
+        for (i = 0; rd_kafka_feature_names[i]; i++) {
+                int r;
+                if (!(features & (1 << i)))
+                        continue;
+
+                r = rd_snprintf(ret[reti] + of, sizeof(ret[reti]) - of, "%s%s",
+                                of == 0 ? "" : ",", rd_kafka_feature_names[i]);
+                if ((size_t)r > sizeof(ret[reti]) - of) {
+                        /* Out of space */
+                        memcpy(&ret[reti][sizeof(ret[reti]) - 3], "..", 3);
+                        break;
+                }
+
+                of += r;
+        }
+
+        return ret[reti];
+}

package/deps/librdkafka/src/rdkafka_feature.h

@@ -0,0 +1,102 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2016-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef _RDKAFKA_FEATURE_H_
+#define _RDKAFKA_FEATURE_H_
+
+
+/**
+ * @brief Kafka protocol features
+ */
+
+/* Message version 1 (MagicByte=1):
+ *  + relative offsets (KIP-31)
+ *  + timestamps (KIP-32) */
+#define RD_KAFKA_FEATURE_MSGVER1 0x1
+
+/* ApiVersionQuery support (KIP-35) */
+#define RD_KAFKA_FEATURE_APIVERSION 0x2
+
+/* >= 0.9: Broker-based Balanced Consumer */
+#define RD_KAFKA_FEATURE_BROKER_BALANCED_CONSUMER 0x4
+
+/* >= 0.9: Produce/Fetch ThrottleTime reporting */
+#define RD_KAFKA_FEATURE_THROTTLETIME 0x8
+
+/* >= 0.9: SASL GSSAPI support */
+#define RD_KAFKA_FEATURE_SASL_GSSAPI 0x10
+
+/* >= 0.10: SaslMechanismRequest (KIP-43) */
+#define RD_KAFKA_FEATURE_SASL_HANDSHAKE 0x20
+
+/* >= 0.8.2.0: Broker-based Group coordinator */
+#define RD_KAFKA_FEATURE_BROKER_GROUP_COORD 0x40
+
+/* >= 0.8.2.0: LZ4 compression (with bad and proper HC checksums) */
+#define RD_KAFKA_FEATURE_LZ4 0x80
+
+/* >= 0.10.1.0: Time-based Offset fetch (KIP-79) */
+#define RD_KAFKA_FEATURE_OFFSET_TIME 0x100
+
+/* >= 0.11.0.0: Message version 2 (MagicByte=2):
+ *  + EOS message format KIP-98 */
+#define RD_KAFKA_FEATURE_MSGVER2 0x200
+
+/* >= 0.11.0.0: Idempotent Producer support */
+#define RD_KAFKA_FEATURE_IDEMPOTENT_PRODUCER 0x400
+
+/* >= 2.1.0-IV2: ZSTD compression */
+#define RD_KAFKA_FEATURE_ZSTD 0x800
+
+/* >= 1.0.0: SaslAuthenticateRequest */
+#define RD_KAFKA_FEATURE_SASL_AUTH_REQ 0x1000
+
+/* Unit-test mock broker: broker supports everything.
+ * Should be used with RD_KAFKA_FEATURE_ALL, but not be included in bitmask */
+#define RD_KAFKA_FEATURE_UNITTEST 0x4000
+
+/* All features (except UNITTEST) */
+#define RD_KAFKA_FEATURE_ALL (0xffff & ~RD_KAFKA_FEATURE_UNITTEST)
+
+
+
+int rd_kafka_get_legacy_ApiVersions(const char *broker_version,
+                                    struct rd_kafka_ApiVersion **apisp,
+                                    size_t *api_cntp,
+                                    const char *fallback);
+int rd_kafka_ApiVersion_is_queryable(const char *broker_version);
+void rd_kafka_ApiVersions_copy(const struct rd_kafka_ApiVersion *src,
+                               size_t src_cnt,
+                               struct rd_kafka_ApiVersion **dstp,
+                               size_t *dst_cntp);
+int rd_kafka_features_check(rd_kafka_broker_t *rkb,
+                            struct rd_kafka_ApiVersion *broker_apis,
+                            size_t broker_api_cnt);
+
+const char *rd_kafka_features2str(int features);
+
+#endif /* _RDKAFKA_FEATURE_H_ */