@platformatic/rdkafka 4.0.0

package/deps/librdkafka/tests/0082-fetch_max_bytes.cpp

@@ -0,0 +1,133 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2016-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <iostream>
+#include <cstring>
+#include <cstdlib>
+#include "testcpp.h"
+
+/**
+ * @brief Test fetch.max.bytes
+ *
+ *  - Produce 1*10 Megs to 3 partitions (~<1 Meg per message)
+ *  - Set max.partition.fetch.bytes to 5 Meg
+ *  - Set fetch.max.bytes to 2 Meg
+ *  - Verify all messages are consumed without error.
+ */
+
+
+static void do_test_fetch_max_bytes(void) {
+  const int partcnt = 3;
+  int msgcnt        = 10 * partcnt;
+  const int msgsize = 900 * 1024; /* Less than 1 Meg to account
+                                   * for batch overhead */
+  std::string errstr;
+  RdKafka::ErrorCode err;
+
+  std::string topic = Test::mk_topic_name("0081-fetch_max_bytes", 1);
+
+  /* Produce messages to partitions */
+  for (int32_t p = 0; p < (int32_t)partcnt; p++)
+    test_produce_msgs_easy_size(topic.c_str(), 0, p, msgcnt, msgsize);
+
+  /* Create consumer */
+  RdKafka::Conf *conf;
+  Test::conf_init(&conf, NULL, 10);
+  Test::conf_set(conf, "group.id", topic);
+  Test::conf_set(conf, "auto.offset.reset", "earliest");
+  /* We try to fetch 20 Megs per partition, but only allow 1 Meg as total
+   * response size, this ends up serving the first batch from the
+   * first partition.
+   * receive.message.max.bytes is set low to trigger the original bug,
+   * but this value is now adjusted upwards automatically by rd_kafka_new()
+   * to hold both fetch.max.bytes and the protocol / batching overhead.
+   * Prior to the introduction of fetch.max.bytes the fetcher code
+   * would use receive.message.max.bytes to limit the total Fetch response,
+   * but due to batching overhead it would result in situations where
+   * the consumer asked for 1000000 bytes and got 1000096 bytes batch, which
+   * was higher than the 1000000 limit.
+   * See https://github.com/confluentinc/librdkafka/issues/1616
+   *
+   * With the added configuration strictness checks, a user-supplied
+   * value is no longer over-written:
+   * receive.message.max.bytes must be configured to be at least 512 bytes
+   * larger than fetch.max.bytes.
+   */
+  Test::conf_set(conf, "max.partition.fetch.bytes", "20000000"); /* ~20MB */
+  Test::conf_set(conf, "fetch.max.bytes", "1000000");            /* ~1MB */
+  Test::conf_set(conf, "receive.message.max.bytes", "1000512");  /* ~1MB+512 */
+
+  RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr);
+  if (!c)
+    Test::Fail("Failed to create KafkaConsumer: " + errstr);
+  delete conf;
+
+  /* Subscribe */
+  std::vector<std::string> topics;
+  topics.push_back(topic);
+  if ((err = c->subscribe(topics)))
+    Test::Fail("subscribe failed: " + RdKafka::err2str(err));
+
+  /* Start consuming */
+  Test::Say("Consuming topic " + topic + "\n");
+  int cnt = 0;
+  while (cnt < msgcnt) {
+    RdKafka::Message *msg = c->consume(tmout_multip(1000));
+    switch (msg->err()) {
+    case RdKafka::ERR__TIMED_OUT:
+      break;
+
+    case RdKafka::ERR_NO_ERROR:
+      cnt++;
+      break;
+
+    default:
+      Test::Fail("Consume error: " + msg->errstr());
+      break;
+    }
+
+    delete msg;
+  }
+  Test::Say("Done\n");
+
+  c->close();
+  delete c;
+}
+
+extern "C" {
+int main_0082_fetch_max_bytes(int argc, char **argv) {
+  if (test_quick) {
+    Test::Skip("Test skipped due to quick mode\n");
+    return 0;
+  }
+
+  do_test_fetch_max_bytes();
+
+  return 0;
+}
+}

package/deps/librdkafka/tests/0083-cb_event.c

@@ -0,0 +1,228 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2018-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * Tests the queue callback IO event signalling.
+ */
+
+
+#include "test.h"
+
+/* Typical include path would be <librdkafka/rdkafka.h>, but this program
+ * is built from within the librdkafka source tree and thus differs. */
+#include "rdkafka.h" /* for Kafka driver */
+
+
+/**
+ * @brief Thread safe event counter */
+static struct {
+        mtx_t lock;
+        int count;
+} event_receiver;
+
+/**
+ * @brief Event callback function. Check the opaque pointer and
+ *        increase the count of received event. */
+static void event_cb(rd_kafka_t *rk_p, void *opaque) {
+        TEST_ASSERT(opaque == (void *)0x1234,
+                    "Opaque pointer is not as expected (got: %p)", opaque);
+        mtx_lock(&event_receiver.lock);
+        event_receiver.count += 1;
+        mtx_unlock(&event_receiver.lock);
+}
+
+/**
+ * @brief Wait for one or more events to be received.
+ *        Return 0 if no event was received within the timeout. */
+static int wait_event_cb(int timeout_secs) {
+        int event_count = 0;
+        for (; timeout_secs >= 0; timeout_secs--) {
+                mtx_lock(&event_receiver.lock);
+                event_count          = event_receiver.count;
+                event_receiver.count = 0;
+                mtx_unlock(&event_receiver.lock);
+                if (event_count > 0 || timeout_secs == 0)
+                        return event_count;
+                rd_sleep(1);
+        }
+        return 0;
+}
+
+
+int main_0083_cb_event(int argc, char **argv) {
+        rd_kafka_conf_t *conf;
+        rd_kafka_topic_conf_t *tconf;
+        rd_kafka_t *rk_p, *rk_c;
+        const char *topic;
+        rd_kafka_topic_t *rkt_p;
+        rd_kafka_queue_t *queue;
+        uint64_t testid;
+        int msgcnt          = 100;
+        int recvd           = 0;
+        int wait_multiplier = 1;
+        rd_kafka_resp_err_t err;
+        enum { _NOPE, _YEP, _REBALANCE } expecting_io = _REBALANCE;
+        int callback_event_count;
+        rd_kafka_event_t *rkev;
+        int eventcnt = 0;
+
+        mtx_init(&event_receiver.lock, mtx_plain);
+
+        testid = test_id_generate();
+        topic  = test_mk_topic_name(__FUNCTION__, 1);
+
+        rk_p  = test_create_producer();
+        rkt_p = test_create_producer_topic(rk_p, topic, NULL);
+        err   = test_auto_create_topic_rkt(rk_p, rkt_p, tmout_multip(5000));
+        TEST_ASSERT(!err, "Topic auto creation failed: %s",
+                    rd_kafka_err2str(err));
+
+        test_conf_init(&conf, &tconf, 0);
+        rd_kafka_conf_set_events(conf, RD_KAFKA_EVENT_REBALANCE);
+        test_conf_set(conf, "session.timeout.ms", "6000");
+        test_conf_set(conf, "enable.partition.eof", "false");
+        /* Speed up propagation of new topics */
+        test_conf_set(conf, "metadata.max.age.ms", "5000");
+        test_topic_conf_set(tconf, "auto.offset.reset", "earliest");
+        rk_c = test_create_consumer(topic, NULL, conf, tconf);
+
+        queue = rd_kafka_queue_get_consumer(rk_c);
+
+        test_consumer_subscribe(rk_c, topic);
+
+        rd_kafka_queue_cb_event_enable(queue, event_cb, (void *)0x1234);
+
+        /**
+         * 1) Wait for rebalance event
+         * 2) Wait 1 interval (1s) expecting no IO (nothing produced).
+         * 3) Produce half the messages
+         * 4) Expect CB
+         * 5) Consume the available messages
+         * 6) Wait 1 interval expecting no CB.
+         * 7) Produce remaing half
+         * 8) Expect CB
+         * 9) Done.
+         */
+        while (recvd < msgcnt) {
+                TEST_SAY("Waiting for event\n");
+                callback_event_count = wait_event_cb(1 * wait_multiplier);
+                TEST_ASSERT(callback_event_count <= 1,
+                            "Event cb called %d times", callback_event_count);
+
+                if (callback_event_count == 1) {
+                        TEST_SAY("Events received: %d\n", callback_event_count);
+
+                        while ((rkev = rd_kafka_queue_poll(queue, 0))) {
+                                eventcnt++;
+                                switch (rd_kafka_event_type(rkev)) {
+                                case RD_KAFKA_EVENT_REBALANCE:
+                                        TEST_SAY(
+                                            "Got %s: %s\n",
+                                            rd_kafka_event_name(rkev),
+                                            rd_kafka_err2str(
+                                                rd_kafka_event_error(rkev)));
+                                        if (expecting_io != _REBALANCE)
+                                                TEST_FAIL(
+                                                    "Got Rebalance when "
+                                                    "expecting message\n");
+                                        if (rd_kafka_event_error(rkev) ==
+                                            RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS) {
+                                                rd_kafka_assign(
+                                                    rk_c,
+                                                    rd_kafka_event_topic_partition_list(
+                                                        rkev));
+                                                expecting_io = _NOPE;
+                                        } else
+                                                rd_kafka_assign(rk_c, NULL);
+                                        break;
+
+                                case RD_KAFKA_EVENT_FETCH:
+                                        if (expecting_io != _YEP)
+                                                TEST_FAIL(
+                                                    "Did not expect more "
+                                                    "messages at %d/%d\n",
+                                                    recvd, msgcnt);
+                                        recvd++;
+                                        if (recvd == (msgcnt / 2) ||
+                                            recvd == msgcnt)
+                                                expecting_io = _NOPE;
+                                        break;
+
+                                case RD_KAFKA_EVENT_ERROR:
+                                        TEST_FAIL(
+                                            "Error: %s\n",
+                                            rd_kafka_event_error_string(rkev));
+                                        break;
+
+                                default:
+                                        TEST_SAY("Ignoring event %s\n",
+                                                 rd_kafka_event_name(rkev));
+                                }
+
+                                rd_kafka_event_destroy(rkev);
+                        }
+                        TEST_SAY("%d events, Consumed %d/%d messages\n",
+                                 eventcnt, recvd, msgcnt);
+
+                        wait_multiplier = 1;
+
+                } else {
+                        if (expecting_io == _REBALANCE) {
+                                continue;
+                        } else if (expecting_io == _YEP) {
+                                TEST_FAIL(
+                                    "Did not see expected IO after %d/%d "
+                                    "msgs\n",
+                                    recvd, msgcnt);
+                        }
+
+                        TEST_SAY("Event wait timeout (good)\n");
+                        TEST_SAY("Got idle period, producing\n");
+                        test_produce_msgs(rk_p, rkt_p, testid, 0, recvd,
+                                          msgcnt / 2, NULL, 10);
+
+                        expecting_io = _YEP;
+                        /* When running slowly (e.g., valgrind) it might take
+                         * some time before the first message is received
+                         * after producing. */
+                        wait_multiplier = 3;
+                }
+        }
+        TEST_SAY("Done\n");
+
+        rd_kafka_topic_destroy(rkt_p);
+        rd_kafka_destroy(rk_p);
+
+        rd_kafka_queue_destroy(queue);
+        rd_kafka_consumer_close(rk_c);
+        rd_kafka_destroy(rk_c);
+
+        mtx_destroy(&event_receiver.lock);
+
+        return 0;
+}

package/deps/librdkafka/tests/0084-destroy_flags.c

@@ -0,0 +1,212 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2018-2022, Magnus Edenhill
+ *               2023, Confluent Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * @name Test rd_kafka_destroy_flags()
+ */
+
+
+#include "test.h"
+
+
+static RD_TLS int rebalance_cnt = 0;
+
+static void destroy_flags_rebalance_cb(rd_kafka_t *rk,
+                                       rd_kafka_resp_err_t err,
+                                       rd_kafka_topic_partition_list_t *parts,
+                                       void *opaque) {
+        rebalance_cnt++;
+
+        TEST_SAY("rebalance_cb: %s with %d partition(s)\n",
+                 rd_kafka_err2str(err), parts->cnt);
+
+        switch (err) {
+        case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS:
+                test_consumer_assign("rebalance", rk, parts);
+                break;
+
+        case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS:
+                test_consumer_unassign("rebalance", rk);
+                break;
+
+        default:
+                TEST_FAIL("rebalance_cb: error: %s", rd_kafka_err2str(err));
+        }
+}
+
+struct df_args {
+        rd_kafka_type_t client_type;
+        int produce_cnt;
+        int consumer_subscribe;
+        int consumer_unsubscribe;
+};
+
+static void do_test_destroy_flags(const char *topic,
+                                  int destroy_flags,
+                                  int local_mode,
+                                  const struct df_args *args) {
+        rd_kafka_t *rk;
+        rd_kafka_conf_t *conf;
+        test_timing_t t_destroy;
+
+        TEST_SAY(_C_MAG
+                 "[ test destroy_flags 0x%x for client_type %d, "
+                 "produce_cnt %d, subscribe %d, unsubscribe %d, "
+                 "%s mode ]\n" _C_CLR,
+                 destroy_flags, args->client_type, args->produce_cnt,
+                 args->consumer_subscribe, args->consumer_unsubscribe,
+                 local_mode ? "local" : "broker");
+
+        test_conf_init(&conf, NULL, 20);
+
+        if (local_mode)
+                test_conf_set(conf, "bootstrap.servers", "");
+
+        if (args->client_type == RD_KAFKA_PRODUCER) {
+
+                rk = test_create_handle(args->client_type, conf);
+
+                if (args->produce_cnt > 0) {
+                        rd_kafka_topic_t *rkt;
+                        int msgcounter = 0;
+
+                        rkt = test_create_producer_topic(rk, topic, NULL);
+                        test_produce_msgs_nowait(
+                            rk, rkt, 0, RD_KAFKA_PARTITION_UA, 0,
+                            args->produce_cnt, NULL, 100, 0, &msgcounter);
+                        rd_kafka_topic_destroy(rkt);
+                }
+
+        } else {
+                int i;
+
+                TEST_ASSERT(args->client_type == RD_KAFKA_CONSUMER);
+
+                rk = test_create_consumer(topic, destroy_flags_rebalance_cb,
+                                          conf, NULL);
+
+                if (args->consumer_subscribe) {
+                        test_consumer_subscribe(rk, topic);
+
+                        if (!local_mode) {
+                                TEST_SAY("Waiting for assignment\n");
+                                while (rebalance_cnt == 0)
+                                        test_consumer_poll_once(rk, NULL, 1000);
+                        }
+                }
+
+                for (i = 0; i < 5; i++)
+                        test_consumer_poll_once(rk, NULL, 100);
+
+                if (args->consumer_unsubscribe) {
+                        /* Test that calling rd_kafka_unsubscribe immediately
+                         * prior to rd_kafka_destroy_flags doesn't cause the
+                         * latter to hang. */
+                        TEST_SAY(_C_YEL "Calling rd_kafka_unsubscribe\n"_C_CLR);
+                        rd_kafka_unsubscribe(rk);
+                }
+        }
+
+        rebalance_cnt = 0;
+        TEST_SAY(_C_YEL "Calling rd_kafka_destroy_flags(0x%x)\n" _C_CLR,
+                 destroy_flags);
+        TIMING_START(&t_destroy, "rd_kafka_destroy_flags(0x%x)", destroy_flags);
+        rd_kafka_destroy_flags(rk, destroy_flags);
+        TIMING_STOP(&t_destroy);
+
+        if (destroy_flags & RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE)
+                TIMING_ASSERT_LATER(&t_destroy, 0, 200);
+        else
+                TIMING_ASSERT_LATER(&t_destroy, 0, 1000);
+
+        if (args->consumer_subscribe &&
+            !(destroy_flags & RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE)) {
+                if (!local_mode)
+                        TEST_ASSERT(rebalance_cnt > 0,
+                                    "expected final rebalance callback");
+        } else
+                TEST_ASSERT(rebalance_cnt == 0,
+                            "expected no rebalance callbacks, got %d",
+                            rebalance_cnt);
+
+        TEST_SAY(_C_GRN
+                 "[ test destroy_flags 0x%x for client_type %d, "
+                 "produce_cnt %d, subscribe %d, unsubscribe %d, "
+                 "%s mode: PASS ]\n" _C_CLR,
+                 destroy_flags, args->client_type, args->produce_cnt,
+                 args->consumer_subscribe, args->consumer_unsubscribe,
+                 local_mode ? "local" : "broker");
+}
+
+
+/**
+ * @brief Destroy with flags
+ */
+static void destroy_flags(int local_mode) {
+        const struct df_args args[] = {
+            {RD_KAFKA_PRODUCER, 0, 0, 0},
+            {RD_KAFKA_PRODUCER, test_quick ? 100 : 10000, 0, 0},
+            {RD_KAFKA_CONSUMER, 0, 1, 0},
+            {RD_KAFKA_CONSUMER, 0, 1, 1},
+            {RD_KAFKA_CONSUMER, 0, 0, 0}};
+        const int flag_combos[] = {0, RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE};
+        const char *topic       = test_mk_topic_name(__FUNCTION__, 1);
+        const rd_bool_t can_subscribe =
+            test_broker_version >= TEST_BRKVER(0, 9, 0, 0);
+        int i, j;
+
+        /* Create the topic to avoid not-yet-auto-created-topics being
+         * subscribed to (and thus raising an error). */
+        if (!local_mode) {
+                test_create_topic(NULL, topic, 3, 1);
+                test_wait_topic_exists(NULL, topic, 5000);
+        }
+
+        for (i = 0; i < (int)RD_ARRAYSIZE(args); i++) {
+                for (j = 0; j < (int)RD_ARRAYSIZE(flag_combos); j++) {
+                        if (!can_subscribe && (args[i].consumer_subscribe ||
+                                               args[i].consumer_unsubscribe))
+                                continue;
+                        do_test_destroy_flags(topic, flag_combos[j], local_mode,
+                                              &args[i]);
+                }
+        }
+}
+
+
+
+int main_0084_destroy_flags_local(int argc, char **argv) {
+        destroy_flags(1 /*no brokers*/);
+        return 0;
+}
+
+int main_0084_destroy_flags(int argc, char **argv) {
+        destroy_flags(0 /*with brokers*/);
+        return 0;
+}