@platformatic/rdkafka 4.0.1 → 4.1.0-alpha.1

package/deps/librdkafka/tests/0143-exponential_backoff_mock.c

@@ -110,6 +110,23 @@ static void test_find_coordinator(rd_kafka_mock_cluster_t *mcluster,
         SUB_TEST_PASS();
 }
 
+/** @brief Max Jitter is 20 percent each side so buffer chosen
+ *         is +- a fixed error to account for latency delays. */
+static void check_time_difference(int64_t time_difference, int retry_count) {
+        int32_t error = retry_ms / 2;
+        int64_t low   = (1 << retry_count) * retry_ms * 0.8 - error;
+        int64_t high  = (1 << retry_count) * retry_ms * 1.2 + error;
+        if (high > (retry_max_ms * 1.2 + error))
+                high = retry_max_ms * 1.2 + error;
+        if (low > (retry_max_ms * 0.8 - error))
+                low = retry_max_ms * 0.8 - error;
+        TEST_ASSERT((time_difference < high) && (time_difference > low),
+                    "Time difference is not respected, should "
+                    "be between %" PRId64 " and %" PRId64
+                    " where time difference is %" PRId64 "\n",
+                    low, high, time_difference);
+}
+
 /**
  * Exponential Backoff needs to be checked for the request_type. Also the
  * request_type should only be retried if one previous has failed for correct
@@ -138,22 +155,7 @@ static void helper_exponential_backoff(rd_kafka_mock_cluster_t *mcluster,
                             (rd_kafka_mock_request_timestamp(requests[i]) -
                              previous_request_ts) /
                             1000;
-                        /* Max Jitter is 20 percent each side so buffer chosen
-                         * is 25 percent to account for latency delays */
-                        int64_t low =
-                            ((1 << retry_count) * (retry_ms)*75) / 100;
-                        int64_t high =
-                            ((1 << retry_count) * (retry_ms)*125) / 100;
-                        if (high > ((retry_max_ms * 125) / 100))
-                                high = (retry_max_ms * 125) / 100;
-                        if (low > ((retry_max_ms * 75) / 100))
-                                low = (retry_max_ms * 75) / 100;
-                        TEST_ASSERT((time_difference < high) &&
-                                        (time_difference > low),
-                                    "Time difference is not respected, should "
-                                    "be between %" PRId64 " and %" PRId64
-                                    " where time difference is %" PRId64 "\n",
-                                    low, high, time_difference);
+                        check_time_difference(time_difference, retry_count);
                         retry_count++;
                 }
                 previous_request_ts =
@@ -226,6 +228,7 @@ static void test_produce(rd_kafka_mock_cluster_t *mcluster,
 
         producer = test_create_handle(RD_KAFKA_PRODUCER, conf);
         rkt      = test_create_producer_topic(producer, topic, NULL);
+        test_wait_topic_exists(producer, topic, 5000);
 
         rd_kafka_mock_push_request_errors(
             mcluster, RD_KAFKAP_Produce, 7,
@@ -304,15 +307,20 @@ static void test_heartbeat_find_coordinator(rd_kafka_mock_cluster_t *mcluster,
                                             rd_kafka_conf_t *conf) {
         rd_kafka_t *consumer;
         rd_kafka_message_t *rkm;
+        int ApiKey;
         SUB_TEST();
         test_conf_set(conf, "auto.offset.reset", "earliest");
         test_conf_set(conf, "enable.auto.commit", "false");
 
         consumer = test_create_consumer(topic, NULL, conf, NULL);
 
+        if (test_consumer_group_protocol_classic()) {
+                ApiKey = RD_KAFKAP_Heartbeat;
+        } else {
+                ApiKey = RD_KAFKAP_ConsumerGroupHeartbeat;
+        }
         rd_kafka_mock_push_request_errors(
-            mcluster, RD_KAFKAP_Heartbeat, 1,
-            RD_KAFKA_RESP_ERR_NOT_COORDINATOR_FOR_GROUP);
+            mcluster, ApiKey, 1, RD_KAFKA_RESP_ERR_NOT_COORDINATOR_FOR_GROUP);
 
         rd_kafka_mock_clear_requests(mcluster);
         test_consumer_subscribe(consumer, topic);
@@ -323,7 +331,7 @@ static void test_heartbeat_find_coordinator(rd_kafka_mock_cluster_t *mcluster,
         rd_sleep(6);
 
 
-        helper_find_coordinator_trigger(mcluster, RD_KAFKAP_Heartbeat);
+        helper_find_coordinator_trigger(mcluster, ApiKey);
 
 
         rd_kafka_destroy(consumer);
@@ -389,6 +397,7 @@ static void test_produce_fast_leader_query(rd_kafka_mock_cluster_t *mcluster,
 
         producer = test_create_handle(RD_KAFKA_PRODUCER, conf);
         rkt      = test_create_producer_topic(producer, topic, NULL);
+        test_wait_topic_exists(producer, topic, 5000);
 
         rd_kafka_mock_push_request_errors(
             mcluster, RD_KAFKAP_Produce, 1,
@@ -418,24 +427,8 @@ static void test_produce_fast_leader_query(rd_kafka_mock_cluster_t *mcluster,
                                          requests[i]) -
                                      previous_request_ts) /
                                     1000;
-                                /* Max Jitter is 20 percent each side so buffer
-                                 * chosen is 25 percent to account for latency
-                                 * delays */
-                                int64_t low =
-                                    ((1 << retry_count) * (retry_ms)*75) / 100;
-                                int64_t high =
-                                    ((1 << retry_count) * (retry_ms)*125) / 100;
-                                if (high > ((retry_max_ms * 125) / 100))
-                                        high = (retry_max_ms * 125) / 100;
-                                if (low > ((retry_max_ms * 75) / 100))
-                                        low = (retry_max_ms * 75) / 100;
-                                TEST_ASSERT(
-                                    (time_difference < high) &&
-                                        (time_difference > low),
-                                    "Time difference is not respected, should "
-                                    "be between %" PRId64 " and %" PRId64
-                                    " where time difference is %" PRId64 "\n",
-                                    low, high, time_difference);
+                                check_time_difference(time_difference,
+                                                      retry_count);
                                 retry_count++;
                         }
                         previous_request_ts =
@@ -500,7 +493,8 @@ static void test_fetch_fast_leader_query(rd_kafka_mock_cluster_t *mcluster,
                          previous_request_was_Fetch) {
                         Metadata_after_Fetch = rd_true;
                         break;
-                } else
+                } else if (rd_kafka_mock_request_api_key(requests[i]) !=
+                           RD_KAFKAP_ConsumerGroupHeartbeat)
                         previous_request_was_Fetch = rd_false;
         }
         rd_kafka_destroy(consumer);
@@ -542,11 +536,16 @@ int main_0143_exponential_backoff_mock(int argc, char **argv) {
         test_offset_commit(mcluster, topic, rd_kafka_conf_dup(conf));
         test_heartbeat_find_coordinator(mcluster, topic,
                                         rd_kafka_conf_dup(conf));
-        test_joingroup_find_coordinator(mcluster, topic,
-                                        rd_kafka_conf_dup(conf));
         test_fetch_fast_leader_query(mcluster, topic, rd_kafka_conf_dup(conf));
         test_produce_fast_leader_query(mcluster, topic,
                                        rd_kafka_conf_dup(conf));
+
+        /* No JoinGroup RPC in KIP 848 protocol. */
+        if (test_consumer_group_protocol_classic()) {
+                test_joingroup_find_coordinator(mcluster, topic,
+                                                rd_kafka_conf_dup(conf));
+        }
+
         test_mock_cluster_destroy(mcluster);
         rd_kafka_conf_destroy(conf);
         return 0;

package/deps/librdkafka/tests/0146-metadata_mock.c

@@ -104,19 +104,41 @@ static void do_test_metadata_persists_in_cache(const char *assignor) {
 }
 
 /**
- * @brief No loop of metadata requests should be started
- *        when a metadata request is made without leader epoch change.
- *        See issue #4577
+ * @brief Tests about the fast metadata refresh feature.
+ *
+ * - variation 0: no loop of metadata requests should be started
+ *   when a metadata request is made without leader epoch change.
+ *   It's expected it stops after the first request.
+ *   See issue #4577
+ *
+ * - variation 1: an error is returned on subsequent metadata response too,
+ *   metadata refreshes continue every second until the error is cleared.
+ *
+ * It's expected that the temporary error doesn't remove the topic from cache,
+ * produce requests aren't blocked from metadata errors and no error is
+ * to be surfaced to the application.
  */
-static void do_test_fast_metadata_refresh_stops(void) {
+static void do_test_fast_metadata_refresh(int variation) {
         rd_kafka_t *rk;
         const char *bootstraps;
         rd_kafka_mock_cluster_t *mcluster;
         const char *topic = test_mk_topic_name(__FUNCTION__, 1);
         rd_kafka_conf_t *conf;
         int metadata_requests;
+        int expected_metadata_requests;
+        switch (variation) {
+        case 0:
+                expected_metadata_requests = 2;
+                break;
+        case 1:
+                expected_metadata_requests = 7;
+                break;
+        default:
+                TEST_FAIL("Invalid variation %d", variation);
+                break;
+        }
 
-        SUB_TEST_QUICK();
+        SUB_TEST_QUICK("%s", variation == 0 ? "stops" : "retries");
 
         mcluster = test_mock_cluster_new(3, &bootstraps);
         rd_kafka_mock_topic_create(mcluster, topic, 1, 1);
@@ -127,22 +149,51 @@ static void do_test_fast_metadata_refresh_stops(void) {
 
         rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
 
+        rd_kafka_mock_start_request_tracking(mcluster);
+
+        if (variation == 1) {
+                /* Produce some messages to the topic
+                 * and keep metadata in cache. */
+                test_produce_msgs2(rk, topic, 0, 0, 0, 3, NULL, 5);
+        }
+
         /* This error triggers a metadata refresh but no leader change
          * happened */
         rd_kafka_mock_push_request_errors(
             mcluster, RD_KAFKAP_Produce, 1,
             RD_KAFKA_RESP_ERR_KAFKA_STORAGE_ERROR);
 
-        rd_kafka_mock_start_request_tracking(mcluster);
-        test_produce_msgs2(rk, topic, 0, 0, 0, 1, NULL, 5);
+        if (variation == 1) {
+                rd_kafka_mock_topic_set_error(
+                    mcluster, topic, RD_KAFKA_RESP_ERR_KAFKA_STORAGE_ERROR);
+        }
+
+        test_produce_msgs2(rk, topic, 0, 0, 0, 3, NULL, 5);
+
+        /* Wait some time for seeing the retries */
+        rd_sleep(3);
+
+        if (variation == 1) {
+                /* Clear topic error to stop the retries */
+                rd_kafka_mock_topic_set_error(mcluster, topic,
+                                              RD_KAFKA_RESP_ERR_NO_ERROR);
+        }
 
         /* First call is for getting initial metadata,
          * second one happens after the error,
-         * it should stop refreshing metadata after that. */
+         * it should stop refreshing metadata after that.
+         *
+         * There can be an additional metadata request originating from
+         * the 1s timer when the partition is being delegated or
+         * the broker is connecting but still not up. */
         metadata_requests = test_mock_wait_matching_requests(
-            mcluster, 2, 500, is_metadata_request, NULL);
-        TEST_ASSERT(metadata_requests == 2,
-                    "Expected 2 metadata request, got %d", metadata_requests);
+            mcluster, expected_metadata_requests, 500, is_metadata_request,
+            NULL);
+        TEST_ASSERT(expected_metadata_requests <= metadata_requests &&
+                        metadata_requests <= expected_metadata_requests + 1,
+                    "Expected %d or %d metadata request, got %d",
+                    expected_metadata_requests, expected_metadata_requests + 1,
+                    metadata_requests);
         rd_kafka_mock_stop_request_tracking(mcluster);
 
         rd_kafka_destroy(rk);
@@ -344,8 +395,16 @@ static void do_test_metadata_update_operation(rd_bool_t producer,
                 test_produce_msgs2(rk, topic, 0, 1, 0, 1, NULL, 0);
                 rd_kafka_flush(rk, 1000);
         } else {
+                test_produce_msgs_easy_v(topic, 0, 0, 0, 1, 0,
+                                         "bootstrap.servers", bootstraps, NULL);
+                test_produce_msgs_easy_v(topic, 0, 1, 0, 1, 0,
+                                         "bootstrap.servers", bootstraps, NULL);
+
                 rd_kafka_topic_partition_list_t *assignment;
                 test_conf_set(conf, "group.id", topic);
+                test_conf_set(conf, "fetch.wait.max.ms", "100");
+                test_conf_set(conf, "auto.offset.reset", "earliest");
+
                 rk = test_create_handle(RD_KAFKA_CONSUMER, conf);
 
                 assignment = rd_kafka_topic_partition_list_new(1);
@@ -355,7 +414,8 @@ static void do_test_metadata_update_operation(rd_bool_t producer,
                 rd_kafka_topic_partition_list_destroy(assignment);
 
                 /* Start consuming from leader 1 and 2 */
-                test_consumer_poll_no_msgs("no errors", rk, 0, 1000);
+                test_consumer_poll_timeout("initial leaders", rk, 0, -1, -1, 2,
+                                           NULL, 5000);
         }
 
         TIMING_START(&timing, "Metadata update and partition migration");
@@ -377,10 +437,10 @@ static void do_test_metadata_update_operation(rd_bool_t producer,
                                          "bootstrap.servers", bootstraps, NULL);
                 test_produce_msgs_easy_v(topic, 0, 1, 0, 1, 0,
                                          "bootstrap.servers", bootstraps, NULL);
-                test_consumer_poll_timeout("partition 0", rk, 0, -1, -1, 2,
+                test_consumer_poll_timeout("changed leaders", rk, 0, -1, -1, 2,
                                            NULL, 5000);
         }
-        TIMING_ASSERT_LATER(&timing, 0, 2000);
+        TIMING_ASSERT_LATER(&timing, 0, 500);
 
         /* Leader change triggers the metadata update and migration
          * of partition 0 to brokers 3 and with 'second_leader_change' also
@@ -429,7 +489,8 @@ int main_0146_metadata_mock(int argc, char **argv) {
 
         do_test_metadata_call_before_join();
 
-        do_test_fast_metadata_refresh_stops();
+        do_test_fast_metadata_refresh(0);
+        do_test_fast_metadata_refresh(1);
 
         do_test_stale_metadata_doesnt_migrate_partition();