@platformatic/rdkafka 4.0.0 → 4.1.0-alpha.1

package/deps/librdkafka/src/rdkafka_op.h

@@ -400,6 +400,8 @@ struct rd_kafka_op_s {
                 struct {
                         rd_kafka_metadata_t *md;
                         rd_kafka_metadata_internal_t *mdi;
+                        /* subscription version for this call */
+                        int32_t subscription_version;
                         int force; /* force request regardless of outstanding
                                     * metadata requests. */
                 } metadata;
@@ -413,7 +415,6 @@ struct rd_kafka_op_s {
                 } dr;
 
                 struct {
-                        int32_t nodeid;
                         char nodename[RD_KAFKA_NODENAME_SIZE];
                 } node;
 
@@ -470,13 +471,14 @@ struct rd_kafka_op_s {
                         struct rd_kafka_admin_worker_cbs *cbs;
 
                         /** Worker state */
-                        enum { RD_KAFKA_ADMIN_STATE_INIT,
-                               RD_KAFKA_ADMIN_STATE_WAIT_BROKER,
-                               RD_KAFKA_ADMIN_STATE_WAIT_CONTROLLER,
-                               RD_KAFKA_ADMIN_STATE_WAIT_FANOUTS,
-                               RD_KAFKA_ADMIN_STATE_CONSTRUCT_REQUEST,
-                               RD_KAFKA_ADMIN_STATE_WAIT_RESPONSE,
-                               RD_KAFKA_ADMIN_STATE_WAIT_BROKER_LIST,
+                        enum {
+                                RD_KAFKA_ADMIN_STATE_INIT,
+                                RD_KAFKA_ADMIN_STATE_WAIT_BROKER,
+                                RD_KAFKA_ADMIN_STATE_WAIT_CONTROLLER,
+                                RD_KAFKA_ADMIN_STATE_WAIT_FANOUTS,
+                                RD_KAFKA_ADMIN_STATE_CONSTRUCT_REQUEST,
+                                RD_KAFKA_ADMIN_STATE_WAIT_RESPONSE,
+                                RD_KAFKA_ADMIN_STATE_WAIT_BROKER_LIST,
                         } state;
 
                         int32_t broker_id; /**< Requested broker id to
@@ -544,6 +546,11 @@ struct rd_kafka_op_s {
                         /** Result cb for this op */
                         void (*result_cb)(rd_kafka_op_t *);
 
+                        struct rd_kafka_admin_worker_cbs
+                            *cbs; /**< Worker Callbacks
+                                   *   Moved from admin request
+                                   */
+
                         rd_list_t results; /**< Type depends on request type:
                                             *
                                             * (rd_kafka_topic_result_t *):
@@ -570,19 +577,22 @@ struct rd_kafka_op_s {
 
                 /**< Mock cluster command */
                 struct {
-                        enum { RD_KAFKA_MOCK_CMD_TOPIC_SET_ERROR,
-                               RD_KAFKA_MOCK_CMD_TOPIC_CREATE,
-                               RD_KAFKA_MOCK_CMD_PART_SET_LEADER,
-                               RD_KAFKA_MOCK_CMD_PART_SET_FOLLOWER,
-                               RD_KAFKA_MOCK_CMD_PART_SET_FOLLOWER_WMARKS,
-                               RD_KAFKA_MOCK_CMD_PART_PUSH_LEADER_RESPONSE,
-                               RD_KAFKA_MOCK_CMD_BROKER_SET_UPDOWN,
-                               RD_KAFKA_MOCK_CMD_BROKER_SET_RTT,
-                               RD_KAFKA_MOCK_CMD_BROKER_SET_RACK,
-                               RD_KAFKA_MOCK_CMD_COORD_SET,
-                               RD_KAFKA_MOCK_CMD_APIVERSION_SET,
-                               RD_KAFKA_MOCK_CMD_REQUESTED_METRICS_SET,
-                               RD_KAFKA_MOCK_CMD_TELEMETRY_PUSH_INTERVAL_SET,
+                        enum {
+                                RD_KAFKA_MOCK_CMD_TOPIC_SET_ERROR,
+                                RD_KAFKA_MOCK_CMD_TOPIC_CREATE,
+                                RD_KAFKA_MOCK_CMD_PART_SET_LEADER,
+                                RD_KAFKA_MOCK_CMD_PART_SET_FOLLOWER,
+                                RD_KAFKA_MOCK_CMD_PART_SET_FOLLOWER_WMARKS,
+                                RD_KAFKA_MOCK_CMD_PART_PUSH_LEADER_RESPONSE,
+                                RD_KAFKA_MOCK_CMD_BROKER_SET_UPDOWN,
+                                RD_KAFKA_MOCK_CMD_BROKER_SET_RTT,
+                                RD_KAFKA_MOCK_CMD_BROKER_SET_RACK,
+                                RD_KAFKA_MOCK_CMD_BROKER_DECOMMISSION,
+                                RD_KAFKA_MOCK_CMD_BROKER_ADD,
+                                RD_KAFKA_MOCK_CMD_COORD_SET,
+                                RD_KAFKA_MOCK_CMD_APIVERSION_SET,
+                                RD_KAFKA_MOCK_CMD_REQUESTED_METRICS_SET,
+                                RD_KAFKA_MOCK_CMD_TELEMETRY_PUSH_INTERVAL_SET,
                         } cmd;
 
                         rd_kafka_resp_err_t err; /**< Error for:
@@ -610,6 +620,8 @@ struct rd_kafka_op_s {
                                                   *    PART_SET_LEADER
                                                   *    BROKER_SET_UPDOWN
                                                   *    BROKER_SET_RACK
+                                                  *    BROKER_DECOMMISSION
+                                                  *    BROKER_ADD
                                                   *    COORD_SET */
                         int64_t lo;              /**< Low offset, for:
                                                   *    TOPIC_CREATE (part cnt)
@@ -700,6 +712,18 @@ struct rd_kafka_op_s {
                         rd_kafka_broker_t *rkb;
                 } telemetry_broker;
 
+                struct {
+                        /**
+                         * Terminated and freed broker pointer,
+                         * can only be used for pointer comparison.
+                         */
+                        void *rkb;
+
+                        /** Termination callback to trigger
+                         * on the op handler's thread. */
+                        void (*cb)(rd_kafka_t *rk, void *rkb);
+                } terminated;
+
         } rko_u;
 };
 

package/deps/librdkafka/src/rdkafka_partition.c

@@ -146,8 +146,8 @@ static void rd_kafka_toppar_consumer_lag_req(rd_kafka_toppar_t *rktp) {
          * broker supports FETCH >= v5, since this will be set when
          * doing fetch requests.
          */
-        if (rd_kafka_broker_ApiVersion_supported(
-                rktp->rktp_broker, RD_KAFKAP_Fetch, 0, 5, NULL) == 5) {
+        if (rd_kafka_broker_ApiVersion_at_least(rktp->rktp_broker,
+                                                RD_KAFKAP_Fetch, 5)) {
                 rd_kafka_toppar_unlock(rktp);
                 return;
         }
@@ -376,14 +376,24 @@ void rd_kafka_toppar_set_fetch_state(rd_kafka_toppar_t *rktp, int fetch_state) {
 
         rktp->rktp_fetch_state = fetch_state;
 
-        if (fetch_state == RD_KAFKA_TOPPAR_FETCH_ACTIVE)
+        if (fetch_state == RD_KAFKA_TOPPAR_FETCH_ACTIVE) {
+                rktp->rktp_ts_fetch_backoff = 0;
+
+                /* Wake-up broker thread which might be idling on IO */
+                if (rktp->rktp_broker)
+                        rd_kafka_broker_wakeup(rktp->rktp_broker,
+                                               "fetch start");
+
                 rd_kafka_dbg(
                     rktp->rktp_rkt->rkt_rk, CONSUMER | RD_KAFKA_DBG_TOPIC,
                     "FETCH",
                     "Partition %.*s [%" PRId32 "] start fetching at %s",
                     RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
                     rktp->rktp_partition,
-                    rd_kafka_fetch_pos2str(rktp->rktp_next_fetch_start));
+                    rd_kafka_fetch_pos2str(
+                        rd_kafka_toppar_fetch_decide_next_fetch_start_pos(
+                            rktp)));
+        }
 }
 
 
@@ -1225,7 +1235,8 @@ void rd_kafka_toppar_broker_delegate(rd_kafka_toppar_t *rktp,
 
         /* Undelegated toppars are delgated to the internal
          * broker for bookkeeping. */
-        if (!rkb && !rd_kafka_terminating(rk)) {
+        if (!rd_kafka_terminating(rk) &&
+            (!rkb || rd_kafka_broker_termination_in_progress(rkb))) {
                 rkb               = rd_kafka_broker_internal(rk);
                 internal_fallback = 1;
         }
@@ -1338,7 +1349,7 @@ void rd_kafka_toppar_next_offset_handle(rd_kafka_toppar_t *rktp,
         if (rktp->rktp_query_pos.offset <= RD_KAFKA_OFFSET_TAIL_BASE) {
                 int64_t orig_offset = next_pos.offset;
                 int64_t tail_cnt    = llabs(rktp->rktp_query_pos.offset -
-                                         RD_KAFKA_OFFSET_TAIL_BASE);
+                                            RD_KAFKA_OFFSET_TAIL_BASE);
 
                 if (tail_cnt > next_pos.offset)
                         next_pos.offset = 0;
@@ -1359,10 +1370,6 @@ void rd_kafka_toppar_next_offset_handle(rd_kafka_toppar_t *rktp,
         rd_kafka_toppar_set_next_fetch_position(rktp, next_pos);
 
         rd_kafka_toppar_set_fetch_state(rktp, RD_KAFKA_TOPPAR_FETCH_ACTIVE);
-
-        /* Wake-up broker thread which might be idling on IO */
-        if (rktp->rktp_broker)
-                rd_kafka_broker_wakeup(rktp->rktp_broker, "ready to fetch");
 }
 
 
@@ -1746,11 +1753,6 @@ static void rd_kafka_toppar_fetch_start(rd_kafka_toppar_t *rktp,
 
                 rd_kafka_toppar_set_fetch_state(rktp,
                                                 RD_KAFKA_TOPPAR_FETCH_ACTIVE);
-
-                /* Wake-up broker thread which might be idling on IO */
-                if (rktp->rktp_broker)
-                        rd_kafka_broker_wakeup(rktp->rktp_broker,
-                                               "fetch start");
         }
 
         rktp->rktp_offsets_fin.eof_offset = RD_KAFKA_OFFSET_INVALID;
@@ -2160,6 +2162,7 @@ static rd_kafka_op_res_t rd_kafka_toppar_op_serve(rd_kafka_t *rk,
                 rd_kafka_toppar_lock(rktp);
 
                 if (rko->rko_err) {
+                        int actions;
                         rd_kafka_dbg(
                             rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
                             "Failed to fetch offset for "
@@ -2174,10 +2177,15 @@ static rd_kafka_op_res_t rd_kafka_toppar_op_serve(rd_kafka_t *rk,
                         rd_kafka_toppar_unlock(rktp);
 
 
-                        /* Propagate error to application */
+                        actions = rd_kafka_handle_OffsetFetch_err_action(
+                            NULL, rko->rko_err, NULL);
+                        /* Propagate error to application. Exclude
+                         * permanent errors that caused a coordinator
+                         * refresh like `NOT_COORDINATOR` */
                         if (rko->rko_err != RD_KAFKA_RESP_ERR__WAIT_COORD &&
                             rko->rko_err !=
-                                RD_KAFKA_RESP_ERR_UNSTABLE_OFFSET_COMMIT)
+                                RD_KAFKA_RESP_ERR_UNSTABLE_OFFSET_COMMIT &&
+                            !(actions & RD_KAFKA_ERR_ACTION_REFRESH))
                                 rd_kafka_consumer_err(
                                     rktp->rktp_fetchq, RD_KAFKA_NODEID_UA,
                                     rko->rko_err, 0, NULL, rktp,
@@ -2582,7 +2590,8 @@ void rd_kafka_toppar_leader_unavailable(rd_kafka_toppar_t *rktp,
         rkt->rkt_flags |= RD_KAFKA_TOPIC_F_LEADER_UNAVAIL;
         rd_kafka_topic_wrunlock(rkt);
 
-        rd_kafka_topic_fast_leader_query(rkt->rkt_rk);
+        rd_kafka_topic_fast_leader_query(rkt->rkt_rk,
+                                         rd_false /* don't force */);
 }
 
 
@@ -3674,11 +3683,12 @@ static rd_bool_t rd_kafka_topic_partition_list_get_leaders(
                 struct rd_kafka_partition_leader *leader;
                 const rd_kafka_metadata_topic_t *mtopic;
                 const rd_kafka_metadata_partition_t *mpart;
+                const rd_kafka_metadata_partition_internal_t *mdpi;
                 rd_bool_t topic_wait_cache;
 
                 rd_kafka_metadata_cache_topic_partition_get(
-                    rk, &mtopic, &mpart, rktpar->topic, rktpar->partition,
-                    0 /*negative entries too*/);
+                    rk, &mtopic, &mpart, &mdpi, rktpar->topic,
+                    rktpar->partition, 0 /*negative entries too*/);
 
                 topic_wait_cache =
                     !mtopic ||
@@ -3746,9 +3756,11 @@ static rd_bool_t rd_kafka_topic_partition_list_get_leaders(
                         rd_kafka_topic_partition_update(rktpar2, rktpar);
                 } else {
                         /* Make a copy of rktpar and add to partitions list */
-                        rd_kafka_topic_partition_list_add_copy(
+                        rktpar2 = rd_kafka_topic_partition_list_add_copy(
                             leader->partitions, rktpar);
                 }
+                rd_kafka_topic_partition_set_current_leader_epoch(
+                    rktpar2, mdpi->leader_epoch);
 
                 rktpar->err = RD_KAFKA_RESP_ERR_NO_ERROR;
 
@@ -3872,7 +3884,7 @@ rd_kafka_topic_partition_list_query_leaders_async_worker(rd_kafka_op_t *rko) {
                 rd_kafka_metadata_refresh_topics(
                     rk, NULL, &query_topics, rd_true /*force*/,
                     rd_false /*!allow_auto_create*/, rd_false /*!cgrp_update*/,
-                    "query partition leaders");
+                    -1, "query partition leaders");
         }
 
         rd_list_destroy(leaders);
@@ -4061,7 +4073,7 @@ rd_kafka_resp_err_t rd_kafka_topic_partition_list_query_leaders(
                         rd_kafka_metadata_refresh_topics(
                             rk, NULL, &query_topics, rd_true /*force*/,
                             rd_false /*!allow_auto_create*/,
-                            rd_false /*!cgrp_update*/,
+                            rd_false /*!cgrp_update*/, -1,
                             "query partition leaders");
                         ts_query = now;
                         query_cnt++;
@@ -4421,6 +4433,89 @@ int rd_kafka_topic_partition_list_regex_cnt(
 }
 
 
+/**
+ * @brief Match function that returns true if topic is not a regex.
+ */
+static int rd_kafka_topic_partition_not_regex(const void *elem,
+                                              const void *opaque) {
+        const rd_kafka_topic_partition_t *rktpar = elem;
+        return *rktpar->topic != '^';
+}
+
+/**
+ * @brief Return a new list with all regex topics removed.
+ *
+ * @remark The caller is responsible for freeing the returned list.
+ */
+rd_kafka_topic_partition_list_t *rd_kafka_topic_partition_list_remove_regexes(
+    const rd_kafka_topic_partition_list_t *rktparlist) {
+        return rd_kafka_topic_partition_list_match(
+            rktparlist, rd_kafka_topic_partition_not_regex, NULL);
+}
+
+
+/**
+ * @brief Combine regexes present in the list into a single regex.
+ */
+rd_kafkap_str_t *rd_kafka_topic_partition_list_combine_regexes(
+    const rd_kafka_topic_partition_list_t *rktparlist) {
+        int i;
+        int combined_regex_len   = 1; /* 1 for null-terminator */
+        int regex_cnt            = 0;
+        int j                    = 1;
+        rd_bool_t is_first_regex = rd_true;
+        char *combined_regex_str;
+        rd_kafkap_str_t *combined_regex_kstr;
+
+        // Count the number of characters needed for the combined regex string
+        for (i = 0; i < rktparlist->cnt; i++) {
+                const rd_kafka_topic_partition_t *rktpar =
+                    &(rktparlist->elems[i]);
+                if (*rktpar->topic == '^') {
+                        combined_regex_len += strlen(rktpar->topic);
+                        regex_cnt++;
+                }
+        }
+
+        if (regex_cnt == 0)
+                return rd_kafkap_str_new("", 0);
+
+        combined_regex_len +=
+            3 * (regex_cnt - 1); /* 1 for each ')|(' separator */
+        combined_regex_len += 2; /* 2 for enclosing brackets */
+
+        // memory allocation for the combined regex string
+        combined_regex_str = rd_malloc(combined_regex_len);
+
+        // Construct the combined regex string
+        combined_regex_str[0] = '(';
+        for (i = 0; i < rktparlist->cnt; i++) {
+                const rd_kafka_topic_partition_t *rktpar =
+                    &(rktparlist->elems[i]);
+                char *topic = rktpar->topic;
+                if (*topic == '^') {
+                        if (!is_first_regex) {
+                                combined_regex_str[j++] = ')';
+                                combined_regex_str[j++] = '|';
+                                combined_regex_str[j++] = '(';
+                        }
+                        while (*topic) {
+                                combined_regex_str[j++] = *topic;
+                                topic++;
+                        }
+                        is_first_regex = rd_false;
+                }
+        }
+        combined_regex_str[j++] = ')';
+        combined_regex_str[j]   = '\0';
+
+        combined_regex_kstr =
+            rd_kafkap_str_new(combined_regex_str, combined_regex_len - 1);
+        rd_free(combined_regex_str);
+        return combined_regex_kstr;
+}
+
+
 /**
  * @brief Reset base sequence for this toppar.
  *

package/deps/librdkafka/src/rdkafka_partition.h

@@ -292,13 +292,14 @@ struct rd_kafka_toppar_s {                           /* rd_kafka_toppar_t */
         int32_t rktp_fetch_version; /* Op version of curr fetch.
                                        (broker thread) */
 
-        enum { RD_KAFKA_TOPPAR_FETCH_NONE = 0,
-               RD_KAFKA_TOPPAR_FETCH_STOPPING,
-               RD_KAFKA_TOPPAR_FETCH_STOPPED,
-               RD_KAFKA_TOPPAR_FETCH_OFFSET_QUERY,
-               RD_KAFKA_TOPPAR_FETCH_OFFSET_WAIT,
-               RD_KAFKA_TOPPAR_FETCH_VALIDATE_EPOCH_WAIT,
-               RD_KAFKA_TOPPAR_FETCH_ACTIVE,
+        enum {
+                RD_KAFKA_TOPPAR_FETCH_NONE = 0,
+                RD_KAFKA_TOPPAR_FETCH_STOPPING,
+                RD_KAFKA_TOPPAR_FETCH_STOPPED,
+                RD_KAFKA_TOPPAR_FETCH_OFFSET_QUERY,
+                RD_KAFKA_TOPPAR_FETCH_OFFSET_WAIT,
+                RD_KAFKA_TOPPAR_FETCH_VALIDATE_EPOCH_WAIT,
+                RD_KAFKA_TOPPAR_FETCH_ACTIVE,
         } rktp_fetch_state; /* Broker thread's state */
 
 #define RD_KAFKA_TOPPAR_FETCH_IS_STARTED(fetch_state)                          \
@@ -433,6 +434,8 @@ struct rd_kafka_toppar_s {                           /* rd_kafka_toppar_t */
 #define RD_KAFKA_TOPPAR_F_ASSIGNED                                             \
         0x2000 /**< Toppar is part of the consumer                             \
                 *   assignment. */
+#define RD_KAFKA_TOPPAR_F_VALIDATING                                           \
+        0x4000 /**< Toppar is currently requesting validation. */
 
         /*
          * Timers
@@ -1008,6 +1011,12 @@ rd_kafka_resp_err_t rd_kafka_topic_partition_list_get_err(
 int rd_kafka_topic_partition_list_regex_cnt(
     const rd_kafka_topic_partition_list_t *rktparlist);
 
+rd_kafka_topic_partition_list_t *rd_kafka_topic_partition_list_remove_regexes(
+    const rd_kafka_topic_partition_list_t *rktparlist);
+
+rd_kafkap_str_t *rd_kafka_topic_partition_list_combine_regexes(
+    const rd_kafka_topic_partition_list_t *rktparlist);
+
 void *rd_kafka_topic_partition_list_copy_opaque(const void *src, void *opaque);
 
 /**

package/deps/librdkafka/src/rdkafka_proto.h

@@ -172,6 +172,7 @@ static RD_UNUSED const char *rd_kafka_ApiKey2str(int16_t ApiKey) {
             [RD_KAFKAP_ListTransactions]          = "ListTransactions",
             [RD_KAFKAP_AllocateProducerIds]       = "AllocateProducerIds",
             [RD_KAFKAP_ConsumerGroupHeartbeat]    = "ConsumerGroupHeartbeat",
+            [RD_KAFKAP_ConsumerGroupDescribe]     = "ConsumerGroupDescribe",
             [RD_KAFKAP_GetTelemetrySubscriptions] = "GetTelemetrySubscriptions",
             [RD_KAFKAP_PushTelemetry]             = "PushTelemetry",
 
@@ -283,6 +284,8 @@ typedef struct rd_kafkap_str_s {
 #define RD_KAFKAP_STR_INITIALIZER                                              \
         { .len = RD_KAFKAP_STR_LEN_NULL, .str = NULL }
 
+#define RD_KAFKAP_STR_INITIALIZER_EMPTY                                        \
+        { .len = 0, .str = "" }
 /**
  * Frees a Kafka string previously allocated with `rd_kafkap_str_new()`
  */
@@ -620,7 +623,7 @@ static RD_INLINE RD_UNUSED int rd_kafka_Uuid_ptr_cmp(void *a, void *b) {
 
 rd_kafka_Uuid_t rd_kafka_Uuid_random();
 
-const char *rd_kafka_Uuid_str(const rd_kafka_Uuid_t *uuid);
+char *rd_kafka_Uuid_str(const rd_kafka_Uuid_t *uuid);
 
 unsigned int rd_kafka_Uuid_hash(const rd_kafka_Uuid_t *uuid);
 

package/deps/librdkafka/src/rdkafka_protocol.h

@@ -123,5 +123,6 @@
 
 #define RD_KAFKAP__NUM 74
 
+#define RD_KAFKAP_RPC_VERSION_MAX INT16_MAX
 
 #endif /* _RDKAFKA_PROTOCOL_H_ */