@platformatic/rdkafka 4.0.1 → 4.1.0-alpha.1

package/deps/librdkafka/src/rdkafka_int.h

@@ -290,17 +290,29 @@ struct rd_kafka_s {
         TAILQ_HEAD(, rd_kafka_broker_s) rk_brokers;
         rd_list_t rk_broker_by_id; /* Fast id lookups. */
         rd_atomic32_t rk_broker_cnt;
-        /**< Number of brokers in state >= UP */
+        /**  Logical brokers count.
+         *   Used for calculating ERR__ALL_BROKERS_DOWN. */
+        rd_atomic32_t rk_logical_broker_cnt;
+        /**  Number of configured or learned brokers in state >= UP */
         rd_atomic32_t rk_broker_up_cnt;
-        /**< Number of logical brokers in state >= UP, this is a sub-set
-         *   of rk_broker_up_cnt. */
-        rd_atomic32_t rk_logical_broker_up_cnt;
-        /**< Number of brokers that are down, only includes brokers
-         *   that have had at least one connection attempt. */
+        /**  Number of brokers that are down, only includes brokers
+         *   that have had at least one connection attempt
+         *   and are configured or learned. */
         rd_atomic32_t rk_broker_down_cnt;
-        /**< Logical brokers currently without an address.
-         *   Used for calculating ERR__ALL_BROKERS_DOWN. */
-        rd_atomic32_t rk_broker_addrless_cnt;
+        /**  Set to 1 when there's a re-bootstrap in progress.
+         *   Set to 0 when the re-bootstrap is done.
+         *   Accessed from the main thread and the broker threads. */
+        rd_atomic32_t rk_rebootstrap_in_progress;
+
+        /**< Additional bootstrap servers list.
+         *   contains all brokers added through rd_kafka_brokers_add().
+         *   Doesn't contain the initially configured bootstrap brokers. */
+        rd_list_t additional_brokerlists;
+
+        /** Decommissioned threads to await */
+        rd_list_t wait_decommissioned_thrds;
+        /** Decommissioned brokers to await */
+        rd_list_t wait_decommissioned_brokers;
 
         mtx_t rk_internal_rkb_lock;
         rd_kafka_broker_t *rk_internal_rkb;
@@ -384,16 +396,7 @@ struct rd_kafka_s {
                                         *   to avoid reaching
                                         * max.poll.interval.ms. during that time
                                         * frame. Only relevant for consumer. */
-        rd_ts_t rk_ts_last_poll_start; /**< Timestamp of last application
-                                        *   consumer_poll() call start
-                                        *   Only relevant for consumer.
-                                        *   Not an atomic as Kafka consumer
-                                        *   isn't thread safe. */
-        rd_ts_t rk_ts_last_poll_end;   /**< Timestamp of last application
-                                        *   consumer_poll() call end
-                                        *   Only relevant for consumer.
-                                        *   Not an atomic as Kafka consumer
-                                        *   isn't thread safe. */
+
         /* First fatal error. */
         struct {
                 rd_atomic32_t err; /**< rd_kafka_resp_err_t */
@@ -409,18 +412,20 @@ struct rd_kafka_s {
         rd_ts_t rk_ts_metadata; /* Timestamp of most recent
                                  * metadata. */
 
-        rd_kafka_metadata_internal_t
-            *rk_full_metadata;       /* Last full metadata. */
-        rd_ts_t rk_ts_full_metadata; /* Timestamp of .. */
+        rd_ts_t rk_ts_full_metadata;                      /* Timestamp of most
+                                                           * recent full
+                                                           * metadata */
         struct rd_kafka_metadata_cache rk_metadata_cache; /* Metadata cache */
 
         char *rk_clusterid;      /* ClusterId from metadata */
         int32_t rk_controllerid; /* ControllerId from metadata */
 
         /**< Producer: Delivery report mode */
-        enum { RD_KAFKA_DR_MODE_NONE,  /**< No delivery reports */
-               RD_KAFKA_DR_MODE_CB,    /**< Delivery reports through callback */
-               RD_KAFKA_DR_MODE_EVENT, /**< Delivery reports through event API*/
+        enum {
+                RD_KAFKA_DR_MODE_NONE, /**< No delivery reports */
+                RD_KAFKA_DR_MODE_CB,   /**< Delivery reports through callback */
+                RD_KAFKA_DR_MODE_EVENT, /**< Delivery reports through event
+                                           API*/
         } rk_drmode;
 
         /* Simple consumer count:
@@ -608,6 +613,16 @@ struct rd_kafka_s {
         } rk_curr_msgs;
 
         rd_kafka_timers_t rk_timers;
+
+        /** Metadata refresh timer */
+        rd_kafka_timer_t metadata_refresh_tmr;
+        /** 1s interval timer */
+        rd_kafka_timer_t one_s_tmr;
+        /** Rebootstrap timer.
+         *  Will add bootstrap brokers again
+         *  when it's fired. */
+        rd_kafka_timer_t rebootstrap_tmr;
+
         thrd_t rk_thread;
 
         int rk_initialized; /**< Will be > 0 when the rd_kafka_t
@@ -652,6 +667,12 @@ struct rd_kafka_s {
                  *   Use 10 < reconnect.backoff.jitter.ms / 2 < 1000.
                  */
                 rd_interval_t sparse_connect_random;
+
+                /**  Sparse connection timer: fires after remaining time of
+                 *   `sparse_connect_random` interval + 1ms.
+                 */
+                rd_kafka_timer_t sparse_connect_random_tmr;
+
                 /**< Lock for sparse_connect_random */
                 mtx_t sparse_connect_lock;
 
@@ -877,15 +898,13 @@ rd_kafka_curr_msgs_wait_zero(rd_kafka_t *rk,
                              int timeout_ms,
                              unsigned int *curr_msgsp) {
         unsigned int cnt;
-        struct timespec tspec;
-
-        rd_timeout_init_timespec(&tspec, timeout_ms);
+        rd_ts_t abs_timeout = rd_timeout_init(timeout_ms);
 
         mtx_lock(&rk->rk_curr_msgs.lock);
         while ((cnt = rk->rk_curr_msgs.cnt) > 0) {
                 if (cnd_timedwait_abs(&rk->rk_curr_msgs.cnd,
                                       &rk->rk_curr_msgs.lock,
-                                      &tspec) == thrd_timedout)
+                                      abs_timeout) == thrd_timedout)
                         break;
         }
         mtx_unlock(&rk->rk_curr_msgs.lock);
@@ -894,6 +913,9 @@ rd_kafka_curr_msgs_wait_zero(rd_kafka_t *rk,
         return cnt == 0;
 }
 
+void rd_kafka_decommissioned_broker_thread_join(rd_kafka_t *rk,
+                                                void *rkb_decommissioned);
+
 void rd_kafka_destroy_final(rd_kafka_t *rk);
 
 void rd_kafka_global_init(void);
@@ -1070,6 +1092,14 @@ int rd_kafka_set_fatal_error0(rd_kafka_t *rk,
 
 rd_kafka_error_t *rd_kafka_get_fatal_error(rd_kafka_t *rk);
 
+#define rd_kafka_producer_can_have_fatal_errors(rk)                            \
+        (rk->rk_type == RD_KAFKA_PRODUCER && rk->rk_conf.eos.idempotence)
+
+#define rd_kafka_consumer_can_have_fatal_errors(rk)                            \
+        (rk->rk_type == RD_KAFKA_CONSUMER &&                                   \
+         (rk->rk_conf.group_instance_id ||                                     \
+          rk->rk_conf.group_protocol == RD_KAFKA_GROUP_PROTOCOL_CONSUMER))
+
 static RD_INLINE RD_UNUSED rd_kafka_resp_err_t
 rd_kafka_fatal_error_code(rd_kafka_t *rk) {
         /* This is an optimization to avoid an atomic read which are costly
@@ -1079,10 +1109,8 @@ rd_kafka_fatal_error_code(rd_kafka_t *rk) {
          * 2) static consumers (group.instance.id)
          * 3) Group using consumer protocol (Introduced in KIP-848). See exact
          *    errors in rd_kafka_cgrp_handle_ConsumerGroupHeartbeat() */
-        if ((rk->rk_type == RD_KAFKA_PRODUCER && rk->rk_conf.eos.idempotence) ||
-            (rk->rk_type == RD_KAFKA_CONSUMER &&
-             (rk->rk_conf.group_instance_id ||
-              rk->rk_conf.group_protocol == RD_KAFKA_GROUP_PROTOCOL_CONSUMER)))
+        if (rd_kafka_producer_can_have_fatal_errors(rk) ||
+            rd_kafka_consumer_can_have_fatal_errors(rk))
                 return rd_atomic32_get(&rk->rk_fatal.err);
 
         return RD_KAFKA_RESP_ERR_NO_ERROR;
@@ -1152,8 +1180,10 @@ static RD_INLINE RD_UNUSED int rd_kafka_max_poll_exceeded(rd_kafka_t *rk) {
  * @locality any
  * @locks none
  */
-static RD_INLINE RD_UNUSED void
-rd_kafka_app_poll_start(rd_kafka_t *rk, rd_ts_t now, rd_bool_t is_blocking) {
+static RD_INLINE RD_UNUSED void rd_kafka_app_poll_start(rd_kafka_t *rk,
+                                                        rd_kafka_q_t *rkq,
+                                                        rd_ts_t now,
+                                                        rd_bool_t is_blocking) {
         if (rk->rk_type != RD_KAFKA_CONSUMER)
                 return;
 
@@ -1161,20 +1191,20 @@ rd_kafka_app_poll_start(rd_kafka_t *rk, rd_ts_t now, rd_bool_t is_blocking) {
                 now = rd_clock();
         if (is_blocking)
                 rd_atomic64_set(&rk->rk_ts_last_poll, INT64_MAX);
-        if (rk->rk_ts_last_poll_end) {
+        if (rkq->rkq_ts_last_poll_end) {
                 int64_t poll_idle_ratio = 0;
-                rd_ts_t poll_interval   = now - rk->rk_ts_last_poll_start;
+                rd_ts_t poll_interval   = now - rkq->rkq_ts_last_poll_start;
                 if (poll_interval) {
-                        rd_ts_t idle_interval =
-                            rk->rk_ts_last_poll_end - rk->rk_ts_last_poll_start;
+                        rd_ts_t idle_interval = rkq->rkq_ts_last_poll_end -
+                                                rkq->rkq_ts_last_poll_start;
                         poll_idle_ratio =
                             idle_interval * 1000000 / poll_interval;
                 }
                 rd_avg_add(
                     &rk->rk_telemetry.rd_avg_current.rk_avg_poll_idle_ratio,
                     poll_idle_ratio);
-                rk->rk_ts_last_poll_start = now;
-                rk->rk_ts_last_poll_end   = 0;
+                rkq->rkq_ts_last_poll_start = now;
+                rkq->rkq_ts_last_poll_end   = 0;
         }
 }
 
@@ -1186,7 +1216,8 @@ rd_kafka_app_poll_start(rd_kafka_t *rk, rd_ts_t now, rd_bool_t is_blocking) {
  * @locality any
  * @locks none
  */
-static RD_INLINE RD_UNUSED void rd_kafka_app_polled(rd_kafka_t *rk) {
+static RD_INLINE RD_UNUSED void rd_kafka_app_polled(rd_kafka_t *rk,
+                                                    rd_kafka_q_t *rkq) {
         if (rk->rk_type == RD_KAFKA_CONSUMER) {
                 rd_ts_t now = rd_clock();
                 rd_atomic64_set(&rk->rk_ts_last_poll, now);
@@ -1199,10 +1230,10 @@ static RD_INLINE RD_UNUSED void rd_kafka_app_polled(rd_kafka_t *rk) {
                             rk->rk_cgrp,
                             "app polled after poll interval exceeded");
                 }
-                if (!rk->rk_ts_last_poll_end)
-                        rk->rk_ts_last_poll_end = now;
-                rd_dassert(rk->rk_ts_last_poll_end >=
-                           rk->rk_ts_last_poll_start);
+                if (!rkq->rkq_ts_last_poll_end)
+                        rkq->rkq_ts_last_poll_end = now;
+                rd_dassert(rkq->rkq_ts_last_poll_end >=
+                           rkq->rkq_ts_last_poll_start);
         }
 }
 
@@ -1218,5 +1249,12 @@ rd_kafka_resp_err_t rd_kafka_background_thread_create(rd_kafka_t *rk,
                                                       char *errstr,
                                                       size_t errstr_size);
 
+void rd_kafka_rebootstrap(rd_kafka_t *rk);
+
+void rd_kafka_rebootstrap_tmr_start_maybe(rd_kafka_t *rk);
+
+int rd_kafka_rebootstrap_tmr_stop(rd_kafka_t *rk);
+
+void rd_kafka_reset_any_broker_down_reported(rd_kafka_t *rk);
 
 #endif /* _RDKAFKA_INT_H_ */