@platformatic/rdkafka 4.0.1 → 4.1.0

package/deps/librdkafka/src/rdkafka_topic.c

@@ -354,6 +354,7 @@ rd_kafka_topic_t *rd_kafka_topic_new0(rd_kafka_t *rk,
         rkt->rkt_rk    = rk;
 
         rkt->rkt_ts_create = rd_clock();
+        rkt->rkt_ts_state  = rkt->rkt_ts_create;
 
         rkt->rkt_conf = *conf;
         rd_free(conf); /* explicitly not rd_kafka_topic_destroy()
@@ -538,7 +539,7 @@ rd_kafka_topic_t *rd_kafka_topic_new(rd_kafka_t *rk,
 
         /* Query for the topic leader (async) */
         if (!existing)
-                rd_kafka_topic_leader_query(rk, rkt);
+                rd_kafka_topic_fast_leader_query(rk, rd_true /* force */);
 
         /* Drop our reference since there is already/now an app refcnt */
         rd_kafka_topic_destroy0(rkt);
@@ -565,7 +566,8 @@ static void rd_kafka_topic_set_state(rd_kafka_topic_t *rkt, int state) {
         if (rkt->rkt_state == RD_KAFKA_TOPIC_S_ERROR)
                 rkt->rkt_err = RD_KAFKA_RESP_ERR_NO_ERROR;
 
-        rkt->rkt_state = state;
+        rkt->rkt_state    = state;
+        rkt->rkt_ts_state = rd_clock();
 }
 
 /**
@@ -643,6 +645,7 @@ int rd_kafka_toppar_broker_update(rd_kafka_toppar_t *rktp,
  * @remark If a toppar is currently delegated to a preferred replica,
  *         it will not be delegated to the leader broker unless there
  *         has been a leader change.
+ * @remark The new leader, if present, should not be terminating.
  *
  * @param leader_id The id of the new leader broker.
  * @param leader A reference to the leader broker or NULL if the
@@ -663,7 +666,8 @@ static int rd_kafka_toppar_leader_update(rd_kafka_topic_t *rkt,
                                          int32_t leader_epoch) {
         rd_kafka_toppar_t *rktp;
         rd_bool_t need_epoch_validation = rd_false;
-        int r                           = 0;
+        rd_bool_t fetching_from_follower;
+        int r = 0;
 
         rktp = rd_kafka_toppar_get(rkt, partition, 0);
         if (unlikely(!rktp)) {
@@ -681,7 +685,11 @@ static int rd_kafka_toppar_leader_update(rd_kafka_topic_t *rkt,
 
         rd_kafka_toppar_lock(rktp);
 
-        if (leader_epoch < rktp->rktp_leader_epoch) {
+        /* -1 (null) is excluded to allow to switch back to a
+         * leader not supporting KIP-320 still, for example
+         * during a cluster roll for upgrading brokers to
+         * a version supporting that KIP. */
+        if (leader_epoch != -1 && leader_epoch < rktp->rktp_leader_epoch) {
                 rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "BROKER",
                              "%s [%" PRId32
                              "]: ignoring outdated metadata update with "
@@ -691,81 +699,69 @@ static int rd_kafka_toppar_leader_update(rd_kafka_topic_t *rkt,
                              rktp->rktp_rkt->rkt_topic->str,
                              rktp->rktp_partition, leader_epoch,
                              rktp->rktp_leader_epoch);
-                if (rktp->rktp_fetch_state !=
-                    RD_KAFKA_TOPPAR_FETCH_VALIDATE_EPOCH_WAIT) {
-                        rd_kafka_toppar_unlock(rktp);
-                        rd_kafka_toppar_destroy(rktp); /* from get() */
-                        return 0;
-                }
+                rd_kafka_toppar_unlock(rktp);
+                rd_kafka_toppar_destroy(rktp); /* from get() */
+                return 0;
         }
 
-        if (rktp->rktp_leader_epoch == -1 ||
-            leader_epoch > rktp->rktp_leader_epoch) {
-                rd_bool_t fetching_from_follower;
-                rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "BROKER",
-                             "%s [%" PRId32 "]: leader %" PRId32
-                             " epoch %" PRId32 " -> leader %" PRId32
-                             " epoch %" PRId32,
-                             rktp->rktp_rkt->rkt_topic->str,
-                             rktp->rktp_partition, rktp->rktp_leader_id,
-                             rktp->rktp_leader_epoch, leader_id, leader_epoch);
-                if (leader_epoch > rktp->rktp_leader_epoch)
-                        rktp->rktp_leader_epoch = leader_epoch;
+        rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "BROKER",
+                     "%s [%" PRId32 "]: leader %" PRId32 " epoch %" PRId32
+                     " -> leader %" PRId32 " epoch %" PRId32,
+                     rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition,
+                     rktp->rktp_leader_id, rktp->rktp_leader_epoch, leader_id,
+                     leader_epoch);
+
+        if (leader_epoch > rktp->rktp_leader_epoch ||
+            rktp->rktp_fetch_state ==
+                RD_KAFKA_TOPPAR_FETCH_VALIDATE_EPOCH_WAIT) {
+                /* Epoch increased and needs to be validated (leader_epoch > -1)
+                 * or we need to complete the validation. */
                 need_epoch_validation = rd_true;
+        }
 
+        rktp->rktp_leader_epoch = leader_epoch;
 
-                fetching_from_follower =
-                    leader != NULL && rktp->rktp_broker != NULL &&
-                    rktp->rktp_broker->rkb_source != RD_KAFKA_INTERNAL &&
-                    rktp->rktp_broker != leader;
-
-                if (fetching_from_follower &&
-                    rktp->rktp_leader_id == leader_id) {
-                        rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "BROKER",
-                                     "Topic %s [%" PRId32 "]: leader %" PRId32
-                                     " unchanged, "
-                                     "not migrating away from preferred "
-                                     "replica %" PRId32,
-                                     rktp->rktp_rkt->rkt_topic->str,
-                                     rktp->rktp_partition, leader_id,
-                                     rktp->rktp_broker_id);
-                        r = 0;
+        fetching_from_follower =
+            leader != NULL && rktp->rktp_broker != NULL &&
+            rktp->rktp_broker->rkb_source != RD_KAFKA_INTERNAL &&
+            rktp->rktp_broker != leader;
 
-                } else {
+        if (fetching_from_follower && rktp->rktp_leader_id == leader_id) {
+                rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "BROKER",
+                             "Topic %s [%" PRId32 "]: leader %" PRId32
+                             " unchanged, "
+                             "not migrating away from preferred "
+                             "replica %" PRId32,
+                             rktp->rktp_rkt->rkt_topic->str,
+                             rktp->rktp_partition, leader_id,
+                             rktp->rktp_broker_id);
+                r = 0;
 
-                        if (rktp->rktp_leader_id != leader_id ||
-                            rktp->rktp_leader != leader) {
-                                /* Update leader if it has changed */
-                                rktp->rktp_leader_id = leader_id;
-                                if (rktp->rktp_leader)
-                                        rd_kafka_broker_destroy(
-                                            rktp->rktp_leader);
-                                if (leader)
-                                        rd_kafka_broker_keep(leader);
-                                rktp->rktp_leader = leader;
-                        }
+        } else {
 
-                        /* Update handling broker */
-                        r = rd_kafka_toppar_broker_update(
-                            rktp, leader_id, leader, "leader updated");
+                if (rktp->rktp_leader_id != leader_id ||
+                    rktp->rktp_leader != leader) {
+                        /* Update leader if it has changed */
+                        rktp->rktp_leader_id = leader_id;
+                        if (rktp->rktp_leader)
+                                rd_kafka_broker_destroy(rktp->rktp_leader);
+                        if (leader)
+                                rd_kafka_broker_keep(leader);
+                        rktp->rktp_leader = leader;
                 }
 
-        } else if (rktp->rktp_fetch_state ==
-                   RD_KAFKA_TOPPAR_FETCH_VALIDATE_EPOCH_WAIT)
-                need_epoch_validation = rd_true;
+                /* Update handling broker */
+                r = rd_kafka_toppar_broker_update(rktp, leader_id, leader,
+                                                  "leader updated");
+        }
 
         if (need_epoch_validation) {
                 /* Set offset validation position,
                  * depending it if should continue with current position or
                  * with next fetch start position. */
-                if (rd_kafka_toppar_fetch_decide_start_from_next_fetch_start(
-                        rktp)) {
-                        rd_kafka_toppar_set_offset_validation_position(
-                            rktp, rktp->rktp_next_fetch_start);
-                } else {
-                        rd_kafka_toppar_set_offset_validation_position(
-                            rktp, rktp->rktp_offsets.fetch_pos);
-                }
+                rd_kafka_toppar_set_offset_validation_position(
+                    rktp,
+                    rd_kafka_toppar_fetch_decide_next_fetch_start_pos(rktp));
                 rd_kafka_offset_validate(rktp, "epoch updated from metadata");
         }
 
@@ -794,7 +790,12 @@ int rd_kafka_toppar_delegate_to_leader(rd_kafka_toppar_t *rktp) {
         rd_kafka_rdlock(rktp->rktp_rkt->rkt_rk);
         rd_kafka_toppar_lock(rktp);
 
-        rd_assert(rktp->rktp_leader_id != rktp->rktp_broker_id);
+        if (rktp->rktp_leader_id == rktp->rktp_broker_id) {
+                /* Given lock was released we need to check again */
+                rd_kafka_toppar_unlock(rktp);
+                rd_kafka_rdunlock(rktp->rktp_rkt->rkt_rk);
+                return 0;
+        }
 
         rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "BROKER",
                      "Topic %s [%" PRId32
@@ -822,6 +823,45 @@ int rd_kafka_toppar_delegate_to_leader(rd_kafka_toppar_t *rktp) {
 }
 
 
+/**
+ * @brief Forgets current rktp leader, to reduce reference count
+ *        and allow the broker to be destroyed.
+ *
+ * @locks none
+ * @locks_acquired rk rdlock, rktp
+ * @locality any
+ */
+void rd_kafka_toppar_forget_leader(rd_kafka_toppar_t *rktp) {
+        rd_kafka_rdlock(rktp->rktp_rkt->rkt_rk);
+        rd_kafka_toppar_lock(rktp);
+
+        if (rktp->rktp_leader) {
+                rd_kafka_broker_destroy(rktp->rktp_leader);
+                rktp->rktp_leader       = NULL;
+                rktp->rktp_leader_id    = -1;
+                rktp->rktp_leader_epoch = -1;
+        }
+
+        rd_kafka_toppar_unlock(rktp);
+        rd_kafka_rdunlock(rktp->rktp_rkt->rkt_rk);
+}
+
+/**
+ * @brief Revert the topic+partition delegation to the internal broker.
+ *
+ * @locks none
+ * @locks_acquired rk rdlock, rktp
+ * @locality any
+ */
+void rd_kafka_toppar_undelegate(rd_kafka_toppar_t *rktp) {
+        rd_kafka_rdlock(rktp->rktp_rkt->rkt_rk);
+        rd_kafka_toppar_lock(rktp);
+
+        rd_kafka_toppar_broker_delegate(rktp, NULL);
+
+        rd_kafka_toppar_unlock(rktp);
+        rd_kafka_rdunlock(rktp->rktp_rkt->rkt_rk);
+}
 
 /**
  * @brief Save idempotent producer state for a partition that is about to
@@ -1188,11 +1228,14 @@ rd_bool_t rd_kafka_topic_set_notexists(rd_kafka_topic_t *rkt,
         rd_assert(err != RD_KAFKA_RESP_ERR_NO_ERROR);
 
         remains_us =
-            (rkt->rkt_ts_create +
+            (rkt->rkt_ts_state +
              (rkt->rkt_rk->rk_conf.metadata_propagation_max_ms * 1000)) -
             rkt->rkt_ts_metadata;
 
-        if (!permanent && rkt->rkt_state == RD_KAFKA_TOPIC_S_UNKNOWN &&
+        if (!permanent &&
+            (rkt->rkt_state == RD_KAFKA_TOPIC_S_UNKNOWN ||
+             rkt->rkt_state == RD_KAFKA_TOPIC_S_ERROR ||
+             rkt->rkt_state == RD_KAFKA_TOPIC_S_EXISTS) &&
             remains_us > 0) {
                 /* Still allowing topic metadata to propagate. */
                 rd_kafka_dbg(
@@ -1220,6 +1263,47 @@ rd_bool_t rd_kafka_topic_set_notexists(rd_kafka_topic_t *rkt,
         return rd_true;
 }
 
+/**
+ * @brief Mark topic as existent, unless metadata propagation configuration
+ *        disallows it.
+ *
+ * @returns true if the topic was marked as existent, else false.
+ *
+ * @locks topic_wrlock() MUST be held.
+ */
+rd_bool_t rd_kafka_topic_set_exists(rd_kafka_topic_t *rkt,
+                                    rd_kafka_Uuid_t topic_id) {
+        rd_ts_t remains_us;
+
+        if (unlikely(rd_kafka_terminating(rkt->rkt_rk))) {
+                /* Dont update metadata while terminating. */
+                return rd_false;
+        }
+
+        remains_us =
+            (rkt->rkt_ts_state +
+             (rkt->rkt_rk->rk_conf.metadata_propagation_max_ms * 1000)) -
+            rkt->rkt_ts_metadata;
+
+        if (/* Same topic id */
+            rd_kafka_Uuid_cmp(rkt->rkt_topic_id, topic_id) == 0 &&
+            rkt->rkt_state == RD_KAFKA_TOPIC_S_NOTEXISTS && remains_us > 0) {
+                /* Still allowing topic metadata to propagate. */
+                rd_kafka_dbg(
+                    rkt->rkt_rk, TOPIC | RD_KAFKA_DBG_METADATA, "TOPICPROP",
+                    "Topic %.*s exists after being deleted, "
+                    " allowing %dms for metadata propagation before marking "
+                    "topic "
+                    "as existent",
+                    RD_KAFKAP_STR_PR(rkt->rkt_topic), (int)(remains_us / 1000));
+                return rd_false;
+        }
+
+        rd_kafka_topic_set_state(rkt, RD_KAFKA_TOPIC_S_EXISTS);
+
+        return rd_true;
+}
+
 /**
  * @brief Mark topic as errored, such as when topic authorization fails.
  *
@@ -1327,18 +1411,24 @@ rd_kafka_topic_metadata_update(rd_kafka_topic_t *rkt,
             mdt->err == RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART ||
             mdt->err == RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID)
                 rd_kafka_topic_set_notexists(rkt, mdt->err);
-        else if (mdt->partition_cnt > 0)
-                rd_kafka_topic_set_state(rkt, RD_KAFKA_TOPIC_S_EXISTS);
-        else if (mdt->err)
+        else if (mdt->err == RD_KAFKA_RESP_ERR_NO_ERROR &&
+                 mdt->partition_cnt > 0)
+                rd_kafka_topic_set_exists(rkt, mdit->topic_id);
+        else if (mdt->err == RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED)
+                /* Only set an error when it's permanent and it needs
+                 * to be surfaced to the application. */
                 rd_kafka_topic_set_error(rkt, mdt->err);
 
         /* Update number of partitions, but not if there are
          * (possibly intermittent) errors (e.g., "Leader not available"). */
         if (mdt->err == RD_KAFKA_RESP_ERR_NO_ERROR) {
-                upd += rd_kafka_topic_partition_cnt_update(rkt,
-                                                           mdt->partition_cnt);
-                if (rd_kafka_Uuid_cmp(mdit->topic_id, RD_KAFKA_UUID_ZERO) &&
-                    rd_kafka_Uuid_cmp(mdit->topic_id, rkt->rkt_topic_id)) {
+                rd_bool_t different_topic_id =
+                    rd_kafka_Uuid_cmp(mdit->topic_id, rkt->rkt_topic_id) != 0;
+                if (different_topic_id ||
+                    mdt->partition_cnt > rkt->rkt_partition_cnt)
+                        upd += rd_kafka_topic_partition_cnt_update(
+                            rkt, mdt->partition_cnt);
+                if (different_topic_id) {
                         /* FIXME: an offset reset must be triggered.
                          * when rkt_topic_id wasn't zero.
                          * There are no problems
@@ -1364,71 +1454,63 @@ rd_kafka_topic_metadata_update(rd_kafka_topic_t *rkt,
                  * Issue #1985. */
                 if (old_state == RD_KAFKA_TOPIC_S_UNKNOWN)
                         upd++;
-        }
 
-        /* Update leader for each partition */
-        for (j = 0; j < mdt->partition_cnt; j++) {
-                int r = 0;
-                rd_kafka_broker_t *leader;
-                int32_t leader_epoch = mdit->partitions[j].leader_epoch;
-                rd_kafka_toppar_t *rktp =
-                    rd_kafka_toppar_get(rkt, mdt->partitions[j].id, 0);
+                /* Update leader for each partition
+                 * only when topic response has no errors. */
+                for (j = 0;
+                     j < mdt->partition_cnt && j < rkt->rkt_partition_cnt;
+                     j++) {
+                        int r = 0;
+                        rd_kafka_broker_t *leader;
+                        int32_t leader_epoch = mdit->partitions[j].leader_epoch;
+                        rd_kafka_toppar_t *rktp =
+                            rd_kafka_toppar_get(rkt, mdt->partitions[j].id, 0);
+
+                        rd_kafka_dbg(rk, TOPIC | RD_KAFKA_DBG_METADATA,
+                                     "METADATA",
+                                     "Topic %s [%" PRId32 "] Leader %" PRId32
+                                     " Epoch %" PRId32,
+                                     rkt->rkt_topic->str, mdt->partitions[j].id,
+                                     mdt->partitions[j].leader, leader_epoch);
+
+                        leader         = partbrokers[j];
+                        partbrokers[j] = NULL;
 
-                rd_kafka_dbg(rk, TOPIC | RD_KAFKA_DBG_METADATA, "METADATA",
-                             "Topic %s [%" PRId32 "] Leader %" PRId32
-                             " Epoch %" PRId32,
-                             rkt->rkt_topic->str, mdt->partitions[j].id,
-                             mdt->partitions[j].leader, leader_epoch);
-
-                leader         = partbrokers[j];
-                partbrokers[j] = NULL;
-
-                /* If broker does not support leaderEpoch(KIP 320) then it is
-                 * set to -1, we assume that metadata is not stale. */
-                if (leader_epoch == -1)
-                        partition_exists_with_no_leader_epoch = rd_true;
-                else if (leader_epoch < rktp->rktp_leader_epoch)
-                        partition_exists_with_stale_leader_epoch = rd_true;
-
-
-                /* Update leader for partition */
-                r = rd_kafka_toppar_leader_update(rkt, mdt->partitions[j].id,
-                                                  mdt->partitions[j].leader,
-                                                  leader, leader_epoch);
-
-                upd += (r != 0 ? 1 : 0);
-
-                if (leader) {
-                        if (r != -1)
-                                leader_cnt++;
-                        /* Drop reference to broker (from find()) */
-                        rd_kafka_broker_destroy(leader);
+                        /* If broker does not support leaderEpoch(KIP 320) then
+                         * it is set to -1, we assume that metadata is not
+                         * stale. */
+                        if (leader_epoch == -1)
+                                partition_exists_with_no_leader_epoch = rd_true;
+                        else if (rktp && leader_epoch < rktp->rktp_leader_epoch)
+                                partition_exists_with_stale_leader_epoch =
+                                    rd_true;
+
+
+                        /* Update leader for partition */
+                        r = rd_kafka_toppar_leader_update(
+                            rkt, mdt->partitions[j].id,
+                            mdt->partitions[j].leader, leader, leader_epoch);
+
+                        upd += (r != 0 ? 1 : 0);
+
+                        if (leader) {
+                                if (r != -1)
+                                        leader_cnt++;
+                                /* Drop reference to broker (from find()) */
+                                rd_kafka_broker_destroy(leader);
+                        }
+                        RD_IF_FREE(rktp, rd_kafka_toppar_destroy);
                 }
-                RD_IF_FREE(rktp, rd_kafka_toppar_destroy);
         }
 
         /* If all partitions have leaders, and this metadata update was not
          * stale, we can turn off fast leader query. */
-        if (mdt->partition_cnt > 0 && leader_cnt == mdt->partition_cnt &&
+        if (rkt->rkt_partition_cnt > 0 &&
+            leader_cnt == rkt->rkt_partition_cnt &&
             (partition_exists_with_no_leader_epoch ||
              !partition_exists_with_stale_leader_epoch))
                 rkt->rkt_flags &= ~RD_KAFKA_TOPIC_F_LEADER_UNAVAIL;
 
-        if (mdt->err != RD_KAFKA_RESP_ERR_NO_ERROR && rkt->rkt_partition_cnt) {
-                /* (Possibly intermittent) topic-wide error:
-                 * remove leaders for partitions */
-
-                for (j = 0; j < rkt->rkt_partition_cnt; j++) {
-                        rd_kafka_toppar_t *rktp;
-                        if (!rkt->rkt_p[j])
-                                continue;
-
-                        rktp = rkt->rkt_p[j];
-                        rd_kafka_toppar_lock(rktp);
-                        rd_kafka_toppar_broker_delegate(rktp, NULL);
-                        rd_kafka_toppar_unlock(rktp);
-                }
-        }
 
         /* If there was an update to the partitions try to assign
          * unassigned messages to new partitions, or fail them */
@@ -1647,8 +1729,8 @@ void rd_kafka_topic_scan_all(rd_kafka_t *rk, rd_ts_t now) {
 
                 /* Check if metadata information has timed out. */
                 if (rkt->rkt_state != RD_KAFKA_TOPIC_S_UNKNOWN &&
-                    !rd_kafka_metadata_cache_topic_get(rk, rkt->rkt_topic->str,
-                                                       1 /*only valid*/)) {
+                    !rd_kafka_metadata_cache_topic_get(
+                        rk, rkt->rkt_topic->str, NULL, 1 /*only valid*/)) {
                         rd_kafka_dbg(rk, TOPIC, "NOINFO",
                                      "Topic %s metadata information timed out "
                                      "(%" PRId64 "ms old)",
@@ -1766,7 +1848,8 @@ void rd_kafka_topic_scan_all(rd_kafka_t *rk, rd_ts_t now) {
                                                       * info exists*/
                     ,
                     rk->rk_conf.allow_auto_create_topics,
-                    rd_false /*!cgrp_update*/, "refresh unavailable topics");
+                    rd_false /*!cgrp_update*/, -1,
+                    "refresh unavailable topics");
         rd_list_destroy(&query_topics);
 }
 
@@ -1965,7 +2048,7 @@ rd_kafka_topic_info_t *rd_kafka_topic_info_new_with_rack(
 /**
  * Destroy/free topic_info
  */
-void rd_kafka_topic_info_destroy(rd_kafka_topic_info_t *ti) {
+void rd_kafka_topic_info_destroy_free(void *ti) {
         rd_free(ti);
 }
 
@@ -2013,7 +2096,7 @@ void rd_kafka_topic_leader_query0(rd_kafka_t *rk,
 
         rd_kafka_metadata_refresh_topics(
             rk, NULL, &topics, force, rk->rk_conf.allow_auto_create_topics,
-            rd_false /*!cgrp_update*/, "leader query");
+            rd_false /*!cgrp_update*/, -1, "leader query");
 
         rd_list_destroy(&topics);
 }
@@ -2039,7 +2122,7 @@ void rd_kafka_local_topics_to_list(rd_kafka_t *rk,
         rd_list_grow(topics, rk->rk_topic_cnt);
         TAILQ_FOREACH(rkt, &rk->rk_topics, rkt_link)
         rd_list_add(topics, rd_strdup(rkt->rkt_topic->str));
-        cache_cnt = rd_kafka_metadata_cache_topics_to_list(rk, topics);
+        cache_cnt = rd_kafka_metadata_cache_topics_to_list(rk, topics, rd_true);
         if (cache_cntp)
                 *cache_cntp = cache_cnt;
         rd_kafka_rdunlock(rk);
@@ -2056,8 +2139,8 @@ void rd_ut_kafka_topic_set_topic_exists(rd_kafka_topic_t *rkt,
         rd_kafka_metadata_partition_internal_t *partitions =
             rd_calloc(partition_cnt, sizeof(*partitions));
         struct rd_kafka_metadata_topic mdt      = {.topic =
-                                                  (char *)rkt->rkt_topic->str,
-                                              .partition_cnt = partition_cnt};
+                                                       (char *)rkt->rkt_topic->str,
+                                                   .partition_cnt = partition_cnt};
         rd_kafka_metadata_topic_internal_t mdit = {.partitions = partitions};
         int i;
 
@@ -2071,7 +2154,7 @@ void rd_ut_kafka_topic_set_topic_exists(rd_kafka_topic_t *rkt,
 
         rd_kafka_wrlock(rkt->rkt_rk);
         rd_kafka_metadata_cache_topic_update(rkt->rkt_rk, &mdt, &mdit, rd_true,
-                                             rd_false, NULL, 0, rd_false);
+                                             rd_false, rd_true);
         rd_kafka_topic_metadata_update(rkt, &mdt, &mdit, rd_clock());
         rd_kafka_wrunlock(rkt->rkt_rk);
         rd_free(partitions);

package/deps/librdkafka/src/rdkafka_topic.h

@@ -159,12 +159,14 @@ struct rd_kafka_topic_s {
         rd_refcnt_t rkt_app_refcnt; /**< Number of active rkt's new()ed
                                      *   by application. */
 
-        enum { RD_KAFKA_TOPIC_S_UNKNOWN,   /* No cluster information yet */
-               RD_KAFKA_TOPIC_S_EXISTS,    /* Topic exists in cluster */
-               RD_KAFKA_TOPIC_S_NOTEXISTS, /* Topic is not known in cluster */
-               RD_KAFKA_TOPIC_S_ERROR,     /* Topic exists but is in an errored
-                                            * state, such as auth failure. */
+        enum {
+                RD_KAFKA_TOPIC_S_UNKNOWN,   /* No cluster information yet */
+                RD_KAFKA_TOPIC_S_EXISTS,    /* Topic exists in cluster */
+                RD_KAFKA_TOPIC_S_NOTEXISTS, /* Topic is not known in cluster */
+                RD_KAFKA_TOPIC_S_ERROR,     /* Topic exists but is in an errored
+                                             * state, such as auth failure. */
         } rkt_state;
+        rd_ts_t rkt_ts_state; /**< State change time. */
 
         int rkt_flags;
 #define RD_KAFKA_TOPIC_F_LEADER_UNAVAIL                                        \
@@ -289,7 +291,7 @@ rd_kafka_topic_info_t *rd_kafka_topic_info_new_with_rack(
     const char *topic,
     int partition_cnt,
     const rd_kafka_metadata_partition_internal_t *mdpi);
-void rd_kafka_topic_info_destroy(rd_kafka_topic_info_t *ti);
+void rd_kafka_topic_info_destroy_free(void *ti);
 
 int rd_kafka_topic_match(rd_kafka_t *rk,
                          const char *pattern,
@@ -302,6 +304,10 @@ int rd_kafka_toppar_broker_update(rd_kafka_toppar_t *rktp,
 
 int rd_kafka_toppar_delegate_to_leader(rd_kafka_toppar_t *rktp);
 
+void rd_kafka_toppar_undelegate(rd_kafka_toppar_t *rktp);
+
+void rd_kafka_toppar_forget_leader(rd_kafka_toppar_t *rktp);
+
 rd_kafka_resp_err_t rd_kafka_topics_leader_query_sync(rd_kafka_t *rk,
                                                       int all_topics,
                                                       const rd_list_t *topics,
@@ -314,8 +320,8 @@ void rd_kafka_topic_leader_query0(rd_kafka_t *rk,
         rd_kafka_topic_leader_query0(rk, rkt, 1 /*lock*/,                      \
                                      rd_false /*dont force*/)
 
-#define rd_kafka_topic_fast_leader_query(rk)                                   \
-        rd_kafka_metadata_fast_leader_query(rk)
+#define rd_kafka_topic_fast_leader_query(rk, force)                            \
+        rd_kafka_metadata_fast_leader_query(rk, force)
 
 void rd_kafka_local_topics_to_list(rd_kafka_t *rk,
                                    rd_list_t *topics,