@platformatic/rdkafka 4.0.1 → 4.1.0-alpha.1

package/deps/librdkafka/src/rdkafka_broker.h

@@ -51,7 +51,6 @@ typedef enum {
         /* Any state >= STATE_UP means the Kafka protocol layer
          * is operational (to some degree). */
         RD_KAFKA_BROKER_STATE_UP,
-        RD_KAFKA_BROKER_STATE_UPDATE,
         RD_KAFKA_BROKER_STATE_APIVERSION_QUERY,
         RD_KAFKA_BROKER_STATE_AUTH_HANDSHAKE,
         RD_KAFKA_BROKER_STATE_AUTH_REQ,
@@ -82,8 +81,7 @@ typedef struct rd_kafka_broker_monitor_s {
 struct rd_kafka_broker_s { /* rd_kafka_broker_t */
         TAILQ_ENTRY(rd_kafka_broker_s) rkb_link;
 
-        int32_t rkb_nodeid; /**< Broker Node Id.
-                             *   @locks rkb_lock */
+        int32_t rkb_nodeid; /**< Broker Node Id, read only. */
 #define RD_KAFKA_NODEID_UA -1
 
         rd_sockaddr_list_t *rkb_rsal;
@@ -189,8 +187,27 @@ struct rd_kafka_broker_s { /* rd_kafka_broker_t */
                 rd_atomic64_t reqtype[RD_KAFKAP__NUM]; /**< Per request-type
                                                         *   counter */
 
-                rd_atomic64_t ts_send; /**< Timestamp of last send */
-                rd_atomic64_t ts_recv; /**< Timestamp of last receive */
+                rd_atomic64_t ts_send;       /**< Timestamp of last send */
+                rd_atomic64_t ts_recv;       /**< Timestamp of last receive */
+                rd_bool_t skip_broker_down;  /**< Avoid reporting the
+                                              *   broker down on next
+                                              *   state change.
+                                              *   Useful for a planned
+                                              *   disconnection to avoid
+                                              *   reaching the all
+                                              *   brokers down state. */
+                int connections_max_idle_ms; /**< Maximum idle time
+                                              *   for this broker connections.
+                                              *   jitter is different for
+                                              *   each broker to avoid the
+                                              *   ALL_BROKERS_DOWN error.
+                                              *   Initial value is `-1` as
+                                              *   `connections.max.idle.ms=0`
+                                              *   means disabled, but must
+                                              *   never be used when
+                                              *   `connections.max.idle.ms=0`
+                                              *   for the same reason
+                                              *   (an assert is present). */
         } rkb_c;
 
         struct {
@@ -333,7 +350,9 @@ struct rd_kafka_broker_s { /* rd_kafka_broker_t */
 
         rd_kafka_secproto_t rkb_proto;
 
-        int rkb_down_reported; /* Down event reported */
+        /** Down event was reported for this broker
+         *  after last connection to any broker. */
+        rd_atomic32_t rkb_down_reported;
 #if WITH_SASL_CYRUS
         rd_kafka_timer_t rkb_sasl_kinit_refresh_tmr;
 #endif
@@ -365,6 +384,9 @@ struct rd_kafka_broker_s { /* rd_kafka_broker_t */
 
 
         rd_kafka_timer_t rkb_sasl_reauth_tmr;
+
+        /** > 0 if this broker thread is terminating */
+        rd_atomic32_t termination_in_progress;
 };
 
 #define rd_kafka_broker_keep(rkb) rd_refcnt_add(&(rkb)->rkb_refcnt)
@@ -392,12 +414,28 @@ rd_kafka_broker_get_state(rd_kafka_broker_t *rkb) {
 
 
 /**
- * @returns true if the broker state is UP or UPDATE
+ * @returns true if the broker state is UP
  */
-#define rd_kafka_broker_state_is_up(state)                                     \
-        ((state) == RD_KAFKA_BROKER_STATE_UP ||                                \
-         (state) == RD_KAFKA_BROKER_STATE_UPDATE)
+#define rd_kafka_broker_state_is_up(state) ((state) == RD_KAFKA_BROKER_STATE_UP)
 
+/**
+ * @returns true if the broker state is DOWN
+ */
+#define rd_kafka_broker_state_is_down(state)                                   \
+        ((state) == RD_KAFKA_BROKER_STATE_DOWN)
+
+/**
+ * @returns true if the error is a broker destroy error, because of
+ *          termination or because of decommissioning.
+ */
+#define rd_kafka_broker_is_any_err_destroy(err)                                \
+        ((err) == RD_KAFKA_RESP_ERR__DESTROY ||                                \
+         (err) == RD_KAFKA_RESP_ERR__DESTROY_BROKER)
+
+
+#define rd_kafka_broker_or_instance_terminating(rkb)                           \
+        (rd_kafka_broker_termination_in_progress(rkb) ||                       \
+         rd_kafka_terminating((rkb)->rkb_rk))
 
 /**
  * @returns true if the broker connection is up, else false.
@@ -410,6 +448,14 @@ rd_kafka_broker_is_up(rd_kafka_broker_t *rkb) {
         return rd_kafka_broker_state_is_up(state);
 }
 
+/**
+ * @returns true if the broker needs a persistent connection
+ * @locality any
+ */
+static RD_UNUSED RD_INLINE rd_bool_t
+rd_kafka_broker_termination_in_progress(rd_kafka_broker_t *rkb) {
+        return rd_atomic32_get(&rkb->termination_in_progress) > 0;
+}
 
 /**
  * @brief Broker comparator
@@ -452,6 +498,14 @@ int16_t rd_kafka_broker_ApiVersion_supported0(rd_kafka_broker_t *rkb,
                                               int *featuresp,
                                               rd_bool_t do_lock);
 
+rd_bool_t rd_kafka_broker_ApiVersion_at_least(rd_kafka_broker_t *rkb,
+                                              int16_t ApiKey,
+                                              int16_t minver);
+
+rd_bool_t rd_kafka_broker_ApiVersion_at_least_no_lock(rd_kafka_broker_t *rkb,
+                                                      int16_t ApiKey,
+                                                      int16_t minver);
+
 rd_kafka_broker_t *rd_kafka_broker_find_by_nodeid0_fl(const char *func,
                                                       int line,
                                                       rd_kafka_t *rk,
@@ -681,6 +735,10 @@ void rd_kafka_broker_start_reauth_timer(rd_kafka_broker_t *rkb,
 
 void rd_kafka_broker_start_reauth_cb(rd_kafka_timers_t *rkts, void *rkb);
 
+void rd_kafka_broker_decommission(rd_kafka_t *rk,
+                                  rd_kafka_broker_t *rkb,
+                                  rd_list_t *wait_thrds);
+
 int unittest_broker(void);
 
 #endif /* _RDKAFKA_BROKER_H_ */

package/deps/librdkafka/src/rdkafka_buf.c

@@ -42,9 +42,6 @@ void rd_kafka_buf_destroy_final(rd_kafka_buf_t *rkbuf) {
                         rd_list_destroy(rkbuf->rkbuf_u.Metadata.topic_ids);
                 if (rkbuf->rkbuf_u.Metadata.reason)
                         rd_free(rkbuf->rkbuf_u.Metadata.reason);
-                if (rkbuf->rkbuf_u.Metadata.rko)
-                        rd_kafka_op_reply(rkbuf->rkbuf_u.Metadata.rko,
-                                          RD_KAFKA_RESP_ERR__DESTROY);
                 if (rkbuf->rkbuf_u.Metadata.decr) {
                         /* Decrease metadata cache's full_.._sent state. */
                         mtx_lock(rkbuf->rkbuf_u.Metadata.decr_lock);
@@ -244,6 +241,12 @@ void rd_kafka_bufq_init(rd_kafka_bufq_t *rkbufq) {
         rd_atomic32_init(&rkbufq->rkbq_msg_cnt, 0);
 }
 
+static void rd_kafka_bufq_reset(rd_kafka_bufq_t *rkbufq) {
+        TAILQ_INIT(&rkbufq->rkbq_bufs);
+        rd_atomic32_set(&rkbufq->rkbq_cnt, 0);
+        rd_atomic32_set(&rkbufq->rkbq_msg_cnt, 0);
+}
+
 /**
  * Concat all buffers from 'src' to tail of 'dst'
  */
@@ -252,7 +255,7 @@ void rd_kafka_bufq_concat(rd_kafka_bufq_t *dst, rd_kafka_bufq_t *src) {
         (void)rd_atomic32_add(&dst->rkbq_cnt, rd_atomic32_get(&src->rkbq_cnt));
         (void)rd_atomic32_add(&dst->rkbq_msg_cnt,
                               rd_atomic32_get(&src->rkbq_msg_cnt));
-        rd_kafka_bufq_init(src);
+        rd_kafka_bufq_reset(src);
 }
 
 /**
@@ -393,7 +396,7 @@ int rd_kafka_buf_retry(rd_kafka_broker_t *rkb, rd_kafka_buf_t *rkbuf) {
         rd_assert(rd_buf_len(&rkbuf->rkbuf_buf) > 0);
 
         if (unlikely(!rkb || rkb->rkb_source == RD_KAFKA_INTERNAL ||
-                     rd_kafka_terminating(rkb->rkb_rk) ||
+                     rd_kafka_broker_or_instance_terminating(rkb) ||
                      rkbuf->rkbuf_retries + incr_retry >
                          rkbuf->rkbuf_max_retries))
                 return 0;

package/deps/librdkafka/src/rdkafka_buf.h

@@ -379,11 +379,12 @@ struct rd_kafka_buf_s { /* rd_kafka_buf_t */
                         rd_list_t *
                             topic_ids; /* Requested topic ids rd_kafka_Uuid_t */
                         char *reason;  /* Textual reason */
-                        rd_kafka_op_t *rko;    /* Originating rko with replyq
-                                                * (if any) */
                         rd_bool_t all_topics;  /**< Full/All topics requested */
                         rd_bool_t cgrp_update; /**< Update cgrp with topic
                                                 *   status from response. */
+                        int32_t cgrp_subscription_version;
+                        /**< Consumer group subscription version, to
+                         *   check before updating cgrp state. */
                         rd_bool_t force_racks; /**< Force the returned metadata
                                                 *   to contain partition to
                                                 *   rack mapping. */
@@ -524,7 +525,7 @@ struct rd_kafka_buf_s { /* rd_kafka_buf_t */
 #define rd_kafka_buf_skip_to(rkbuf, pos)                                       \
         do {                                                                   \
                 size_t __len1 =                                                \
-                    (size_t)(pos)-rd_slice_offset(&(rkbuf)->rkbuf_reader);     \
+                    (size_t)(pos) - rd_slice_offset(&(rkbuf)->rkbuf_reader);   \
                 if (__len1 &&                                                  \
                     !rd_slice_read(&(rkbuf)->rkbuf_reader, NULL, __len1))      \
                         rd_kafka_buf_check_len(rkbuf, __len1);                 \

package/deps/librdkafka/src/rdkafka_cert.c

@@ -78,6 +78,8 @@ static void rd_kafka_cert_destroy(rd_kafka_cert_t *cert) {
 
         if (cert->x509)
                 X509_free(cert->x509);
+        if (cert->chain)
+                sk_X509_pop_free(cert->chain, X509_free);
         if (cert->pkey)
                 EVP_PKEY_free(cert->pkey);
         if (cert->store)
@@ -314,10 +316,11 @@ static rd_kafka_cert_t *rd_kafka_cert_new(const rd_kafka_conf_t *conf,
                 switch (encoding) {
                 case RD_KAFKA_CERT_ENC_PKCS12: {
                         EVP_PKEY *ign_pkey;
+                        STACK_OF(X509) *ca = NULL;
 
                         action = "parse PKCS#12";
                         if (!PKCS12_parse(p12, conf->ssl.key_password,
-                                          &ign_pkey, &cert->x509, NULL))
+                                          &ign_pkey, &cert->x509, &ca))
                                 goto fail;
 
                         EVP_PKEY_free(ign_pkey);
@@ -325,6 +328,13 @@ static rd_kafka_cert_t *rd_kafka_cert_new(const rd_kafka_conf_t *conf,
                         action = "retrieve public key";
                         if (!cert->x509)
                                 goto fail;
+
+                        if (ca) {
+                                if (sk_X509_num(ca) > 0)
+                                        cert->chain = ca;
+                                else
+                                        sk_X509_pop_free(ca, X509_free);
+                        }
                 } break;
 
                 case RD_KAFKA_CERT_ENC_DER:
@@ -341,6 +351,20 @@ static rd_kafka_cert_t *rd_kafka_cert_new(const rd_kafka_conf_t *conf,
                             (void *)conf);
                         if (!cert->x509)
                                 goto fail;
+
+                        cert->chain = sk_X509_new_null();
+                        if (rd_kafka_ssl_read_cert_chain_from_BIO(
+                                bio, cert->chain, rd_kafka_conf_ssl_passwd_cb,
+                                (void *)conf) != 0) {
+                                sk_X509_pop_free(cert->chain, X509_free);
+                                cert->chain = NULL;
+                                goto fail;
+                        }
+
+                        if (sk_X509_num(cert->chain) == 0) {
+                                sk_X509_pop_free(cert->chain, X509_free);
+                                cert->chain = NULL;
+                        }
                         break;
 
                 default:

package/deps/librdkafka/src/rdkafka_cert.h

@@ -43,9 +43,10 @@ typedef struct rd_kafka_cert_s {
         rd_kafka_cert_enc_t encoding;
         rd_refcnt_t refcnt;
 #if WITH_SSL
-        X509 *x509;        /**< Certificate (public key) */
-        EVP_PKEY *pkey;    /**< Private key */
-        X509_STORE *store; /**< CA certificate chain store */
+        X509 *x509;             /**< Certificate (public key) */
+        STACK_OF(X509) * chain; /**< Certificate chain (public key) */
+        EVP_PKEY *pkey;         /**< Private key */
+        X509_STORE *store;      /**< CA trusted certificates */
 #endif
 } rd_kafka_cert_t;