@platformatic/rdkafka 4.0.1 → 4.1.0-alpha.1

package/deps/librdkafka/src/rdkafka_ssl.h

@@ -61,4 +61,26 @@ int rd_kafka_ssl_hmac(rd_kafka_broker_t *rkb,
                       int itcnt,
                       rd_chariov_t *out);
 
+int rd_kafka_ssl_read_cert_chain_from_BIO(BIO *in,
+                                          STACK_OF(X509) * chainp,
+                                          pem_password_cb *password_cb,
+                                          void *password_cb_opaque);
+
+int rd_kafka_ssl_probe_and_set_default_ca_location(rd_kafka_t *rk,
+                                                   const char *ctx_identifier,
+                                                   SSL_CTX *ctx);
+
+char *rd_kafka_ssl_error0(rd_kafka_t *rk,
+                          rd_kafka_broker_t *rkb,
+                          const char *ctx_identifier,
+                          char *errstr,
+                          size_t errstr_size);
+
+#ifdef _WIN32
+int rd_kafka_ssl_win_load_cert_stores(rd_kafka_t *rk,
+                                      const char *ctx_identifier,
+                                      SSL_CTX *ctx,
+                                      const char *store_names);
+#endif
+
 #endif /* _RDKAFKA_SSL_H_ */

package/deps/librdkafka/src/rdkafka_sticky_assignor.c

@@ -3027,6 +3027,10 @@ static int ut_testLargeAssignmentWithMultipleConsumersLeaving(
     rd_kafka_t *rk,
     const rd_kafka_assignor_t *rkas,
     rd_kafka_assignor_ut_rack_config_t parametrization) {
+        if (rd_unittest_with_valgrind)
+                RD_UT_SKIP(
+                    "Skipping large assignment test when using Valgrind");
+
         rd_kafka_resp_err_t err;
         char errstr[512];
         rd_kafka_metadata_t *metadata;

package/deps/librdkafka/src/rdkafka_telemetry.c

@@ -33,6 +33,7 @@
 #include "nanopb/pb.h"
 #include "rdkafka_lz4.h"
 #include "snappy.h"
+#include "rdunittest.h"
 
 #if WITH_ZSTD
 #include "rdkafka_zstd.h"
@@ -51,10 +52,8 @@
 static int
 rd_kafka_filter_broker_by_GetTelemetrySubscription(rd_kafka_broker_t *rkb,
                                                    void *opaque) {
-        int features;
-        if (rd_kafka_broker_ApiVersion_supported0(
-                rkb, RD_KAFKAP_GetTelemetrySubscriptions, 0, 0, &features,
-                rd_false) != -1)
+        if (rd_kafka_broker_ApiVersion_at_least_no_lock(
+                rkb, RD_KAFKAP_GetTelemetrySubscriptions, 0))
                 return 0;
         return 1;
 }
@@ -173,6 +172,9 @@ static void update_matched_metrics(rd_kafka_t *rk, size_t j) {
 
 static void rd_kafka_match_requested_metrics(rd_kafka_t *rk) {
         size_t metrics_cnt = RD_KAFKA_TELEMETRY_METRIC_CNT(rk), i;
+        rd_bool_t is_metric_included[RD_MAX(
+            (int)RD_KAFKA_TELEMETRY_PRODUCER_METRIC__CNT,
+            (int)RD_KAFKA_TELEMETRY_CONSUMER_METRIC__CNT)] = {0};
         const rd_kafka_telemetry_metric_info_t *info =
             RD_KAFKA_TELEMETRY_METRIC_INFO(rk);
 
@@ -193,6 +195,9 @@ static void rd_kafka_match_requested_metrics(rd_kafka_t *rk) {
                        j;
 
                 for (j = 0; j < metrics_cnt; j++) {
+                        if (is_metric_included[j])
+                                continue;
+
                         /* Prefix matching the requested metrics with the
                          * available metrics. */
                         char full_metric_name
@@ -205,8 +210,10 @@ static void rd_kafka_match_requested_metrics(rd_kafka_t *rk) {
                                     rk->rk_telemetry.requested_metrics[i],
                                     name_len) == 0;
 
-                        if (name_matches)
+                        if (name_matches) {
                                 update_matched_metrics(rk, j);
+                                is_metric_included[j] = rd_true;
+                        }
                 }
         }
 
@@ -266,6 +273,24 @@ rd_kafka_push_telemetry_payload_compress(rd_kafka_t *rk,
         rd_slice_t payload_slice;
         size_t i;
         rd_kafka_resp_err_t r = RD_KAFKA_RESP_ERR_NO_ERROR;
+
+        if (payload->rbuf_len == 0) {
+                /* We can only initialize the slice to compress
+                 * if not empty. */
+                rd_kafka_dbg(rk, TELEMETRY, "PUSH",
+                             "Empty payload. "
+                             "Sending uncompressed payload");
+
+                /* It's not important the payload isn't actually a segment
+                 * inside the buffer, as size is 0, we can send any allocated
+                 * memory here, but we chose the buffer because it's
+                 * freed like the other COMPRESSION_NONE case, without
+                 * memory leaks. */
+                *compressed_payload      = payload;
+                *compressed_payload_size = 0;
+                return RD_KAFKA_COMPRESSION_NONE;
+        }
+
         rd_slice_init_full(&payload_slice, payload);
         for (i = 0; i < rk->rk_telemetry.accepted_compression_types_cnt; i++) {
                 rd_kafka_compression_t compression_type =
@@ -359,21 +384,24 @@ static void rd_kafka_send_push_telemetry(rd_kafka_t *rk,
                                      compressed_metrics_payload_size,
                                      rk->rk_telemetry.telemetry_max_bytes);
                 }
+
+                rd_kafka_dbg(
+                    rk, TELEMETRY, "PUSH",
+                    "Sending PushTelemetryRequest with terminating = %s",
+                    RD_STR_ToF(terminating));
+                rd_kafka_PushTelemetryRequest(
+                    rkb, &rk->rk_telemetry.client_instance_id,
+                    rk->rk_telemetry.subscription_id, terminating,
+                    compression_used, compressed_metrics_payload,
+                    compressed_metrics_payload_size, NULL, 0,
+                    RD_KAFKA_REPLYQ(rk->rk_ops, 0),
+                    rd_kafka_handle_PushTelemetry, NULL);
         } else {
-                rd_kafka_dbg(rk, TELEMETRY, "PUSH",
-                             "No metrics to push. Sending empty payload.");
+                rd_kafka_log(rk, LOG_WARNING, "PUSH",
+                             "Telemetry metrics encode error, not sending "
+                             "metrics");
         }
 
-        rd_kafka_dbg(rk, TELEMETRY, "PUSH",
-                     "Sending PushTelemetryRequest with terminating = %s",
-                     RD_STR_ToF(terminating));
-        rd_kafka_PushTelemetryRequest(
-            rkb, &rk->rk_telemetry.client_instance_id,
-            rk->rk_telemetry.subscription_id, terminating, compression_used,
-            compressed_metrics_payload, compressed_metrics_payload_size, NULL,
-            0, RD_KAFKA_REPLYQ(rk->rk_ops, 0), rd_kafka_handle_PushTelemetry,
-            NULL);
-
         if (metrics_payload)
                 rd_buf_destroy_free(metrics_payload);
         if (compression_used != RD_KAFKA_COMPRESSION_NONE)
@@ -388,7 +416,7 @@ static void rd_kafka_send_push_telemetry(rd_kafka_t *rk,
  * @brief Progress the telemetry state machine.
  *
  * @locks none
- * @locks_acquired none
+ * @locks_acquired rd_kafka_rdlock()
  * @locality main thread
  */
 static void rd_kafka_telemetry_fsm(rd_kafka_t *rk) {
@@ -464,7 +492,7 @@ void rd_kafka_telemetry_fsm_tmr_cb(rd_kafka_timers_t *rkts, void *rk) {
  * @brief Handles parsed GetTelemetrySubscriptions response.
  *
  * @locks none
- * @locks_acquired none
+ * @locks_acquired rd_kafka_rdlock()
  * @locality main thread
  */
 void rd_kafka_handle_get_telemetry_subscriptions(rd_kafka_t *rk,
@@ -501,10 +529,12 @@ void rd_kafka_handle_get_telemetry_subscriptions(rd_kafka_t *rk,
                 if (rk->rk_telemetry.rk_historic_c.ts_start == 0) {
                         rk->rk_telemetry.rk_historic_c.ts_start = now_ns;
                         rk->rk_telemetry.rk_historic_c.ts_last  = now_ns;
+                        rd_kafka_rdlock(rk);
                         TAILQ_FOREACH(rkb, &rk->rk_brokers, rkb_link) {
                                 rkb->rkb_telemetry.rkb_historic_c.connects =
                                     rd_atomic32_get(&rkb->rkb_c.connects);
                         }
+                        rd_kafka_rdunlock(rk);
                 }
 
         } else {
@@ -617,6 +647,7 @@ void rd_kafka_telemetry_await_termination(rd_kafka_t *rk) {
                 return;
         }
 
+        mtx_lock(&rk->rk_telemetry.lock);
         rko         = rd_kafka_op_new(RD_KAFKA_OP_TERMINATE_TELEMETRY);
         rko->rko_rk = rk;
         rd_kafka_q_enq(rk->rk_ops, rko);
@@ -624,12 +655,8 @@ void rd_kafka_telemetry_await_termination(rd_kafka_t *rk) {
         /* Await termination sequence completion. */
         rd_kafka_dbg(rk, TELEMETRY, "TERM",
                      "Awaiting termination of telemetry.");
-        mtx_lock(&rk->rk_telemetry.lock);
         cnd_timedwait_ms(&rk->rk_telemetry.termination_cnd,
                          &rk->rk_telemetry.lock,
-                         /* TODO(milind): Evaluate this timeout after completion
-                            of all metrics push, is it too much, or too less if
-                            we include serialization? */
                          1000 /* timeout for waiting */);
         mtx_unlock(&rk->rk_telemetry.lock);
         rd_kafka_dbg(rk, TELEMETRY, "TERM",
@@ -701,3 +728,33 @@ void rd_kafka_set_telemetry_broker_maybe(rd_kafka_t *rk,
             &rk->rk_timers, &rk->rk_telemetry.request_timer, rd_false,
             0 /* immediate */, rd_kafka_telemetry_fsm_tmr_cb, (void *)rk);
 }
+
+/**
+ * @brief Overlapping prefixes should not match the metrics
+ *        multiple times.
+ */
+int unit_test_telemetry_match_requested_metrics_no_duplicates(void) {
+        rd_kafka_t *rk = rd_kafka_new(RD_KAFKA_PRODUCER, NULL, NULL, 0);
+        rk->rk_telemetry.requested_metrics_cnt = 3;
+        rk->rk_telemetry.requested_metrics =
+            rd_calloc(rk->rk_telemetry.requested_metrics_cnt, sizeof(char *));
+        rk->rk_telemetry.requested_metrics[0] = rd_strdup("org");
+        rk->rk_telemetry.requested_metrics[1] = rd_strdup("org.apache");
+        rk->rk_telemetry.requested_metrics[2] = rd_strdup("org.apache.kafka");
+        rd_kafka_match_requested_metrics(rk);
+
+        RD_UT_ASSERT(rk->rk_telemetry.matched_metrics_cnt ==
+                         RD_KAFKA_TELEMETRY_PRODUCER_METRIC__CNT,
+                     "Expected %d matched metrics, got %" PRIusz,
+                     RD_KAFKA_TELEMETRY_PRODUCER_METRIC__CNT,
+                     rk->rk_telemetry.matched_metrics_cnt);
+        rd_kafka_destroy(rk);
+        return 0;
+}
+
+
+int unittest_telemetry(void) {
+        int fails = 0;
+        fails += unit_test_telemetry_match_requested_metrics_no_duplicates();
+        return fails;
+}

package/deps/librdkafka/src/rdkafka_telemetry_encode.c

@@ -713,6 +713,10 @@ static void serialize_Metric(
  * @brief Encodes the metrics to opentelemetry_proto_metrics_v1_MetricsData and
  * returns the serialized data. Currently only supports encoding of connection
  * creation total by default
+ *
+ * @locks none
+ * @locks_acquired rd_kafka_rdlock()
+ * @locality main thread
  */
 rd_buf_t *rd_kafka_telemetry_encode_metrics(rd_kafka_t *rk) {
         rd_buf_t *rbuf = NULL;
@@ -731,7 +735,7 @@ rd_buf_t *rd_kafka_telemetry_encode_metrics(rd_kafka_t *rk) {
         size_t i, metric_idx = 0;
 
         if (!metrics_to_encode_count)
-                return NULL;
+                return rd_buf_new(1, 1);
 
         opentelemetry_proto_metrics_v1_MetricsData metrics_data =
             opentelemetry_proto_metrics_v1_MetricsData_init_zero;

package/deps/librdkafka/src/rdkafka_telemetry_encode.h

@@ -163,16 +163,16 @@ static const rd_kafka_telemetry_metric_info_t
                 {.name        = "producer.record.queue.time.avg",
                  .description = "The average time in ms a record spends in the "
                                 "producer queue.",
-                 .unit          = "ms",
-                 .is_int        = rd_false,
+                 .unit        = "ms",
+                 .is_int      = rd_false,
                  .is_per_broker = rd_false,
                  .type          = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE},
             [RD_KAFKA_TELEMETRY_METRIC_PRODUCER_RECORD_QUEUE_TIME_MAX] =
                 {.name        = "producer.record.queue.time.max",
                  .description = "The maximum time in ms a record spends in the "
                                 "producer queue.",
-                 .unit          = "ms",
-                 .is_int        = rd_true,
+                 .unit        = "ms",
+                 .is_int      = rd_true,
                  .is_per_broker = rd_false,
                  .type          = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE},
             [RD_KAFKA_TELEMETRY_METRIC_PRODUCER_PRODUCE_LATENCY_AVG] =
@@ -224,33 +224,33 @@ static const rd_kafka_telemetry_metric_info_t RD_KAFKA_TELEMETRY_CONSUMER_METRIC
              .is_per_broker = rd_true,
              .type          = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE},
         [RD_KAFKA_TELEMETRY_METRIC_CONSUMER_COORDINATOR_ASSIGNED_PARTITIONS] =
-            {.name        = "consumer.coordinator.assigned.partitions",
-             .description = "The number of partitions currently assigned "
-                            "to this consumer.",
+            {.name          = "consumer.coordinator.assigned.partitions",
+             .description   = "The number of partitions currently assigned "
+                              "to this consumer.",
              .unit          = "1",
              .is_int        = rd_true,
              .is_per_broker = rd_false,
              .type          = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE},
         [RD_KAFKA_TELEMETRY_METRIC_CONSUMER_COORDINATOR_REBALANCE_LATENCY_AVG] =
-            {.name        = "consumer.coordinator.rebalance.latency.avg",
-             .description = "The average rebalance latency in ms for the "
-                            "consumer coordinator.",
+            {.name          = "consumer.coordinator.rebalance.latency.avg",
+             .description   = "The average rebalance latency in ms for the "
+                              "consumer coordinator.",
              .unit          = "ms",
              .is_int        = rd_false,
              .is_per_broker = rd_false,
              .type          = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE},
         [RD_KAFKA_TELEMETRY_METRIC_CONSUMER_COORDINATOR_REBALANCE_LATENCY_MAX] =
-            {.name        = "consumer.coordinator.rebalance.latency.max",
-             .description = "The maximum rebalance latency in ms for the "
-                            "consumer coordinator.",
+            {.name          = "consumer.coordinator.rebalance.latency.max",
+             .description   = "The maximum rebalance latency in ms for the "
+                              "consumer coordinator.",
              .unit          = "ms",
              .is_int        = rd_true,
              .is_per_broker = rd_false,
              .type          = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE},
         [RD_KAFKA_TELEMETRY_METRIC_CONSUMER_COORDINATOR_REBALANCE_LATENCY_TOTAL] =
-            {.name        = "consumer.coordinator.rebalance.latency.total",
-             .description = "The total rebalance latency in ms for the "
-                            "consumer coordinator.",
+            {.name          = "consumer.coordinator.rebalance.latency.total",
+             .description   = "The total rebalance latency in ms for the "
+                              "consumer coordinator.",
              .unit          = "ms",
              .is_int        = rd_true,
              .is_per_broker = rd_false,
@@ -282,16 +282,16 @@ static const rd_kafka_telemetry_metric_info_t RD_KAFKA_TELEMETRY_CONSUMER_METRIC
             {.name        = "consumer.coordinator.commit.latency.avg",
              .description = "The average commit latency in ms for the consumer "
                             "coordinator.",
-             .unit          = "ms",
-             .is_int        = rd_false,
+             .unit        = "ms",
+             .is_int      = rd_false,
              .is_per_broker = rd_false,
              .type          = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE},
         [RD_KAFKA_TELEMETRY_METRIC_CONSUMER_COORDINATOR_COMMIT_LATENCY_MAX] =
             {.name        = "consumer.coordinator.commit.latency.max",
              .description = "The maximum commit latency in ms for the consumer "
                             "coordinator.",
-             .unit          = "ms",
-             .is_int        = rd_true,
+             .unit        = "ms",
+             .is_int      = rd_true,
              .is_per_broker = rd_false,
              .type          = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE},
 };