@platformatic/rdkafka 4.0.0 → 4.1.0-alpha.1

package/deps/librdkafka/src/rdkafka_queue.c

@@ -93,10 +93,12 @@ void rd_kafka_q_init0(rd_kafka_q_t *rkq,
         rkq->rkq_flags  = RD_KAFKA_Q_F_READY;
         if (for_consume)
                 rkq->rkq_flags |= RD_KAFKA_Q_F_CONSUMER;
-        rkq->rkq_rk     = rk;
-        rkq->rkq_qio    = NULL;
-        rkq->rkq_serve  = NULL;
-        rkq->rkq_opaque = NULL;
+        rkq->rkq_rk                 = rk;
+        rkq->rkq_qio                = NULL;
+        rkq->rkq_serve              = NULL;
+        rkq->rkq_opaque             = NULL;
+        rkq->rkq_ts_last_poll_start = 0;
+        rkq->rkq_ts_last_poll_end   = 0;
         mtx_init(&rkq->rkq_lock, mtx_plain);
         cnd_init(&rkq->rkq_cond);
 #if ENABLE_DEVEL
@@ -380,16 +382,26 @@ rd_kafka_op_filter(rd_kafka_q_t *rkq, rd_kafka_op_t *rko, int version) {
  * Serve q like rd_kafka_q_serve() until an op is found that can be returned
  * as an event to the application.
  *
+ * @param rkq Queue to pop from.
+ * @param timeout_us Maximum time to wait for an op, in microseconds.
+ * @param version Fetch version to filter out outdated ops.
+ * @param cb_type Callback type to use for the op.
+ * @param callback Callback to use for the op, if any.
+ * @param opaque Opaque pointer to pass to the callback.
+ * @param is_consume_call If `rd_true` and it could be a consumer call it
+ *                        checks if this queue can contain fetched messages.
+ *
  * @returns the first event:able op, or NULL on timeout.
  *
- * Locality: any thread
+ * @locality any thread
  */
-rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq,
-                                    rd_ts_t timeout_us,
-                                    int32_t version,
-                                    rd_kafka_q_cb_type_t cb_type,
-                                    rd_kafka_q_serve_cb_t *callback,
-                                    void *opaque) {
+static rd_kafka_op_t *rd_kafka_q_pop_serve0(rd_kafka_q_t *rkq,
+                                            rd_ts_t timeout_us,
+                                            int32_t version,
+                                            rd_kafka_q_cb_type_t cb_type,
+                                            rd_kafka_q_serve_cb_t *callback,
+                                            void *opaque,
+                                            rd_bool_t is_consume_call) {
         rd_kafka_op_t *rko;
         rd_kafka_q_t *fwdq;
 
@@ -400,14 +412,14 @@ rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq,
         rd_kafka_yield_thread = 0;
         if (!(fwdq = rd_kafka_q_fwd_get(rkq, 0))) {
                 const rd_bool_t can_q_contain_fetched_msgs =
+                    is_consume_call &&
                     rd_kafka_q_can_contain_fetched_msgs(rkq, RD_DONT_LOCK);
 
-                struct timespec timeout_tspec;
-
-                rd_timeout_init_timespec_us(&timeout_tspec, timeout_us);
+                rd_ts_t abs_timeout = rd_timeout_init_us(timeout_us);
 
                 if (can_q_contain_fetched_msgs)
-                        rd_kafka_app_poll_start(rkq->rkq_rk, 0, timeout_us);
+                        rd_kafka_app_poll_start(rkq->rkq_rk, rkq, 0,
+                                                timeout_us);
 
                 while (1) {
                         rd_kafka_op_res_t res;
@@ -447,14 +459,14 @@ rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq,
                                 } else if (unlikely(res ==
                                                     RD_KAFKA_OP_RES_YIELD)) {
                                         if (can_q_contain_fetched_msgs)
-                                                rd_kafka_app_polled(
-                                                    rkq->rkq_rk);
+                                                rd_kafka_app_polled(rkq->rkq_rk,
+                                                                    rkq);
                                         /* Callback yielded, unroll */
                                         return NULL;
                                 } else {
                                         if (can_q_contain_fetched_msgs)
-                                                rd_kafka_app_polled(
-                                                    rkq->rkq_rk);
+                                                rd_kafka_app_polled(rkq->rkq_rk,
+                                                                    rkq);
                                         break; /* Proper op, handle below. */
                                 }
                         }
@@ -463,7 +475,7 @@ rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq,
                                 if (is_locked)
                                         mtx_unlock(&rkq->rkq_lock);
                                 if (can_q_contain_fetched_msgs)
-                                        rd_kafka_app_polled(rkq->rkq_rk);
+                                        rd_kafka_app_polled(rkq->rkq_rk, rkq);
                                 return NULL;
                         }
 
@@ -471,10 +483,10 @@ rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq,
                                 mtx_lock(&rkq->rkq_lock);
 
                         if (cnd_timedwait_abs(&rkq->rkq_cond, &rkq->rkq_lock,
-                                              &timeout_tspec) != thrd_success) {
+                                              abs_timeout) != thrd_success) {
                                 mtx_unlock(&rkq->rkq_lock);
                                 if (can_q_contain_fetched_msgs)
-                                        rd_kafka_app_polled(rkq->rkq_rk);
+                                        rd_kafka_app_polled(rkq->rkq_rk, rkq);
                                 return NULL;
                         }
                 }
@@ -483,8 +495,8 @@ rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq,
                 /* Since the q_pop may block we need to release the parent
                  * queue's lock. */
                 mtx_unlock(&rkq->rkq_lock);
-                rko = rd_kafka_q_pop_serve(fwdq, timeout_us, version, cb_type,
-                                           callback, opaque);
+                rko = rd_kafka_q_pop_serve0(fwdq, timeout_us, version, cb_type,
+                                            callback, opaque, is_consume_call);
                 rd_kafka_q_destroy(fwdq);
         }
 
@@ -492,35 +504,72 @@ rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq,
         return rko;
 }
 
+rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq,
+                                    rd_ts_t timeout_us,
+                                    int32_t version,
+                                    rd_kafka_q_cb_type_t cb_type,
+                                    rd_kafka_q_serve_cb_t *callback,
+                                    void *opaque) {
+        return rd_kafka_q_pop_serve0(rkq, timeout_us, version, cb_type,
+                                     callback, opaque, rd_false);
+}
+
+/**
+ * @brief Same as `rd_kafka_q_pop_serve`, use this call when the queue
+ *        could be a fetch queue, use the other one when it
+ *        can never be.
+ */
+rd_kafka_op_t *
+rd_kafka_q_pop_serve_maybe_consume(rd_kafka_q_t *rkq,
+                                   rd_ts_t timeout_us,
+                                   int32_t version,
+                                   rd_kafka_q_cb_type_t cb_type,
+                                   rd_kafka_q_serve_cb_t *callback,
+                                   void *opaque) {
+        return rd_kafka_q_pop_serve0(rkq, timeout_us, version, cb_type,
+                                     callback, opaque,
+                                     /* Only check if to call app_polled when
+                                      * this is a consumer. */
+                                     rkq->rkq_rk->rk_type == RD_KAFKA_CONSUMER);
+}
+
 rd_kafka_op_t *
 rd_kafka_q_pop(rd_kafka_q_t *rkq, rd_ts_t timeout_us, int32_t version) {
         return rd_kafka_q_pop_serve(rkq, timeout_us, version,
                                     RD_KAFKA_Q_CB_RETURN, NULL, NULL);
 }
 
-
 /**
  * Pop all available ops from a queue and call the provided
  * callback for each op.
- * `max_cnt` limits the number of ops served, 0 = no limit.
  *
- * Returns the number of ops served.
+ * @param rkq Queue to serve.
+ * @param max_cnt Limits the number of ops served, 0 = no limit.
+ * @param cb_type Callback type to use.
+ * @param callback Callback to call for each op.
+ * @param opaque Opaque pointer to pass to the callback.
+ * @param is_consume_call If `rd_true` and it could be a consumer call it
+ *                        checks if this queue can contain fetched messages.
  *
- * Locality: any thread.
+ * @return The number of ops served.
+ *
+ * @locality any thread.
  */
-int rd_kafka_q_serve(rd_kafka_q_t *rkq,
-                     int timeout_ms,
-                     int max_cnt,
-                     rd_kafka_q_cb_type_t cb_type,
-                     rd_kafka_q_serve_cb_t *callback,
-                     void *opaque) {
+int rd_kafka_q_serve0(rd_kafka_q_t *rkq,
+                      int timeout_ms,
+                      int max_cnt,
+                      rd_kafka_q_cb_type_t cb_type,
+                      rd_kafka_q_serve_cb_t *callback,
+                      void *opaque,
+                      rd_bool_t is_consume_call) {
         rd_kafka_t *rk = rkq->rkq_rk;
         rd_kafka_op_t *rko;
         rd_kafka_q_t localq;
         rd_kafka_q_t *fwdq;
         int cnt = 0;
-        struct timespec timeout_tspec;
+        rd_ts_t abs_timeout;
         const rd_bool_t can_q_contain_fetched_msgs =
+            is_consume_call &&
             rd_kafka_q_can_contain_fetched_msgs(rkq, RD_DONT_LOCK);
 
         rd_dassert(cb_type);
@@ -533,23 +582,23 @@ int rd_kafka_q_serve(rd_kafka_q_t *rkq,
                 /* Since the q_pop may block we need to release the parent
                  * queue's lock. */
                 mtx_unlock(&rkq->rkq_lock);
-                ret = rd_kafka_q_serve(fwdq, timeout_ms, max_cnt, cb_type,
-                                       callback, opaque);
+                ret = rd_kafka_q_serve0(fwdq, timeout_ms, max_cnt, cb_type,
+                                        callback, opaque, is_consume_call);
                 rd_kafka_q_destroy(fwdq);
                 return ret;
         }
 
 
-        rd_timeout_init_timespec(&timeout_tspec, timeout_ms);
+        abs_timeout = rd_timeout_init(timeout_ms);
 
         if (can_q_contain_fetched_msgs)
-                rd_kafka_app_poll_start(rk, 0, timeout_ms);
+                rd_kafka_app_poll_start(rk, rkq, 0, timeout_ms);
 
         /* Wait for op */
         while (!(rko = TAILQ_FIRST(&rkq->rkq_q)) &&
                !rd_kafka_q_check_yield(rkq) &&
-               cnd_timedwait_abs(&rkq->rkq_cond, &rkq->rkq_lock,
-                                 &timeout_tspec) == thrd_success)
+               cnd_timedwait_abs(&rkq->rkq_cond, &rkq->rkq_lock, abs_timeout) ==
+                   thrd_success)
                 ;
 
         rd_kafka_q_mark_served(rkq);
@@ -557,7 +606,7 @@ int rd_kafka_q_serve(rd_kafka_q_t *rkq,
         if (!rko) {
                 mtx_unlock(&rkq->rkq_lock);
                 if (can_q_contain_fetched_msgs)
-                        rd_kafka_app_polled(rk);
+                        rd_kafka_app_polled(rk, rkq);
                 return 0;
         }
 
@@ -593,13 +642,41 @@ int rd_kafka_q_serve(rd_kafka_q_t *rkq,
         }
 
         if (can_q_contain_fetched_msgs)
-                rd_kafka_app_polled(rk);
+                rd_kafka_app_polled(rk, rkq);
 
         rd_kafka_q_destroy_owner(&localq);
 
         return cnt;
 }
 
+int rd_kafka_q_serve(rd_kafka_q_t *rkq,
+                     int timeout_ms,
+                     int max_cnt,
+                     rd_kafka_q_cb_type_t cb_type,
+                     rd_kafka_q_serve_cb_t *callback,
+                     void *opaque) {
+        return rd_kafka_q_serve0(rkq, timeout_ms, max_cnt, cb_type, callback,
+                                 opaque, rd_false);
+}
+
+/**
+ * @brief Same as `rd_kafka_q_serve`, use this call when the queue
+ *        could be a fetch queue, use the other one when it
+ *        can never be.
+ */
+int rd_kafka_q_serve_maybe_consume(rd_kafka_q_t *rkq,
+                                   int timeout_ms,
+                                   int max_cnt,
+                                   rd_kafka_q_cb_type_t cb_type,
+                                   rd_kafka_q_serve_cb_t *callback,
+                                   void *opaque) {
+        return rd_kafka_q_serve0(rkq, timeout_ms, max_cnt, cb_type, callback,
+                                 opaque,
+                                 /* Only check if to call app_polled when
+                                  * this is a consumer. */
+                                 rkq->rkq_rk->rk_type == RD_KAFKA_CONSUMER);
+}
+
 /**
  * @brief Filter out and destroy outdated messages.
  *
@@ -665,7 +742,7 @@ int rd_kafka_q_serve_rkmessages(rd_kafka_q_t *rkq,
         rd_kafka_op_t *rko, *next;
         rd_kafka_t *rk = rkq->rkq_rk;
         rd_kafka_q_t *fwdq;
-        struct timespec timeout_tspec;
+        rd_ts_t abs_timeout;
         int i;
 
         mtx_lock(&rkq->rkq_lock);
@@ -681,9 +758,9 @@ int rd_kafka_q_serve_rkmessages(rd_kafka_q_t *rkq,
 
         mtx_unlock(&rkq->rkq_lock);
 
-        rd_timeout_init_timespec(&timeout_tspec, timeout_ms);
+        abs_timeout = rd_timeout_init(timeout_ms);
 
-        rd_kafka_app_poll_start(rk, 0, timeout_ms);
+        rd_kafka_app_poll_start(rk, rkq, 0, timeout_ms);
 
         rd_kafka_yield_thread = 0;
         while (cnt < rkmessages_size) {
@@ -694,7 +771,7 @@ int rd_kafka_q_serve_rkmessages(rd_kafka_q_t *rkq,
                 while (!(rko = TAILQ_FIRST(&rkq->rkq_q)) &&
                        !rd_kafka_q_check_yield(rkq) &&
                        cnd_timedwait_abs(&rkq->rkq_cond, &rkq->rkq_lock,
-                                         &timeout_tspec) == thrd_success)
+                                         abs_timeout) == thrd_success)
                         ;
 
                 rd_kafka_q_mark_served(rkq);
@@ -787,7 +864,7 @@ int rd_kafka_q_serve_rkmessages(rd_kafka_q_t *rkq,
                 rd_kafka_op_destroy(rko);
         }
 
-        rd_kafka_app_polled(rk);
+        rd_kafka_app_polled(rk, rkq);
 
         return cnt;
 }

package/deps/librdkafka/src/rdkafka_queue.h

@@ -91,6 +91,12 @@ struct rd_kafka_q_s {
          * Shall return 1 if op was handled, else 0. */
         rd_kafka_q_serve_cb_t *rkq_serve;
         void *rkq_opaque;
+        rd_ts_t rkq_ts_last_poll_start; /**< Timestamp of last queue
+                                         *   poll() call start
+                                         *   Only relevant for a consumer. */
+        rd_ts_t rkq_ts_last_poll_end;   /**< Timestamp of last queue
+                                         *   poll() call end
+                                         *   Only relevant for a consumer. */
 
 #if ENABLE_DEVEL
         char rkq_name[64]; /* Debugging: queue name (FUNC:LINE) */
@@ -841,6 +847,13 @@ rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq,
                                     rd_kafka_q_serve_cb_t *callback,
                                     void *opaque);
 rd_kafka_op_t *
+rd_kafka_q_pop_serve_maybe_consume(rd_kafka_q_t *rkq,
+                                   rd_ts_t timeout_us,
+                                   int32_t version,
+                                   rd_kafka_q_cb_type_t cb_type,
+                                   rd_kafka_q_serve_cb_t *callback,
+                                   void *opaque);
+rd_kafka_op_t *
 rd_kafka_q_pop(rd_kafka_q_t *rkq, rd_ts_t timeout_us, int32_t version);
 int rd_kafka_q_serve(rd_kafka_q_t *rkq,
                      int timeout_ms,
@@ -848,6 +861,12 @@ int rd_kafka_q_serve(rd_kafka_q_t *rkq,
                      rd_kafka_q_cb_type_t cb_type,
                      rd_kafka_q_serve_cb_t *callback,
                      void *opaque);
+int rd_kafka_q_serve_maybe_consume(rd_kafka_q_t *rkq,
+                                   int timeout_ms,
+                                   int max_cnt,
+                                   rd_kafka_q_cb_type_t cb_type,
+                                   rd_kafka_q_serve_cb_t *callback,
+                                   void *opaque);
 
 
 int rd_kafka_q_move_cnt(rd_kafka_q_t *dstq,

package/deps/librdkafka/src/rdkafka_range_assignor.c

@@ -1424,7 +1424,7 @@ static int ut_testRackAwareAssignmentWithCoPartitioning0(
         char *subscription12[]    = {"t1", "t2"};
         char *subscription34[]    = {"t3", "t4"};
         char **subscriptions[]    = {subscription12, subscription12,
-                                  subscription34, subscription34};
+                                     subscription34, subscription34};
         int racks[]               = {0, 1, 1, 0};
 
         if (parametrization !=