@platformatic/rdkafka 4.0.0

package/deps/librdkafka/tests/0059-bsearch.cpp

@@ -0,0 +1,237 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2016-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <iostream>
+#include "testcpp.h"
+
+/**
+ * binary search by timestamp: excercices KafkaConsumer's seek() API.
+ */
+
+
+static std::string topic;
+static const int partition      = 0;
+static int64_t golden_timestamp = -1;
+static int64_t golden_offset    = -1;
+
+/**
+ * @brief Seek to offset and consume that message.
+ *
+ * Asserts on failure.
+ */
+static RdKafka::Message *get_msg(RdKafka::KafkaConsumer *c,
+                                 int64_t offset,
+                                 bool use_seek) {
+  RdKafka::TopicPartition *next =
+      RdKafka::TopicPartition::create(topic, partition, offset);
+  RdKafka::ErrorCode err;
+
+  /* Since seek() can only be used to change the currently consumed
+   * offset we need to start consuming the first time we run this
+   * loop by calling assign() */
+
+  test_timing_t t_seek;
+  TIMING_START(&t_seek, "seek");
+  if (!use_seek) {
+    std::vector<RdKafka::TopicPartition *> parts;
+    parts.push_back(next);
+    err = c->assign(parts);
+    if (err)
+      Test::Fail("assign() failed: " + RdKafka::err2str(err));
+  } else {
+    err = c->seek(*next, tmout_multip(5000));
+    if (err)
+      Test::Fail("seek() failed: " + RdKafka::err2str(err));
+  }
+  TIMING_STOP(&t_seek);
+  delete next;
+
+  test_timing_t t_consume;
+  TIMING_START(&t_consume, "consume");
+
+  RdKafka::Message *msg = c->consume(tmout_multip(5000));
+  if (!msg)
+    Test::Fail("consume() returned NULL");
+  TIMING_STOP(&t_consume);
+
+  if (msg->err())
+    Test::Fail("consume() returned error: " + msg->errstr());
+
+  if (msg->offset() != offset)
+    Test::Fail(tostr() << "seek()ed to offset " << offset
+                       << " but consume() returned offset " << msg->offset());
+
+  return msg;
+}
+
+class MyDeliveryReportCb : public RdKafka::DeliveryReportCb {
+ public:
+  void dr_cb(RdKafka::Message &msg) {
+    if (msg.err())
+      Test::Fail("Delivery failed: " + msg.errstr());
+
+    if (!msg.msg_opaque())
+      return;
+
+    RdKafka::MessageTimestamp ts = msg.timestamp();
+    if (ts.type != RdKafka::MessageTimestamp::MSG_TIMESTAMP_CREATE_TIME)
+      Test::Fail(tostr() << "Dr msg timestamp type wrong: " << ts.type);
+
+    golden_timestamp = ts.timestamp;
+    golden_offset    = msg.offset();
+  }
+};
+
+static void do_test_bsearch(void) {
+  RdKafka::Conf *conf, *tconf;
+  int msgcnt = 1000;
+  int64_t timestamp;
+  std::string errstr;
+  RdKafka::ErrorCode err;
+  MyDeliveryReportCb my_dr;
+
+  topic = Test::mk_topic_name("0059-bsearch", 1);
+  Test::conf_init(&conf, &tconf, 0);
+  Test::conf_set(tconf, "acks", "all");
+  Test::conf_set(conf, "api.version.request", "true");
+  conf->set("dr_cb", &my_dr, errstr);
+  conf->set("default_topic_conf", tconf, errstr);
+
+  RdKafka::Producer *p = RdKafka::Producer::create(conf, errstr);
+  if (!p)
+    Test::Fail("Failed to create Producer: " + errstr);
+  delete conf;
+  delete tconf;
+
+  timestamp = 1000;
+  for (int i = 0; i < msgcnt; i++) {
+    err = p->produce(topic, partition, RdKafka::Producer::RK_MSG_COPY,
+                     (void *)topic.c_str(), topic.size(), NULL, 0, timestamp,
+                     i == 357 ? (void *)1 /*golden*/ : NULL);
+    if (err != RdKafka::ERR_NO_ERROR)
+      Test::Fail("Produce failed: " + RdKafka::err2str(err));
+    timestamp += 100 + (timestamp % 9);
+  }
+
+  if (p->flush(tmout_multip(5000)) != 0)
+    Test::Fail("Not all messages flushed");
+
+  Test::Say(tostr() << "Produced " << msgcnt << " messages, "
+                    << "golden message with timestamp " << golden_timestamp
+                    << " at offset " << golden_offset << "\n");
+
+  delete p;
+
+  /*
+   * Now find the golden message using bsearch
+   */
+
+  /* Create consumer */
+  Test::conf_init(&conf, NULL, 10);
+  Test::conf_set(conf, "group.id", topic);
+  Test::conf_set(conf, "api.version.request", "true");
+  Test::conf_set(conf, "fetch.wait.max.ms", "1");
+  Test::conf_set(conf, "fetch.error.backoff.ms", "1");
+  Test::conf_set(conf, "queued.min.messages", "1");
+  Test::conf_set(conf, "enable.auto.commit", "false");
+
+  RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr);
+  if (!c)
+    Test::Fail("Failed to create KafkaConsumer: " + errstr);
+  delete conf;
+
+  Test::Say("Find initial middle offset\n");
+  int64_t low, high;
+  test_timing_t t_qr;
+  TIMING_START(&t_qr, "query_watermark_offsets");
+  err = c->query_watermark_offsets(topic, partition, &low, &high,
+                                   tmout_multip(5000));
+  TIMING_STOP(&t_qr);
+  if (err)
+    Test::Fail("query_watermark_offsets failed: " + RdKafka::err2str(err));
+
+  /* Divide and conquer */
+  test_timing_t t_bsearch;
+  TIMING_START(&t_bsearch, "actual bsearch");
+  int itcnt = 0;
+  do {
+    int64_t mid;
+
+    mid = low + ((high - low) / 2);
+
+    Test::Say(1, tostr() << "Get message at mid point of " << low << ".."
+                         << high << " -> " << mid << "\n");
+
+    RdKafka::Message *msg = get_msg(c, mid,
+                                    /* use assign() on first iteration,
+                                     * then seek() */
+                                    itcnt > 0);
+
+    RdKafka::MessageTimestamp ts = msg->timestamp();
+    if (ts.type != RdKafka::MessageTimestamp::MSG_TIMESTAMP_CREATE_TIME)
+      Test::Fail(tostr() << "Expected CreateTime timestamp, not " << ts.type
+                         << " at offset " << msg->offset());
+
+    Test::Say(1, tostr() << "Message at offset " << msg->offset()
+                         << " with timestamp " << ts.timestamp << "\n");
+
+    if (ts.timestamp == golden_timestamp) {
+      Test::Say(1, tostr() << "Found golden timestamp " << ts.timestamp
+                           << " at offset " << msg->offset() << " in "
+                           << itcnt + 1 << " iterations\n");
+      delete msg;
+      break;
+    }
+
+    if (low == high) {
+      Test::Fail(tostr() << "Search exhausted at offset " << msg->offset()
+                         << " with timestamp " << ts.timestamp
+                         << " without finding golden timestamp "
+                         << golden_timestamp << " at offset " << golden_offset);
+
+    } else if (ts.timestamp < golden_timestamp)
+      low = msg->offset() + 1;
+    else if (ts.timestamp > golden_timestamp)
+      high = msg->offset() - 1;
+
+    delete msg;
+    itcnt++;
+  } while (true);
+  TIMING_STOP(&t_bsearch);
+
+  c->close();
+
+  delete c;
+}
+
+extern "C" {
+int main_0059_bsearch(int argc, char **argv) {
+  do_test_bsearch();
+  return 0;
+}
+}

package/deps/librdkafka/tests/0060-op_prio.cpp

@@ -0,0 +1,163 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2016-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <iostream>
+#include "testcpp.h"
+
+/**
+ * Verify prioritization of non-message ops.
+ * MO:
+ *
+ *  - Seed topic with 1000 messages
+ *  - Start consumer with auto offset commit disabled,
+ *    but with commit and stats callbacks registered,
+ *  - Consume one message
+ *  - Commit that message manually
+ *  - Consume one message per second
+ *  - The commit callback should be fired within reasonable time, long before
+ *  - The stats callback should behave the same.
+ *    all messages are consumed.
+ */
+
+
+
+class MyCbs : public RdKafka::OffsetCommitCb, public RdKafka::EventCb {
+ public:
+  int seen_commit;
+  int seen_stats;
+
+  void offset_commit_cb(RdKafka::ErrorCode err,
+                        std::vector<RdKafka::TopicPartition *> &offsets) {
+    if (err)
+      Test::Fail("Offset commit failed: " + RdKafka::err2str(err));
+
+    seen_commit++;
+    Test::Say("Got commit callback!\n");
+  }
+
+  void event_cb(RdKafka::Event &event) {
+    switch (event.type()) {
+    case RdKafka::Event::EVENT_STATS:
+      Test::Say("Got stats callback!\n");
+      seen_stats++;
+      break;
+    default:
+      break;
+    }
+  }
+};
+
+
+
+static void do_test_commit_cb(void) {
+  const int msgcnt = test_quick ? 100 : 1000;
+  std::string errstr;
+  RdKafka::ErrorCode err;
+  std::string topic = Test::mk_topic_name("0060-op_prio", 1);
+
+  test_produce_msgs_easy(topic.c_str(), 0, 0, msgcnt);
+
+  /*
+   * Create consumer
+   */
+
+  /* Create consumer */
+  RdKafka::Conf *conf;
+  Test::conf_init(&conf, NULL, 10);
+  Test::conf_set(conf, "group.id", topic);
+  Test::conf_set(conf, "socket.timeout.ms", "10000");
+  Test::conf_set(conf, "enable.auto.commit", "false");
+  Test::conf_set(conf, "enable.partition.eof", "false");
+  Test::conf_set(conf, "auto.offset.reset", "earliest");
+  Test::conf_set(conf, "statistics.interval.ms", "1000");
+
+  MyCbs cbs;
+  cbs.seen_commit = 0;
+  cbs.seen_stats  = 0;
+  if (conf->set("offset_commit_cb", (RdKafka::OffsetCommitCb *)&cbs, errstr) !=
+      RdKafka::Conf::CONF_OK)
+    Test::Fail("Failed to set commit callback: " + errstr);
+  if (conf->set("event_cb", (RdKafka::EventCb *)&cbs, errstr) !=
+      RdKafka::Conf::CONF_OK)
+    Test::Fail("Failed to set event callback: " + errstr);
+
+  RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr);
+  if (!c)
+    Test::Fail("Failed to create KafkaConsumer: " + errstr);
+  delete conf;
+
+  /* Subscribe */
+  std::vector<std::string> topics;
+  topics.push_back(topic);
+  if ((err = c->subscribe(topics)))
+    Test::Fail("subscribe failed: " + RdKafka::err2str(err));
+
+  /* Wait for messages and commit callback. */
+  Test::Say("Consuming topic " + topic + "\n");
+  int cnt = 0;
+  while (!cbs.seen_commit || !cbs.seen_stats) {
+    RdKafka::Message *msg = c->consume(tmout_multip(1000));
+    if (!msg->err()) {
+      cnt++;
+      Test::Say(tostr() << "Received message #" << cnt << "\n");
+      if (cnt > 10)
+        Test::Fail(tostr() << "Should've seen the "
+                              "offset commit ("
+                           << cbs.seen_commit
+                           << ") and "
+                              "stats callbacks ("
+                           << cbs.seen_stats << ") by now");
+
+      /* Commit the first message to trigger the offset commit_cb */
+      if (cnt == 1) {
+        err = c->commitAsync(msg);
+        if (err)
+          Test::Fail("commitAsync() failed: " + RdKafka::err2str(err));
+        rd_sleep(1); /* Sleep to simulate slow processing, making sure
+                      * that the offset commit callback op gets
+                      * inserted on the consume queue in front of
+                      * the messages. */
+      }
+
+    } else if (msg->err() == RdKafka::ERR__TIMED_OUT)
+      ; /* Stil rebalancing? */
+    else
+      Test::Fail("consume() failed: " + msg->errstr());
+    delete msg;
+  }
+
+  c->close();
+  delete c;
+}
+
+extern "C" {
+int main_0060_op_prio(int argc, char **argv) {
+  do_test_commit_cb();
+  return 0;
+}
+}

package/deps/librdkafka/tests/0061-consumer_lag.cpp

@@ -0,0 +1,275 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2016-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <iostream>
+#include <cstring>
+#include <cstdlib>
+#include "testcpp.h"
+
+/**
+ * Verify consumer_lag
+ */
+
+static std::string topic;
+
+class StatsCb : public RdKafka::EventCb {
+ public:
+  int64_t calc_lag;  // calculated lag
+  int lag_valid;     // number of times lag has been valid
+
+  StatsCb() {
+    calc_lag  = -1;
+    lag_valid = 0;
+  }
+
+  /**
+   * @brief Event callback
+   */
+  void event_cb(RdKafka::Event &event) {
+    if (event.type() == RdKafka::Event::EVENT_LOG) {
+      Test::Say(tostr() << "LOG-" << event.severity() << "-" << event.fac()
+                        << ": " << event.str() << "\n");
+      return;
+    } else if (event.type() != RdKafka::Event::EVENT_STATS) {
+      Test::Say(tostr() << "Dropping event " << event.type() << "\n");
+      return;
+    }
+
+    int64_t consumer_lag = parse_json(event.str().c_str());
+
+    Test::Say(3, tostr() << "Stats: consumer_lag is " << consumer_lag << "\n");
+    if (consumer_lag == -1) {
+      Test::Say(2, "Skipping old stats with invalid consumer_lag\n");
+      return; /* Old stats generated before first message consumed */
+    } else if (consumer_lag != calc_lag)
+      Test::Fail(tostr() << "Stats consumer_lag " << consumer_lag
+                         << ", expected " << calc_lag << "\n");
+    else
+      lag_valid++;
+  }
+
+
+  /**
+   * @brief Naiive JSON parsing, find the consumer_lag for partition 0
+   * and return it.
+   */
+  static int64_t parse_json(const char *json_doc) {
+    const std::string match_topic(std::string("\"") + topic + "\":");
+    const char *search[] = {
+        "\"topics\":", match_topic.c_str(),        "\"partitions\":",
+        "\"0\":",      "\"consumer_lag_stored\":", NULL};
+    const char *remain = json_doc;
+
+    for (const char **sp = search; *sp; sp++) {
+      const char *t = strstr(remain, *sp);
+      if (!t)
+        Test::Fail(tostr() << "Couldnt find " << *sp
+                           << " in remaining stats output:\n"
+                           << remain << "\n====================\n"
+                           << json_doc << "\n");
+      remain = t + strlen(*sp);
+    }
+
+    while (*remain == ' ')
+      remain++;
+
+    if (!*remain)
+      Test::Fail("Nothing following consumer_lag");
+
+    int64_t lag = strtoull(remain, NULL, 0);
+    if (lag == -1) {
+      Test::Say(tostr() << "Consumer lag " << lag << " is invalid, stats:\n");
+      Test::Say(3, tostr() << json_doc << "\n");
+    }
+    return lag;
+  }
+};
+
+
+/**
+ * @brief Produce \p msgcnt in a transaction that is aborted.
+ */
+static void produce_aborted_txns(const std::string &topic,
+                                 int32_t partition,
+                                 int msgcnt) {
+  RdKafka::Producer *p;
+  RdKafka::Conf *conf;
+  RdKafka::Error *error;
+
+  Test::Say(tostr() << "Producing " << msgcnt << " transactional messages "
+                    << "which will be aborted\n");
+  Test::conf_init(&conf, NULL, 0);
+
+  Test::conf_set(conf, "transactional.id", "txn_id_" + topic);
+
+  std::string errstr;
+  p = RdKafka::Producer::create(conf, errstr);
+  if (!p)
+    Test::Fail("Failed to create Producer:  " + errstr);
+  delete conf;
+
+  error = p->init_transactions(-1);
+  if (error)
+    Test::Fail("init_transactions() failed: " + error->str());
+
+  error = p->begin_transaction();
+  if (error)
+    Test::Fail("begin_transaction() failed: " + error->str());
+
+  for (int i = 0; i < msgcnt; i++) {
+    RdKafka::ErrorCode err;
+
+    err = p->produce(topic, partition, RdKafka::Producer::RK_MSG_COPY, &i,
+                     sizeof(i), NULL, 0, 0, NULL);
+    if (err)
+      Test::Fail("produce() failed: " + RdKafka::err2str(err));
+  }
+
+  /* Flush is typically not needed for transactions since
+   * commit_transaction() will do it automatically, but in the case of
+   * abort_transaction() nothing might have been sent to the broker yet,
+   * so call flush() here so we know the messages are sent and the
+   * partitions are added to the transaction, so that a control(abort)
+   * message is written to the partition. */
+  p->flush(-1);
+
+  error = p->abort_transaction(-1);
+  if (error)
+    Test::Fail("abort_transaction() failed: " + error->str());
+
+  delete p;
+}
+
+
+static void do_test_consumer_lag(bool with_txns) {
+  int msgcnt     = test_quick ? 5 : 10;
+  int txn_msgcnt = 3;
+  int addcnt     = 0;
+  std::string errstr;
+  RdKafka::ErrorCode err;
+
+  SUB_TEST("Test consumer lag %s transactions", with_txns ? "with" : "without");
+
+  topic = Test::mk_topic_name("0061-consumer_lag", 1);
+
+  test_produce_msgs_easy(topic.c_str(), 0, 0, msgcnt);
+
+  if (with_txns) {
+    /* After the standard messages have been produced,
+     * produce some transactional messages that are aborted to advance
+     * the end offset with control messages. */
+    produce_aborted_txns(topic, 0, txn_msgcnt);
+    addcnt = txn_msgcnt + 1 /* ctrl msg */;
+  }
+
+  /*
+   * Create consumer
+   */
+
+  /* Create consumer */
+  RdKafka::Conf *conf;
+  Test::conf_init(&conf, NULL, 40);
+  StatsCb stats;
+  if (conf->set("event_cb", &stats, errstr) != RdKafka::Conf::CONF_OK)
+    Test::Fail("set event_cb failed: " + errstr);
+  Test::conf_set(conf, "group.id", topic);
+  Test::conf_set(conf, "enable.auto.commit", "false");
+  Test::conf_set(conf, "auto.offset.reset", "earliest");
+  Test::conf_set(conf, "statistics.interval.ms", "100");
+
+  RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr);
+  if (!c)
+    Test::Fail("Failed to create KafkaConsumer: " + errstr);
+  delete conf;
+
+  /* Assign partitions */
+  std::vector<RdKafka::TopicPartition *> parts;
+  parts.push_back(RdKafka::TopicPartition::create(topic, 0));
+  if ((err = c->assign(parts)))
+    Test::Fail("assign failed: " + RdKafka::err2str(err));
+  RdKafka::TopicPartition::destroy(parts);
+
+  /* Start consuming */
+  Test::Say("Consuming topic " + topic + "\n");
+  int cnt = 0;
+  while (cnt < msgcnt + addcnt) {
+    RdKafka::Message *msg = c->consume(1000);
+
+    switch (msg->err()) {
+    case RdKafka::ERR__TIMED_OUT:
+      if (with_txns && cnt >= msgcnt && stats.calc_lag == 0)
+        addcnt = 0; /* done */
+      break;
+    case RdKafka::ERR__PARTITION_EOF:
+      Test::Fail(tostr() << "Unexpected PARTITION_EOF (not enbaled) after "
+                         << cnt << "/" << msgcnt
+                         << " messages: " << msg->errstr());
+      break;
+
+    case RdKafka::ERR_NO_ERROR:
+      /* Proper message. Update calculated lag for later
+       * checking in stats callback */
+      if (msg->offset() + 1 >= msgcnt && with_txns)
+        stats.calc_lag = 0;
+      else
+        stats.calc_lag = (msgcnt + addcnt) - (msg->offset() + 1);
+      cnt++;
+      Test::Say(2, tostr() << "Received message #" << cnt << "/" << msgcnt
+                           << " at offset " << msg->offset() << " (calc lag "
+                           << stats.calc_lag << ")\n");
+      /* Slow down message "processing" to make sure we get
+       * at least one stats callback per message. */
+      if (cnt < msgcnt)
+        rd_sleep(1);
+      break;
+
+    default:
+      Test::Fail("Consume error: " + msg->errstr());
+      break;
+    }
+
+    delete msg;
+  }
+  Test::Say(tostr() << "Done, lag was valid " << stats.lag_valid << " times\n");
+  if (stats.lag_valid == 0)
+    Test::Fail("No valid consumer_lag in statistics seen");
+
+  c->close();
+  delete c;
+
+  SUB_TEST_PASS();
+}
+
+extern "C" {
+int main_0061_consumer_lag(int argc, char **argv) {
+  do_test_consumer_lag(false /*no txns*/);
+  if (test_broker_version >= TEST_BRKVER(0, 11, 0, 0))
+    do_test_consumer_lag(true /*txns*/);
+  return 0;
+}
+}