@ohos-ports/confluentinc-kafka-javascript 1.10.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +457 -0
- package/LICENSE.kafkajs +31 -0
- package/LICENSE.node-rdkafka +25 -0
- package/LICENSE.txt +20 -0
- package/README.md +133 -0
- package/binding.gyp +184 -0
- package/build/Release/confluent-kafka-javascript.node +0 -0
- package/build/deps/librdkafka++.so.1 +0 -0
- package/build/deps/librdkafka.Makefile +6 -0
- package/build/deps/librdkafka.so.1 +0 -0
- package/build/deps/librdkafka.target.mk +55 -0
- package/deps/librdkafka/.clang-format +136 -0
- package/deps/librdkafka/.clang-format-cpp +103 -0
- package/deps/librdkafka/.dir-locals.el +10 -0
- package/deps/librdkafka/.formatignore +35 -0
- package/deps/librdkafka/.gdbmacros +19 -0
- package/deps/librdkafka/CHANGELOG.md +2477 -0
- package/deps/librdkafka/CMakeLists.txt +291 -0
- package/deps/librdkafka/CODE_OF_CONDUCT.md +46 -0
- package/deps/librdkafka/CONFIGURATION.md +212 -0
- package/deps/librdkafka/CONTRIBUTING.md +431 -0
- package/deps/librdkafka/Doxyfile +2375 -0
- package/deps/librdkafka/INTRODUCTION.md +2752 -0
- package/deps/librdkafka/LICENSE +26 -0
- package/deps/librdkafka/LICENSE.cjson +22 -0
- package/deps/librdkafka/LICENSE.crc32c +28 -0
- package/deps/librdkafka/LICENSE.fnv1a +18 -0
- package/deps/librdkafka/LICENSE.hdrhistogram +27 -0
- package/deps/librdkafka/LICENSE.lz4 +26 -0
- package/deps/librdkafka/LICENSE.murmur2 +25 -0
- package/deps/librdkafka/LICENSE.nanopb +22 -0
- package/deps/librdkafka/LICENSE.opentelemetry +203 -0
- package/deps/librdkafka/LICENSE.pycrc +23 -0
- package/deps/librdkafka/LICENSE.queue +31 -0
- package/deps/librdkafka/LICENSE.regexp +5 -0
- package/deps/librdkafka/LICENSE.snappy +36 -0
- package/deps/librdkafka/LICENSE.tinycthread +26 -0
- package/deps/librdkafka/LICENSE.wingetopt +49 -0
- package/deps/librdkafka/LICENSES.txt +625 -0
- package/deps/librdkafka/Makefile +125 -0
- package/deps/librdkafka/README.md +201 -0
- package/deps/librdkafka/README.win32 +26 -0
- package/deps/librdkafka/STATISTICS.md +624 -0
- package/deps/librdkafka/configure +214 -0
- package/deps/librdkafka/configure.self +341 -0
- package/deps/librdkafka/dev-conf.sh +123 -0
- package/deps/librdkafka/lds-gen.py +73 -0
- package/deps/librdkafka/mainpage.doxy +40 -0
- package/deps/librdkafka/mklove/Makefile.base +329 -0
- package/deps/librdkafka/mklove/modules/configure.atomics +144 -0
- package/deps/librdkafka/mklove/modules/configure.base +2484 -0
- package/deps/librdkafka/mklove/modules/configure.builtin +70 -0
- package/deps/librdkafka/mklove/modules/configure.cc +186 -0
- package/deps/librdkafka/mklove/modules/configure.cxx +8 -0
- package/deps/librdkafka/mklove/modules/configure.fileversion +65 -0
- package/deps/librdkafka/mklove/modules/configure.gitversion +29 -0
- package/deps/librdkafka/mklove/modules/configure.good_cflags +18 -0
- package/deps/librdkafka/mklove/modules/configure.host +132 -0
- package/deps/librdkafka/mklove/modules/configure.lib +49 -0
- package/deps/librdkafka/mklove/modules/configure.libcurl +99 -0
- package/deps/librdkafka/mklove/modules/configure.libsasl2 +36 -0
- package/deps/librdkafka/mklove/modules/configure.libssl +147 -0
- package/deps/librdkafka/mklove/modules/configure.libzstd +58 -0
- package/deps/librdkafka/mklove/modules/configure.parseversion +95 -0
- package/deps/librdkafka/mklove/modules/configure.pic +16 -0
- package/deps/librdkafka/mklove/modules/configure.socket +20 -0
- package/deps/librdkafka/mklove/modules/configure.zlib +61 -0
- package/deps/librdkafka/mklove/modules/patches/README.md +8 -0
- package/deps/librdkafka/mklove/modules/patches/libcurl.0000-no-runtime-linking-check.patch +11 -0
- package/deps/librdkafka/mklove/modules/patches/libssl.0000-osx-rand-include-fix-OpenSSL-PR16409.patch +56 -0
- package/deps/librdkafka/service.yml +182 -0
- package/deps/librdkafka/src/CMakeLists.txt +379 -0
- package/deps/librdkafka/src/Makefile +105 -0
- package/deps/librdkafka/src/README.lz4.md +30 -0
- package/deps/librdkafka/src/cJSON.c +3191 -0
- package/deps/librdkafka/src/cJSON.h +306 -0
- package/deps/librdkafka/src/crc32c.c +430 -0
- package/deps/librdkafka/src/crc32c.h +38 -0
- package/deps/librdkafka/src/generate_proto.sh +66 -0
- package/deps/librdkafka/src/librdkafka_cgrp_synch.png +0 -0
- package/deps/librdkafka/src/lz4.c +2727 -0
- package/deps/librdkafka/src/lz4.h +842 -0
- package/deps/librdkafka/src/lz4frame.c +2078 -0
- package/deps/librdkafka/src/lz4frame.h +692 -0
- package/deps/librdkafka/src/lz4frame_static.h +47 -0
- package/deps/librdkafka/src/lz4hc.c +1631 -0
- package/deps/librdkafka/src/lz4hc.h +413 -0
- package/deps/librdkafka/src/nanopb/pb.h +917 -0
- package/deps/librdkafka/src/nanopb/pb_common.c +388 -0
- package/deps/librdkafka/src/nanopb/pb_common.h +49 -0
- package/deps/librdkafka/src/nanopb/pb_decode.c +1727 -0
- package/deps/librdkafka/src/nanopb/pb_decode.h +193 -0
- package/deps/librdkafka/src/nanopb/pb_encode.c +1000 -0
- package/deps/librdkafka/src/nanopb/pb_encode.h +185 -0
- package/deps/librdkafka/src/opentelemetry/common.pb.c +32 -0
- package/deps/librdkafka/src/opentelemetry/common.pb.h +170 -0
- package/deps/librdkafka/src/opentelemetry/metrics.options +2 -0
- package/deps/librdkafka/src/opentelemetry/metrics.pb.c +67 -0
- package/deps/librdkafka/src/opentelemetry/metrics.pb.h +966 -0
- package/deps/librdkafka/src/opentelemetry/resource.pb.c +12 -0
- package/deps/librdkafka/src/opentelemetry/resource.pb.h +58 -0
- package/deps/librdkafka/src/queue.h +850 -0
- package/deps/librdkafka/src/rd.h +592 -0
- package/deps/librdkafka/src/rdaddr.c +255 -0
- package/deps/librdkafka/src/rdaddr.h +202 -0
- package/deps/librdkafka/src/rdatomic.h +262 -0
- package/deps/librdkafka/src/rdavg.h +260 -0
- package/deps/librdkafka/src/rdavl.c +210 -0
- package/deps/librdkafka/src/rdavl.h +250 -0
- package/deps/librdkafka/src/rdbase64.c +200 -0
- package/deps/librdkafka/src/rdbase64.h +43 -0
- package/deps/librdkafka/src/rdbuf.c +1884 -0
- package/deps/librdkafka/src/rdbuf.h +375 -0
- package/deps/librdkafka/src/rdcrc32.c +114 -0
- package/deps/librdkafka/src/rdcrc32.h +170 -0
- package/deps/librdkafka/src/rddl.c +179 -0
- package/deps/librdkafka/src/rddl.h +43 -0
- package/deps/librdkafka/src/rdendian.h +175 -0
- package/deps/librdkafka/src/rdfloat.h +67 -0
- package/deps/librdkafka/src/rdfnv1a.c +113 -0
- package/deps/librdkafka/src/rdfnv1a.h +35 -0
- package/deps/librdkafka/src/rdgz.c +120 -0
- package/deps/librdkafka/src/rdgz.h +46 -0
- package/deps/librdkafka/src/rdhdrhistogram.c +721 -0
- package/deps/librdkafka/src/rdhdrhistogram.h +87 -0
- package/deps/librdkafka/src/rdhttp.c +831 -0
- package/deps/librdkafka/src/rdhttp.h +101 -0
- package/deps/librdkafka/src/rdinterval.h +177 -0
- package/deps/librdkafka/src/rdkafka.c +7287 -0
- package/deps/librdkafka/src/rdkafka.h +11795 -0
- package/deps/librdkafka/src/rdkafka_admin.c +10165 -0
- package/deps/librdkafka/src/rdkafka_admin.h +658 -0
- package/deps/librdkafka/src/rdkafka_assignment.c +1282 -0
- package/deps/librdkafka/src/rdkafka_assignment.h +73 -0
- package/deps/librdkafka/src/rdkafka_assignor.c +1800 -0
- package/deps/librdkafka/src/rdkafka_assignor.h +402 -0
- package/deps/librdkafka/src/rdkafka_aux.c +409 -0
- package/deps/librdkafka/src/rdkafka_aux.h +174 -0
- package/deps/librdkafka/src/rdkafka_background.c +222 -0
- package/deps/librdkafka/src/rdkafka_broker.c +6736 -0
- package/deps/librdkafka/src/rdkafka_broker.h +851 -0
- package/deps/librdkafka/src/rdkafka_buf.c +543 -0
- package/deps/librdkafka/src/rdkafka_buf.h +1525 -0
- package/deps/librdkafka/src/rdkafka_cert.c +576 -0
- package/deps/librdkafka/src/rdkafka_cert.h +62 -0
- package/deps/librdkafka/src/rdkafka_cgrp.c +8310 -0
- package/deps/librdkafka/src/rdkafka_cgrp.h +589 -0
- package/deps/librdkafka/src/rdkafka_conf.c +5177 -0
- package/deps/librdkafka/src/rdkafka_conf.h +746 -0
- package/deps/librdkafka/src/rdkafka_confval.h +97 -0
- package/deps/librdkafka/src/rdkafka_coord.c +623 -0
- package/deps/librdkafka/src/rdkafka_coord.h +132 -0
- package/deps/librdkafka/src/rdkafka_error.c +228 -0
- package/deps/librdkafka/src/rdkafka_error.h +80 -0
- package/deps/librdkafka/src/rdkafka_event.c +502 -0
- package/deps/librdkafka/src/rdkafka_event.h +126 -0
- package/deps/librdkafka/src/rdkafka_feature.c +898 -0
- package/deps/librdkafka/src/rdkafka_feature.h +104 -0
- package/deps/librdkafka/src/rdkafka_fetcher.c +3820 -0
- package/deps/librdkafka/src/rdkafka_fetcher.h +62 -0
- package/deps/librdkafka/src/rdkafka_header.c +220 -0
- package/deps/librdkafka/src/rdkafka_header.h +76 -0
- package/deps/librdkafka/src/rdkafka_idempotence.c +807 -0
- package/deps/librdkafka/src/rdkafka_idempotence.h +144 -0
- package/deps/librdkafka/src/rdkafka_int.h +1505 -0
- package/deps/librdkafka/src/rdkafka_interceptor.c +842 -0
- package/deps/librdkafka/src/rdkafka_interceptor.h +104 -0
- package/deps/librdkafka/src/rdkafka_lz4.c +450 -0
- package/deps/librdkafka/src/rdkafka_lz4.h +49 -0
- package/deps/librdkafka/src/rdkafka_metadata.c +2347 -0
- package/deps/librdkafka/src/rdkafka_metadata.h +350 -0
- package/deps/librdkafka/src/rdkafka_metadata_cache.c +1183 -0
- package/deps/librdkafka/src/rdkafka_mock.c +4514 -0
- package/deps/librdkafka/src/rdkafka_mock.h +867 -0
- package/deps/librdkafka/src/rdkafka_mock_cgrp.c +1877 -0
- package/deps/librdkafka/src/rdkafka_mock_handlers.c +5255 -0
- package/deps/librdkafka/src/rdkafka_mock_int.h +1005 -0
- package/deps/librdkafka/src/rdkafka_mock_sharegrp.c +1207 -0
- package/deps/librdkafka/src/rdkafka_msg.c +2619 -0
- package/deps/librdkafka/src/rdkafka_msg.h +629 -0
- package/deps/librdkafka/src/rdkafka_msgbatch.h +62 -0
- package/deps/librdkafka/src/rdkafka_msgset.h +105 -0
- package/deps/librdkafka/src/rdkafka_msgset_reader.c +2099 -0
- package/deps/librdkafka/src/rdkafka_msgset_writer.c +1474 -0
- package/deps/librdkafka/src/rdkafka_offset.c +1565 -0
- package/deps/librdkafka/src/rdkafka_offset.h +150 -0
- package/deps/librdkafka/src/rdkafka_op.c +1219 -0
- package/deps/librdkafka/src/rdkafka_op.h +1065 -0
- package/deps/librdkafka/src/rdkafka_partition.c +5109 -0
- package/deps/librdkafka/src/rdkafka_partition.h +1215 -0
- package/deps/librdkafka/src/rdkafka_pattern.c +228 -0
- package/deps/librdkafka/src/rdkafka_pattern.h +70 -0
- package/deps/librdkafka/src/rdkafka_plugin.c +213 -0
- package/deps/librdkafka/src/rdkafka_plugin.h +41 -0
- package/deps/librdkafka/src/rdkafka_proto.h +739 -0
- package/deps/librdkafka/src/rdkafka_protocol.h +131 -0
- package/deps/librdkafka/src/rdkafka_queue.c +1351 -0
- package/deps/librdkafka/src/rdkafka_queue.h +1235 -0
- package/deps/librdkafka/src/rdkafka_range_assignor.c +1748 -0
- package/deps/librdkafka/src/rdkafka_request.c +7298 -0
- package/deps/librdkafka/src/rdkafka_request.h +761 -0
- package/deps/librdkafka/src/rdkafka_roundrobin_assignor.c +123 -0
- package/deps/librdkafka/src/rdkafka_sasl.c +589 -0
- package/deps/librdkafka/src/rdkafka_sasl.h +63 -0
- package/deps/librdkafka/src/rdkafka_sasl_cyrus.c +722 -0
- package/deps/librdkafka/src/rdkafka_sasl_int.h +89 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer.c +1833 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer.h +52 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer_oidc.c +1827 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer_oidc.h +52 -0
- package/deps/librdkafka/src/rdkafka_sasl_plain.c +142 -0
- package/deps/librdkafka/src/rdkafka_sasl_scram.c +858 -0
- package/deps/librdkafka/src/rdkafka_sasl_win32.c +550 -0
- package/deps/librdkafka/src/rdkafka_share_acknowledgement.c +1445 -0
- package/deps/librdkafka/src/rdkafka_share_acknowledgement.h +373 -0
- package/deps/librdkafka/src/rdkafka_ssl.c +2235 -0
- package/deps/librdkafka/src/rdkafka_ssl.h +86 -0
- package/deps/librdkafka/src/rdkafka_sticky_assignor.c +4785 -0
- package/deps/librdkafka/src/rdkafka_subscription.c +411 -0
- package/deps/librdkafka/src/rdkafka_telemetry.c +761 -0
- package/deps/librdkafka/src/rdkafka_telemetry.h +52 -0
- package/deps/librdkafka/src/rdkafka_telemetry_decode.c +1307 -0
- package/deps/librdkafka/src/rdkafka_telemetry_decode.h +59 -0
- package/deps/librdkafka/src/rdkafka_telemetry_encode.c +1309 -0
- package/deps/librdkafka/src/rdkafka_telemetry_encode.h +459 -0
- package/deps/librdkafka/src/rdkafka_timer.c +407 -0
- package/deps/librdkafka/src/rdkafka_timer.h +117 -0
- package/deps/librdkafka/src/rdkafka_topic.c +2382 -0
- package/deps/librdkafka/src/rdkafka_topic.h +369 -0
- package/deps/librdkafka/src/rdkafka_transport.c +1309 -0
- package/deps/librdkafka/src/rdkafka_transport.h +99 -0
- package/deps/librdkafka/src/rdkafka_transport_int.h +100 -0
- package/deps/librdkafka/src/rdkafka_txnmgr.c +3256 -0
- package/deps/librdkafka/src/rdkafka_txnmgr.h +171 -0
- package/deps/librdkafka/src/rdkafka_zstd.c +226 -0
- package/deps/librdkafka/src/rdkafka_zstd.h +57 -0
- package/deps/librdkafka/src/rdlist.c +593 -0
- package/deps/librdkafka/src/rdlist.h +444 -0
- package/deps/librdkafka/src/rdlog.c +89 -0
- package/deps/librdkafka/src/rdlog.h +41 -0
- package/deps/librdkafka/src/rdmap.c +508 -0
- package/deps/librdkafka/src/rdmap.h +492 -0
- package/deps/librdkafka/src/rdmurmur2.c +167 -0
- package/deps/librdkafka/src/rdmurmur2.h +35 -0
- package/deps/librdkafka/src/rdports.c +61 -0
- package/deps/librdkafka/src/rdports.h +38 -0
- package/deps/librdkafka/src/rdposix.h +250 -0
- package/deps/librdkafka/src/rdrand.c +151 -0
- package/deps/librdkafka/src/rdrand.h +72 -0
- package/deps/librdkafka/src/rdregex.c +156 -0
- package/deps/librdkafka/src/rdregex.h +43 -0
- package/deps/librdkafka/src/rdsignal.h +57 -0
- package/deps/librdkafka/src/rdstring.c +645 -0
- package/deps/librdkafka/src/rdstring.h +98 -0
- package/deps/librdkafka/src/rdsysqueue.h +404 -0
- package/deps/librdkafka/src/rdtime.h +356 -0
- package/deps/librdkafka/src/rdtypes.h +86 -0
- package/deps/librdkafka/src/rdunittest.c +564 -0
- package/deps/librdkafka/src/rdunittest.h +232 -0
- package/deps/librdkafka/src/rdunittest_acknowledge.c +1177 -0
- package/deps/librdkafka/src/rdunittest_fetcher.c +1211 -0
- package/deps/librdkafka/src/rdunittest_msgset_errors.c +1521 -0
- package/deps/librdkafka/src/rdvarint.c +134 -0
- package/deps/librdkafka/src/rdvarint.h +165 -0
- package/deps/librdkafka/src/rdwin32.h +382 -0
- package/deps/librdkafka/src/rdxxhash.c +1030 -0
- package/deps/librdkafka/src/rdxxhash.h +328 -0
- package/deps/librdkafka/src/regexp.c +1352 -0
- package/deps/librdkafka/src/regexp.h +41 -0
- package/deps/librdkafka/src/snappy.c +1866 -0
- package/deps/librdkafka/src/snappy.h +62 -0
- package/deps/librdkafka/src/snappy_compat.h +138 -0
- package/deps/librdkafka/src/statistics_schema.json +444 -0
- package/deps/librdkafka/src/tinycthread.c +932 -0
- package/deps/librdkafka/src/tinycthread.h +503 -0
- package/deps/librdkafka/src/tinycthread_extra.c +210 -0
- package/deps/librdkafka/src/tinycthread_extra.h +218 -0
- package/deps/librdkafka/src/win32_config.h +58 -0
- package/deps/librdkafka/src-cpp/CMakeLists.txt +90 -0
- package/deps/librdkafka/src-cpp/ConfImpl.cpp +84 -0
- package/deps/librdkafka/src-cpp/ConsumerImpl.cpp +244 -0
- package/deps/librdkafka/src-cpp/HandleImpl.cpp +436 -0
- package/deps/librdkafka/src-cpp/HeadersImpl.cpp +48 -0
- package/deps/librdkafka/src-cpp/KafkaConsumerImpl.cpp +296 -0
- package/deps/librdkafka/src-cpp/Makefile +55 -0
- package/deps/librdkafka/src-cpp/MessageImpl.cpp +38 -0
- package/deps/librdkafka/src-cpp/MetadataImpl.cpp +170 -0
- package/deps/librdkafka/src-cpp/ProducerImpl.cpp +197 -0
- package/deps/librdkafka/src-cpp/QueueImpl.cpp +70 -0
- package/deps/librdkafka/src-cpp/README.md +16 -0
- package/deps/librdkafka/src-cpp/RdKafka.cpp +59 -0
- package/deps/librdkafka/src-cpp/TopicImpl.cpp +124 -0
- package/deps/librdkafka/src-cpp/TopicPartitionImpl.cpp +57 -0
- package/deps/librdkafka/src-cpp/rdkafkacpp.h +3797 -0
- package/deps/librdkafka/src-cpp/rdkafkacpp_int.h +1641 -0
- package/deps/librdkafka/tests/0000-unittests.c +72 -0
- package/deps/librdkafka/tests/0001-multiobj.c +102 -0
- package/deps/librdkafka/tests/0002-unkpart.c +244 -0
- package/deps/librdkafka/tests/0003-msgmaxsize.c +173 -0
- package/deps/librdkafka/tests/0004-conf.c +934 -0
- package/deps/librdkafka/tests/0005-order.c +133 -0
- package/deps/librdkafka/tests/0006-symbols.c +163 -0
- package/deps/librdkafka/tests/0007-autotopic.c +136 -0
- package/deps/librdkafka/tests/0008-reqacks.c +179 -0
- package/deps/librdkafka/tests/0009-mock_cluster.c +272 -0
- package/deps/librdkafka/tests/0011-produce_batch.c +753 -0
- package/deps/librdkafka/tests/0012-produce_consume.c +537 -0
- package/deps/librdkafka/tests/0013-null-msgs.c +473 -0
- package/deps/librdkafka/tests/0014-reconsume-191.c +512 -0
- package/deps/librdkafka/tests/0015-offset_seeks.c +172 -0
- package/deps/librdkafka/tests/0016-client_swname.c +181 -0
- package/deps/librdkafka/tests/0017-compression.c +140 -0
- package/deps/librdkafka/tests/0018-cgrp_term.c +444 -0
- package/deps/librdkafka/tests/0019-list_groups.c +289 -0
- package/deps/librdkafka/tests/0020-destroy_hang.c +162 -0
- package/deps/librdkafka/tests/0021-rkt_destroy.c +72 -0
- package/deps/librdkafka/tests/0022-consume_batch.c +331 -0
- package/deps/librdkafka/tests/0025-timers.c +147 -0
- package/deps/librdkafka/tests/0026-consume_pause.c +547 -0
- package/deps/librdkafka/tests/0028-long_topicnames.c +79 -0
- package/deps/librdkafka/tests/0029-assign_offset.c +202 -0
- package/deps/librdkafka/tests/0030-offset_commit.c +589 -0
- package/deps/librdkafka/tests/0031-get_offsets.c +235 -0
- package/deps/librdkafka/tests/0033-regex_subscribe.c +536 -0
- package/deps/librdkafka/tests/0034-offset_reset.c +398 -0
- package/deps/librdkafka/tests/0035-api_version.c +73 -0
- package/deps/librdkafka/tests/0036-partial_fetch.c +87 -0
- package/deps/librdkafka/tests/0037-destroy_hang_local.c +85 -0
- package/deps/librdkafka/tests/0038-performance.c +121 -0
- package/deps/librdkafka/tests/0039-event.c +284 -0
- package/deps/librdkafka/tests/0040-io_event.c +257 -0
- package/deps/librdkafka/tests/0041-fetch_max_bytes.c +97 -0
- package/deps/librdkafka/tests/0042-many_topics.c +252 -0
- package/deps/librdkafka/tests/0043-no_connection.c +77 -0
- package/deps/librdkafka/tests/0044-partition_cnt.c +94 -0
- package/deps/librdkafka/tests/0045-subscribe_update.c +1010 -0
- package/deps/librdkafka/tests/0046-rkt_cache.c +65 -0
- package/deps/librdkafka/tests/0047-partial_buf_tmout.c +98 -0
- package/deps/librdkafka/tests/0048-partitioner.c +283 -0
- package/deps/librdkafka/tests/0049-consume_conn_close.c +162 -0
- package/deps/librdkafka/tests/0050-subscribe_adds.c +145 -0
- package/deps/librdkafka/tests/0051-assign_adds.c +126 -0
- package/deps/librdkafka/tests/0052-msg_timestamps.c +238 -0
- package/deps/librdkafka/tests/0053-stats_cb.cpp +527 -0
- package/deps/librdkafka/tests/0054-offset_time.cpp +236 -0
- package/deps/librdkafka/tests/0055-producer_latency.c +539 -0
- package/deps/librdkafka/tests/0056-balanced_group_mt.c +315 -0
- package/deps/librdkafka/tests/0057-invalid_topic.cpp +112 -0
- package/deps/librdkafka/tests/0058-log.cpp +123 -0
- package/deps/librdkafka/tests/0059-bsearch.cpp +241 -0
- package/deps/librdkafka/tests/0060-op_prio.cpp +163 -0
- package/deps/librdkafka/tests/0061-consumer_lag.cpp +295 -0
- package/deps/librdkafka/tests/0062-stats_event.c +126 -0
- package/deps/librdkafka/tests/0063-clusterid.cpp +180 -0
- package/deps/librdkafka/tests/0064-interceptors.c +529 -0
- package/deps/librdkafka/tests/0065-yield.cpp +140 -0
- package/deps/librdkafka/tests/0066-plugins.cpp +143 -0
- package/deps/librdkafka/tests/0067-empty_topic.cpp +151 -0
- package/deps/librdkafka/tests/0068-produce_timeout.c +136 -0
- package/deps/librdkafka/tests/0069-consumer_add_parts.c +119 -0
- package/deps/librdkafka/tests/0070-null_empty.cpp +197 -0
- package/deps/librdkafka/tests/0072-headers_ut.c +448 -0
- package/deps/librdkafka/tests/0073-headers.c +381 -0
- package/deps/librdkafka/tests/0074-producev.c +87 -0
- package/deps/librdkafka/tests/0075-retry.c +290 -0
- package/deps/librdkafka/tests/0076-produce_retry.c +452 -0
- package/deps/librdkafka/tests/0077-compaction.c +366 -0
- package/deps/librdkafka/tests/0078-c_from_cpp.cpp +96 -0
- package/deps/librdkafka/tests/0079-fork.c +93 -0
- package/deps/librdkafka/tests/0080-admin_ut.c +3095 -0
- package/deps/librdkafka/tests/0081-admin.c +5633 -0
- package/deps/librdkafka/tests/0082-fetch_max_bytes.cpp +137 -0
- package/deps/librdkafka/tests/0083-cb_event.c +233 -0
- package/deps/librdkafka/tests/0084-destroy_flags.c +208 -0
- package/deps/librdkafka/tests/0085-headers.cpp +392 -0
- package/deps/librdkafka/tests/0086-purge.c +368 -0
- package/deps/librdkafka/tests/0088-produce_metadata_timeout.c +162 -0
- package/deps/librdkafka/tests/0089-max_poll_interval.c +511 -0
- package/deps/librdkafka/tests/0090-idempotence.c +171 -0
- package/deps/librdkafka/tests/0091-max_poll_interval_timeout.c +295 -0
- package/deps/librdkafka/tests/0092-mixed_msgver.c +103 -0
- package/deps/librdkafka/tests/0093-holb.c +200 -0
- package/deps/librdkafka/tests/0094-idempotence_msg_timeout.c +231 -0
- package/deps/librdkafka/tests/0095-all_brokers_down.cpp +122 -0
- package/deps/librdkafka/tests/0097-ssl_verify.cpp +674 -0
- package/deps/librdkafka/tests/0098-consumer-txn.cpp +1218 -0
- package/deps/librdkafka/tests/0099-commit_metadata.c +194 -0
- package/deps/librdkafka/tests/0100-thread_interceptors.cpp +195 -0
- package/deps/librdkafka/tests/0101-fetch-from-follower.cpp +446 -0
- package/deps/librdkafka/tests/0102-static_group_rebalance.c +836 -0
- package/deps/librdkafka/tests/0103-transactions.c +1383 -0
- package/deps/librdkafka/tests/0104-fetch_from_follower_mock.c +625 -0
- package/deps/librdkafka/tests/0105-transactions_mock.c +3930 -0
- package/deps/librdkafka/tests/0106-cgrp_sess_timeout.c +318 -0
- package/deps/librdkafka/tests/0107-topic_recreate.c +259 -0
- package/deps/librdkafka/tests/0109-auto_create_topics.cpp +278 -0
- package/deps/librdkafka/tests/0110-batch_size.cpp +182 -0
- package/deps/librdkafka/tests/0111-delay_create_topics.cpp +127 -0
- package/deps/librdkafka/tests/0112-assign_unknown_part.c +87 -0
- package/deps/librdkafka/tests/0113-cooperative_rebalance.cpp +3473 -0
- package/deps/librdkafka/tests/0114-sticky_partitioning.cpp +176 -0
- package/deps/librdkafka/tests/0115-producer_auth.cpp +182 -0
- package/deps/librdkafka/tests/0116-kafkaconsumer_close.cpp +216 -0
- package/deps/librdkafka/tests/0117-mock_errors.c +331 -0
- package/deps/librdkafka/tests/0118-commit_rebalance.c +154 -0
- package/deps/librdkafka/tests/0119-consumer_auth.cpp +167 -0
- package/deps/librdkafka/tests/0120-asymmetric_subscription.c +185 -0
- package/deps/librdkafka/tests/0121-clusterid.c +115 -0
- package/deps/librdkafka/tests/0122-buffer_cleaning_after_rebalance.c +227 -0
- package/deps/librdkafka/tests/0123-connections_max_idle.c +98 -0
- package/deps/librdkafka/tests/0124-openssl_invalid_engine.c +69 -0
- package/deps/librdkafka/tests/0125-immediate_flush.c +144 -0
- package/deps/librdkafka/tests/0126-oauthbearer_oidc.c +1164 -0
- package/deps/librdkafka/tests/0127-fetch_queue_backoff.cpp +165 -0
- package/deps/librdkafka/tests/0128-sasl_callback_queue.cpp +217 -0
- package/deps/librdkafka/tests/0129-fetch_aborted_msgs.c +79 -0
- package/deps/librdkafka/tests/0130-store_offsets.c +178 -0
- package/deps/librdkafka/tests/0131-connect_timeout.c +81 -0
- package/deps/librdkafka/tests/0132-strategy_ordering.c +179 -0
- package/deps/librdkafka/tests/0133-ssl_keys.c +150 -0
- package/deps/librdkafka/tests/0134-ssl_provider.c +92 -0
- package/deps/librdkafka/tests/0135-sasl_credentials.cpp +279 -0
- package/deps/librdkafka/tests/0136-resolve_cb.c +181 -0
- package/deps/librdkafka/tests/0137-barrier_batch_consume.c +752 -0
- package/deps/librdkafka/tests/0138-admin_mock.c +281 -0
- package/deps/librdkafka/tests/0139-offset_validation_mock.c +950 -0
- package/deps/librdkafka/tests/0140-commit_metadata.cpp +108 -0
- package/deps/librdkafka/tests/0142-reauthentication.c +931 -0
- package/deps/librdkafka/tests/0143-exponential_backoff_mock.c +552 -0
- package/deps/librdkafka/tests/0144-idempotence_mock.c +373 -0
- package/deps/librdkafka/tests/0145-pause_resume_mock.c +119 -0
- package/deps/librdkafka/tests/0146-metadata_mock.c +505 -0
- package/deps/librdkafka/tests/0147-consumer_group_consumer_mock.c +1210 -0
- package/deps/librdkafka/tests/0148-offset_fetch_commit_error_mock.c +569 -0
- package/deps/librdkafka/tests/0149-broker-same-host-port.c +140 -0
- package/deps/librdkafka/tests/0150-telemetry_mock.c +651 -0
- package/deps/librdkafka/tests/0151-purge-brokers.c +566 -0
- package/deps/librdkafka/tests/0152-rebootstrap.c +59 -0
- package/deps/librdkafka/tests/0153-memberid.c +128 -0
- package/deps/librdkafka/tests/0155-share_group_heartbeat_mock.c +2695 -0
- package/deps/librdkafka/tests/0156-share_consumer_fetch_mock.c +1551 -0
- package/deps/librdkafka/tests/0157-share_consumer_ack_mock.c +1788 -0
- package/deps/librdkafka/tests/0158-share_consumer_transactions_mock.c +467 -0
- package/deps/librdkafka/tests/0170-share_consumer_subscription.c +2192 -0
- package/deps/librdkafka/tests/0171-share_consumer_consume.c +2150 -0
- package/deps/librdkafka/tests/0172-share_consumer_acknowledge.c +1919 -0
- package/deps/librdkafka/tests/0173-share_consumer_commit_async.c +2735 -0
- package/deps/librdkafka/tests/0174-share_consumer_concurrency.c +1434 -0
- package/deps/librdkafka/tests/0175-share_consumer_groups.c +574 -0
- package/deps/librdkafka/tests/0176-share_consumer_commit_sync.c +2986 -0
- package/deps/librdkafka/tests/0177-share_consumer_transactions.c +1815 -0
- package/deps/librdkafka/tests/0178-share_consumer_close.c +2102 -0
- package/deps/librdkafka/tests/0179-share_consumer_destroy.c +2482 -0
- package/deps/librdkafka/tests/0180-share_consumer_config.c +1677 -0
- package/deps/librdkafka/tests/0181-share_consumer_topic_delete.c +354 -0
- package/deps/librdkafka/tests/0182-share_consumer_error_handling_mock.c +3358 -0
- package/deps/librdkafka/tests/0183-share_consumer_leader_change_mock.c +1234 -0
- package/deps/librdkafka/tests/0184-share_consumer_topic_recreate.c +1356 -0
- package/deps/librdkafka/tests/0185-share_consumer_max_poll_interval.c +557 -0
- package/deps/librdkafka/tests/0186-share_consumer_fatal_error.c +360 -0
- package/deps/librdkafka/tests/0190-share_consumer_telemetry.c +387 -0
- package/deps/librdkafka/tests/1000-unktopic.c +164 -0
- package/deps/librdkafka/tests/8000-idle.cpp +60 -0
- package/deps/librdkafka/tests/8001-fetch_from_follower_mock_manual.c +113 -0
- package/deps/librdkafka/tests/CMakeLists.txt +193 -0
- package/deps/librdkafka/tests/LibrdkafkaTestApp.py +291 -0
- package/deps/librdkafka/tests/Makefile +182 -0
- package/deps/librdkafka/tests/README.md +514 -0
- package/deps/librdkafka/tests/autotest.sh +33 -0
- package/deps/librdkafka/tests/backtrace.gdb +30 -0
- package/deps/librdkafka/tests/broker_version_tests.py +315 -0
- package/deps/librdkafka/tests/buildbox.sh +17 -0
- package/deps/librdkafka/tests/cleanup-checker-tests.sh +20 -0
- package/deps/librdkafka/tests/cluster_testing.py +203 -0
- package/deps/librdkafka/tests/delete-test-topics.sh +56 -0
- package/deps/librdkafka/tests/fixtures/oauthbearer/jwt_assertion_template.json +10 -0
- package/deps/librdkafka/tests/fixtures/ssl/Makefile +8 -0
- package/deps/librdkafka/tests/fixtures/ssl/README.md +13 -0
- package/deps/librdkafka/tests/fixtures/ssl/client.keystore.intermediate.p12 +0 -0
- package/deps/librdkafka/tests/fixtures/ssl/client.keystore.p12 +0 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.certificate.intermediate.pem +72 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.certificate.pem +50 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.intermediate.key +46 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.key +46 -0
- package/deps/librdkafka/tests/fixtures/ssl/create_keys.sh +168 -0
- package/deps/librdkafka/tests/fuzzers/Makefile +12 -0
- package/deps/librdkafka/tests/fuzzers/README.md +31 -0
- package/deps/librdkafka/tests/fuzzers/fuzz_regex.c +74 -0
- package/deps/librdkafka/tests/fuzzers/helpers.h +90 -0
- package/deps/librdkafka/tests/gen-ssl-certs.sh +165 -0
- package/deps/librdkafka/tests/interactive_broker_version.py +170 -0
- package/deps/librdkafka/tests/interceptor_test/CMakeLists.txt +16 -0
- package/deps/librdkafka/tests/interceptor_test/Makefile +22 -0
- package/deps/librdkafka/tests/interceptor_test/interceptor_test.c +314 -0
- package/deps/librdkafka/tests/interceptor_test/interceptor_test.h +54 -0
- package/deps/librdkafka/tests/java/IncrementalRebalanceCli.java +97 -0
- package/deps/librdkafka/tests/java/Makefile +13 -0
- package/deps/librdkafka/tests/java/Murmur2Cli.java +46 -0
- package/deps/librdkafka/tests/java/README.md +14 -0
- package/deps/librdkafka/tests/java/TransactionProducerCli.java +162 -0
- package/deps/librdkafka/tests/java/run-class.sh +11 -0
- package/deps/librdkafka/tests/librdkafka.suppressions +483 -0
- package/deps/librdkafka/tests/lz4_manual_test.sh +59 -0
- package/deps/librdkafka/tests/multi-broker-version-test.sh +50 -0
- package/deps/librdkafka/tests/parse-refcnt.sh +43 -0
- package/deps/librdkafka/tests/performance_plot.py +115 -0
- package/deps/librdkafka/tests/plugin_test/Makefile +19 -0
- package/deps/librdkafka/tests/plugin_test/plugin_test.c +58 -0
- package/deps/librdkafka/tests/requirements.txt +2 -0
- package/deps/librdkafka/tests/run-all-tests.sh +81 -0
- package/deps/librdkafka/tests/run-consumer-tests.sh +16 -0
- package/deps/librdkafka/tests/run-producer-tests.sh +16 -0
- package/deps/librdkafka/tests/run-test-batches.py +157 -0
- package/deps/librdkafka/tests/run-test.sh +141 -0
- package/deps/librdkafka/tests/rusage.c +249 -0
- package/deps/librdkafka/tests/sasl_test.py +289 -0
- package/deps/librdkafka/tests/scenarios/README.md +6 -0
- package/deps/librdkafka/tests/scenarios/ak23.json +6 -0
- package/deps/librdkafka/tests/scenarios/default.json +5 -0
- package/deps/librdkafka/tests/scenarios/noautocreate.json +5 -0
- package/deps/librdkafka/tests/share_telemetry_verify/share-otel-collector-config.yaml +28 -0
- package/deps/librdkafka/tests/sockem.c +801 -0
- package/deps/librdkafka/tests/sockem.h +85 -0
- package/deps/librdkafka/tests/sockem_ctrl.c +145 -0
- package/deps/librdkafka/tests/sockem_ctrl.h +61 -0
- package/deps/librdkafka/tests/test.c +8376 -0
- package/deps/librdkafka/tests/test.conf.example +27 -0
- package/deps/librdkafka/tests/test.h +1150 -0
- package/deps/librdkafka/tests/testcpp.cpp +131 -0
- package/deps/librdkafka/tests/testcpp.h +388 -0
- package/deps/librdkafka/tests/testshared.h +416 -0
- package/deps/librdkafka/tests/tools/README.md +4 -0
- package/deps/librdkafka/tests/tools/stats/README.md +21 -0
- package/deps/librdkafka/tests/tools/stats/filter.jq +42 -0
- package/deps/librdkafka/tests/tools/stats/graph.py +150 -0
- package/deps/librdkafka/tests/tools/stats/requirements.txt +3 -0
- package/deps/librdkafka/tests/tools/stats/to_csv.py +124 -0
- package/deps/librdkafka/tests/trivup/trivup-0.15.0.tar.gz +0 -0
- package/deps/librdkafka/tests/until-fail.sh +87 -0
- package/deps/librdkafka/tests/xxxx-assign_partition.c +122 -0
- package/deps/librdkafka/tests/xxxx-metadata.cpp +159 -0
- package/deps/librdkafka/vcpkg.json +23 -0
- package/deps/librdkafka/win32/README.md +5 -0
- package/deps/librdkafka/win32/build-package.bat +3 -0
- package/deps/librdkafka/win32/build.bat +19 -0
- package/deps/librdkafka/win32/common.vcxproj +84 -0
- package/deps/librdkafka/win32/interceptor_test/interceptor_test.vcxproj +87 -0
- package/deps/librdkafka/win32/librdkafka.autopkg.template +54 -0
- package/deps/librdkafka/win32/librdkafka.master.testing.targets +13 -0
- package/deps/librdkafka/win32/librdkafka.sln +226 -0
- package/deps/librdkafka/win32/librdkafka.vcxproj +282 -0
- package/deps/librdkafka/win32/librdkafkacpp/librdkafkacpp.vcxproj +104 -0
- package/deps/librdkafka/win32/msbuild.ps1 +58 -0
- package/deps/librdkafka/win32/openssl_engine_example/openssl_engine_example.vcxproj +132 -0
- package/deps/librdkafka/win32/package-zip.ps1 +46 -0
- package/deps/librdkafka/win32/packages/repositories.config +4 -0
- package/deps/librdkafka/win32/push-package.bat +4 -0
- package/deps/librdkafka/win32/rdkafka_complex_consumer_example_cpp/rdkafka_complex_consumer_example_cpp.vcxproj +67 -0
- package/deps/librdkafka/win32/rdkafka_example/rdkafka_example.vcxproj +97 -0
- package/deps/librdkafka/win32/rdkafka_performance/rdkafka_performance.vcxproj +97 -0
- package/deps/librdkafka/win32/setup-msys2.ps1 +47 -0
- package/deps/librdkafka/win32/setup-vcpkg.ps1 +34 -0
- package/deps/librdkafka/win32/tests/test.conf.example +25 -0
- package/deps/librdkafka/win32/tests/tests.vcxproj +278 -0
- package/deps/librdkafka/win32/win_ssl_cert_store/win_ssl_cert_store.vcxproj +132 -0
- package/deps/librdkafka/win32/wingetopt.c +564 -0
- package/deps/librdkafka/win32/wingetopt.h +101 -0
- package/deps/librdkafka/win32/wintime.h +33 -0
- package/deps/librdkafka.gyp +83 -0
- package/deps/windows-install.py +80 -0
- package/index.d.ts +3 -0
- package/lib/admin.js +732 -0
- package/lib/client.js +679 -0
- package/lib/error.js +501 -0
- package/lib/index.js +8 -0
- package/lib/kafka-consumer-stream.js +399 -0
- package/lib/kafka-consumer.js +798 -0
- package/lib/kafkajs/_admin.js +1052 -0
- package/lib/kafkajs/_common.js +887 -0
- package/lib/kafkajs/_consumer.js +2156 -0
- package/lib/kafkajs/_consumer_cache.js +310 -0
- package/lib/kafkajs/_error.js +298 -0
- package/lib/kafkajs/_kafka.js +125 -0
- package/lib/kafkajs/_linked-list.js +220 -0
- package/lib/kafkajs/_producer.js +863 -0
- package/lib/kafkajs/index.js +5 -0
- package/lib/producer/high-level-producer.js +400 -0
- package/lib/producer-stream.js +337 -0
- package/lib/producer.js +405 -0
- package/lib/rdkafka.js +44 -0
- package/lib/tools/ref-counter.js +53 -0
- package/lib/topic-partition.js +91 -0
- package/lib/topic.js +43 -0
- package/lib/util.js +55 -0
- package/librdkafka.js +12 -0
- package/package.json +85 -0
- package/shim/source_location +54 -0
- package/src/admin.cc +1597 -0
- package/src/admin.h +112 -0
- package/src/binding.cc +76 -0
- package/src/binding.h +25 -0
- package/src/callbacks.cc +676 -0
- package/src/callbacks.h +302 -0
- package/src/common.cc +1628 -0
- package/src/common.h +192 -0
- package/src/config.cc +219 -0
- package/src/config.h +61 -0
- package/src/connection.cc +703 -0
- package/src/connection.h +117 -0
- package/src/errors.cc +119 -0
- package/src/errors.h +60 -0
- package/src/kafka-consumer.cc +1539 -0
- package/src/kafka-consumer.h +147 -0
- package/src/producer.cc +917 -0
- package/src/producer.h +124 -0
- package/src/topic.cc +173 -0
- package/src/topic.h +54 -0
- package/src/workers.cc +1714 -0
- package/src/workers.h +690 -0
- package/types/config.d.ts +1269 -0
- package/types/errors.d.ts +373 -0
- package/types/kafkajs.d.ts +568 -0
- package/types/rdkafka.d.ts +571 -0
- package/util/configure.js +32 -0
- package/util/generate-docs.sh +18 -0
- package/util/get-env.js +6 -0
- package/util/test-compile.js +11 -0
- package/util/test-producer-delivery.js +100 -0
|
@@ -0,0 +1,2482 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* librdkafka - Apache Kafka C library
|
|
3
|
+
*
|
|
4
|
+
* Copyright (c) 2026, Confluent Inc.
|
|
5
|
+
* All rights reserved.
|
|
6
|
+
*
|
|
7
|
+
* Redistribution and use in source and binary forms, with or without
|
|
8
|
+
* modification, are permitted provided that the following conditions are met:
|
|
9
|
+
*
|
|
10
|
+
* 1. Redistributions of source code must retain the above copyright notice,
|
|
11
|
+
* this list of conditions and the following disclaimer.
|
|
12
|
+
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
13
|
+
* this list of conditions and the following disclaimer in the documentation
|
|
14
|
+
* and/or other materials provided with the distribution.
|
|
15
|
+
*
|
|
16
|
+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
17
|
+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
18
|
+
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
19
|
+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
|
20
|
+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
21
|
+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
22
|
+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
23
|
+
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
24
|
+
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
25
|
+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
26
|
+
* POSSIBILITY OF SUCH DAMAGE.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
#include "test.h"
|
|
30
|
+
#include "../src/rdkafka.h"
|
|
31
|
+
#include "../src/rdkafka_proto.h"
|
|
32
|
+
|
|
33
|
+
#define CONSUME_ARRAY 1000
|
|
34
|
+
#define TEST_MSGS 100
|
|
35
|
+
#define MAX_CONSUME_ATTEMPTS 30
|
|
36
|
+
#define MAX_PARTITIONS 32
|
|
37
|
+
|
|
38
|
+
/* Shared producer/admin handles for real-broker tests. Created in
|
|
39
|
+
* main_0179_share_consumer_destroy and reused across sub-tests. */
|
|
40
|
+
static rd_kafka_t *common_producer;
|
|
41
|
+
static rd_kafka_t *common_admin;
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
/****************************************************************************
|
|
45
|
+
* Acknowledgement Callback Tracking Helpers
|
|
46
|
+
****************************************************************************/
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* @brief Create a share consumer for the mock-broker tests in this file.
|
|
50
|
+
*
|
|
51
|
+
* Points at the supplied mock bootstraps (test_create_share_consumer
|
|
52
|
+
* cannot, since it reads the real-broker bootstrap globals), sets
|
|
53
|
+
* a fast topic.metadata.refresh.interval.ms (500) so the client
|
|
54
|
+
* observes mock metadata changes quickly, and conditionally enables
|
|
55
|
+
* explicit ack mode.
|
|
56
|
+
*/
|
|
57
|
+
/**
|
|
58
|
+
* @brief One per-partition entry from a share-ack callback invocation.
|
|
59
|
+
* Each callback may report N partition entries; we record one
|
|
60
|
+
* ack_receipt_t per (topic, partition) entry, carrying the
|
|
61
|
+
* callback-level err and the offsets count for that partition.
|
|
62
|
+
*/
|
|
63
|
+
typedef struct {
|
|
64
|
+
char *topic;
|
|
65
|
+
int32_t partition;
|
|
66
|
+
int offset_cnt;
|
|
67
|
+
rd_kafka_resp_err_t err;
|
|
68
|
+
} ack_receipt_t;
|
|
69
|
+
|
|
70
|
+
typedef struct {
|
|
71
|
+
ack_receipt_t *receipts;
|
|
72
|
+
int receipt_cnt;
|
|
73
|
+
int receipt_capacity;
|
|
74
|
+
int callback_invocations;
|
|
75
|
+
} ack_receipts_t;
|
|
76
|
+
|
|
77
|
+
static void ack_receipts_init(ack_receipts_t *r) {
|
|
78
|
+
memset(r, 0, sizeof(*r));
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
static void ack_receipts_destroy(ack_receipts_t *r) {
|
|
82
|
+
int i;
|
|
83
|
+
for (i = 0; i < r->receipt_cnt; i++)
|
|
84
|
+
rd_free(r->receipts[i].topic);
|
|
85
|
+
if (r->receipts)
|
|
86
|
+
rd_free(r->receipts);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
static void ack_receipts_add(ack_receipts_t *r,
|
|
90
|
+
const char *topic,
|
|
91
|
+
int32_t partition,
|
|
92
|
+
int offset_cnt,
|
|
93
|
+
rd_kafka_resp_err_t err) {
|
|
94
|
+
if (r->receipt_cnt == r->receipt_capacity) {
|
|
95
|
+
int new_cap =
|
|
96
|
+
r->receipt_capacity ? r->receipt_capacity * 2 : 16;
|
|
97
|
+
r->receipts =
|
|
98
|
+
rd_realloc(r->receipts, new_cap * sizeof(*r->receipts));
|
|
99
|
+
r->receipt_capacity = new_cap;
|
|
100
|
+
}
|
|
101
|
+
r->receipts[r->receipt_cnt].topic = rd_strdup(topic);
|
|
102
|
+
r->receipts[r->receipt_cnt].partition = partition;
|
|
103
|
+
r->receipts[r->receipt_cnt].offset_cnt = offset_cnt;
|
|
104
|
+
r->receipts[r->receipt_cnt].err = err;
|
|
105
|
+
r->receipt_cnt++;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* @brief Share-ack callback that funnels each partition entry into
|
|
110
|
+
* the provided ack_receipts_t (passed as opaque).
|
|
111
|
+
*/
|
|
112
|
+
static void record_share_ack_cb(rd_kafka_share_t *rkshare,
|
|
113
|
+
rd_kafka_share_partition_offsets_list_t *parts,
|
|
114
|
+
rd_kafka_resp_err_t err,
|
|
115
|
+
void *opaque) {
|
|
116
|
+
ack_receipts_t *r = opaque;
|
|
117
|
+
size_t pcnt, p;
|
|
118
|
+
|
|
119
|
+
(void)rkshare;
|
|
120
|
+
|
|
121
|
+
r->callback_invocations++;
|
|
122
|
+
|
|
123
|
+
pcnt = rd_kafka_share_partition_offsets_list_count(parts);
|
|
124
|
+
TEST_SAY("ack_cb invocation #%d: err=%s partitions=%zu\n",
|
|
125
|
+
r->callback_invocations, rd_kafka_err2name(err), pcnt);
|
|
126
|
+
|
|
127
|
+
for (p = 0; p < pcnt; p++) {
|
|
128
|
+
const rd_kafka_share_partition_offsets_t *entry =
|
|
129
|
+
rd_kafka_share_partition_offsets_list_get(parts, p);
|
|
130
|
+
const rd_kafka_topic_partition_t *tp;
|
|
131
|
+
int ocnt;
|
|
132
|
+
if (!entry)
|
|
133
|
+
continue;
|
|
134
|
+
tp = rd_kafka_share_partition_offsets_partition(entry);
|
|
135
|
+
ocnt = rd_kafka_share_partition_offsets_offsets_cnt(entry);
|
|
136
|
+
TEST_SAY(" %s [%" PRId32 "] offsets=%d err=%s\n", tp->topic,
|
|
137
|
+
tp->partition, ocnt, rd_kafka_err2name(err));
|
|
138
|
+
ack_receipts_add(r, tp->topic, tp->partition, ocnt, err);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* @brief One expected (topic, partition, err) group. The assertion
|
|
144
|
+
* is: across all recorded receipts whose
|
|
145
|
+
* (topic, partition, err) match this entry, the sum of
|
|
146
|
+
* offset_cnt equals expected_offset_cnt. Multiple receipts
|
|
147
|
+
* for the same (topic, partition, err) — possible if the
|
|
148
|
+
* callback fires more than once for the same partition — are
|
|
149
|
+
* summed.
|
|
150
|
+
*
|
|
151
|
+
* Receipts that don't match any expected group cause a test
|
|
152
|
+
* failure.
|
|
153
|
+
*/
|
|
154
|
+
typedef struct {
|
|
155
|
+
const char *topic;
|
|
156
|
+
int32_t partition;
|
|
157
|
+
rd_kafka_resp_err_t err;
|
|
158
|
+
int expected_offset_cnt;
|
|
159
|
+
} expected_ack_t;
|
|
160
|
+
|
|
161
|
+
static void verify_ack_receipts(ack_receipts_t *r,
|
|
162
|
+
const expected_ack_t *expected,
|
|
163
|
+
int expected_cnt,
|
|
164
|
+
const char *label) {
|
|
165
|
+
int i, j;
|
|
166
|
+
|
|
167
|
+
TEST_SAY(
|
|
168
|
+
"Verifying ack receipts (%s): %d receipts from %d "
|
|
169
|
+
"invocation(s), %d expected (topic, partition, err) group(s)\n",
|
|
170
|
+
label, r->receipt_cnt, r->callback_invocations, expected_cnt);
|
|
171
|
+
|
|
172
|
+
/* Assert each expected group is satisfied. */
|
|
173
|
+
for (j = 0; j < expected_cnt; j++) {
|
|
174
|
+
int observed_offset_cnt = 0;
|
|
175
|
+
int matching_receipts = 0;
|
|
176
|
+
|
|
177
|
+
for (i = 0; i < r->receipt_cnt; i++) {
|
|
178
|
+
if (r->receipts[i].partition == expected[j].partition &&
|
|
179
|
+
r->receipts[i].err == expected[j].err &&
|
|
180
|
+
strcmp(r->receipts[i].topic, expected[j].topic) ==
|
|
181
|
+
0) {
|
|
182
|
+
observed_offset_cnt +=
|
|
183
|
+
r->receipts[i].offset_cnt;
|
|
184
|
+
matching_receipts++;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
if (observed_offset_cnt != expected[j].expected_offset_cnt) {
|
|
189
|
+
TEST_FAIL(
|
|
190
|
+
"ack receipts (%s): expected %d offsets for "
|
|
191
|
+
"%s [%" PRId32
|
|
192
|
+
"] with err=%s, got %d (across "
|
|
193
|
+
"%d receipt(s))",
|
|
194
|
+
label, expected[j].expected_offset_cnt,
|
|
195
|
+
expected[j].topic, expected[j].partition,
|
|
196
|
+
rd_kafka_err2name(expected[j].err),
|
|
197
|
+
observed_offset_cnt, matching_receipts);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
TEST_SAY(" OK: %s [%" PRId32
|
|
201
|
+
"] err=%s -> %d offsets across %d receipt(s)\n",
|
|
202
|
+
expected[j].topic, expected[j].partition,
|
|
203
|
+
rd_kafka_err2name(expected[j].err),
|
|
204
|
+
observed_offset_cnt, matching_receipts);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/* Flag any receipt that didn't match any expected group. */
|
|
208
|
+
for (i = 0; i < r->receipt_cnt; i++) {
|
|
209
|
+
rd_bool_t matched = rd_false;
|
|
210
|
+
for (j = 0; j < expected_cnt; j++) {
|
|
211
|
+
if (r->receipts[i].partition == expected[j].partition &&
|
|
212
|
+
r->receipts[i].err == expected[j].err &&
|
|
213
|
+
strcmp(r->receipts[i].topic, expected[j].topic) ==
|
|
214
|
+
0) {
|
|
215
|
+
matched = rd_true;
|
|
216
|
+
break;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
if (!matched) {
|
|
220
|
+
TEST_FAIL(
|
|
221
|
+
"ack receipts (%s): unexpected receipt for "
|
|
222
|
+
"%s [%" PRId32 "] err=%s offsets=%d",
|
|
223
|
+
label, r->receipts[i].topic,
|
|
224
|
+
r->receipts[i].partition,
|
|
225
|
+
rd_kafka_err2name(r->receipts[i].err),
|
|
226
|
+
r->receipts[i].offset_cnt);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* @brief Drain pending ack callbacks from rk_rep by polling.
|
|
233
|
+
*
|
|
234
|
+
* Polls in 1s ticks for up to \p max_seconds. Stops early if
|
|
235
|
+
* the callback invocation counter advances past
|
|
236
|
+
* \p expected_invocations.
|
|
237
|
+
*/
|
|
238
|
+
static void drain_ack_callbacks(rd_kafka_share_t *rkshare) {
|
|
239
|
+
int max_seconds = 10;
|
|
240
|
+
rd_kafka_t *rk = test_share_consumer_get_rk(rkshare);
|
|
241
|
+
int waited;
|
|
242
|
+
for (waited = 0; waited < max_seconds; waited++) {
|
|
243
|
+
rd_kafka_poll(rk, 0);
|
|
244
|
+
rd_sleep(1);
|
|
245
|
+
}
|
|
246
|
+
/* One final poll for any callback that fired during the
|
|
247
|
+
* last sleep. */
|
|
248
|
+
rd_kafka_poll(rk, 0);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/****************************************************************************
|
|
252
|
+
* General Helpers
|
|
253
|
+
****************************************************************************/
|
|
254
|
+
|
|
255
|
+
static rd_kafka_share_t *
|
|
256
|
+
new_share_consumer_for_mock_test(const char *bootstraps,
|
|
257
|
+
const char *group_id,
|
|
258
|
+
rd_bool_t explicit_ack,
|
|
259
|
+
ack_receipts_t *receipts) {
|
|
260
|
+
rd_kafka_conf_t *conf;
|
|
261
|
+
rd_kafka_share_t *consumer;
|
|
262
|
+
|
|
263
|
+
test_conf_init(&conf, NULL, 0);
|
|
264
|
+
test_conf_set(conf, "bootstrap.servers", bootstraps);
|
|
265
|
+
test_conf_set(conf, "group.id", group_id);
|
|
266
|
+
test_conf_set(conf, "topic.metadata.refresh.interval.ms", "500");
|
|
267
|
+
if (explicit_ack)
|
|
268
|
+
test_conf_set(conf, "share.acknowledgement.mode", "explicit");
|
|
269
|
+
|
|
270
|
+
consumer = rd_kafka_share_consumer_new(conf, NULL, 0);
|
|
271
|
+
TEST_ASSERT(consumer != NULL, "Failed to create share consumer");
|
|
272
|
+
|
|
273
|
+
if (receipts) {
|
|
274
|
+
rd_kafka_error_t *error =
|
|
275
|
+
rd_kafka_share_set_acknowledgement_commit_cb(
|
|
276
|
+
consumer, record_share_ack_cb, receipts);
|
|
277
|
+
TEST_ASSERT(error == NULL,
|
|
278
|
+
"Failed to set acknowledgement commit callback: "
|
|
279
|
+
"%s",
|
|
280
|
+
rd_kafka_error_string(error));
|
|
281
|
+
}
|
|
282
|
+
return consumer;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* @brief Create a share consumer for the real-broker tests in this file.
|
|
288
|
+
*
|
|
289
|
+
* Like test_create_share_consumer but additionally wires the
|
|
290
|
+
* ack-callback (record_share_ack_cb) when @p receipts is
|
|
291
|
+
* non-NULL.
|
|
292
|
+
*/
|
|
293
|
+
static rd_kafka_share_t *
|
|
294
|
+
new_share_consumer_for_real_test(const char *group_id,
|
|
295
|
+
const char *ack_mode,
|
|
296
|
+
ack_receipts_t *receipts) {
|
|
297
|
+
rd_kafka_conf_t *conf;
|
|
298
|
+
rd_kafka_share_t *consumer;
|
|
299
|
+
char errstr[512];
|
|
300
|
+
|
|
301
|
+
test_conf_init(&conf, NULL, 0);
|
|
302
|
+
rd_kafka_conf_set(conf, "group.id", group_id, errstr, sizeof(errstr));
|
|
303
|
+
rd_kafka_conf_set(conf, "share.acknowledgement.mode", ack_mode, errstr,
|
|
304
|
+
sizeof(errstr));
|
|
305
|
+
|
|
306
|
+
consumer = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr));
|
|
307
|
+
TEST_ASSERT(consumer, "Failed to create share consumer: %s", errstr);
|
|
308
|
+
|
|
309
|
+
if (receipts) {
|
|
310
|
+
rd_kafka_error_t *error =
|
|
311
|
+
rd_kafka_share_set_acknowledgement_commit_cb(
|
|
312
|
+
consumer, record_share_ack_cb, receipts);
|
|
313
|
+
TEST_ASSERT(error == NULL,
|
|
314
|
+
"Failed to set acknowledgement commit callback: "
|
|
315
|
+
"%s",
|
|
316
|
+
rd_kafka_error_string(error));
|
|
317
|
+
}
|
|
318
|
+
return consumer;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* @brief Subscribe to topics.
|
|
323
|
+
*/
|
|
324
|
+
static void subscribe_topics(rd_kafka_share_t *consumer,
|
|
325
|
+
const char **topics,
|
|
326
|
+
int topic_cnt) {
|
|
327
|
+
rd_kafka_topic_partition_list_t *tpl =
|
|
328
|
+
rd_kafka_topic_partition_list_new(topic_cnt);
|
|
329
|
+
for (int i = 0; i < topic_cnt; i++) {
|
|
330
|
+
rd_kafka_topic_partition_list_add(tpl, topics[i],
|
|
331
|
+
RD_KAFKA_PARTITION_UA);
|
|
332
|
+
}
|
|
333
|
+
TEST_ASSERT(!rd_kafka_share_subscribe(consumer, tpl),
|
|
334
|
+
"Subscribe failed");
|
|
335
|
+
rd_kafka_topic_partition_list_destroy(tpl);
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* @brief Acknowledge messages in [start, end) and log each result.
|
|
341
|
+
*/
|
|
342
|
+
static void ack_range_logged(rd_kafka_share_t *rkshare,
|
|
343
|
+
rd_kafka_messages_t *batch,
|
|
344
|
+
int start,
|
|
345
|
+
int end,
|
|
346
|
+
int rcvd) {
|
|
347
|
+
int i;
|
|
348
|
+
for (i = start; i < end && i < rcvd; i++) {
|
|
349
|
+
rd_kafka_message_t *rkm = rd_kafka_messages_get(batch, i);
|
|
350
|
+
rd_kafka_resp_err_t ack_err;
|
|
351
|
+
if (!rkm)
|
|
352
|
+
continue;
|
|
353
|
+
ack_err = rd_kafka_share_acknowledge(rkshare, rkm);
|
|
354
|
+
TEST_SAY(" ack msg[%d] %s[%" PRId32 "]@%" PRId64 " -> %s\n", i,
|
|
355
|
+
rd_kafka_topic_name(rkm->rkt), rkm->partition,
|
|
356
|
+
rkm->offset, rd_kafka_err2str(ack_err));
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/**
|
|
361
|
+
* @brief Destroy a share consumer, using flags variant when non-zero.
|
|
362
|
+
*/
|
|
363
|
+
static void destroy_share_consumer(rd_kafka_share_t *rkshare,
|
|
364
|
+
int destroy_flags) {
|
|
365
|
+
TEST_SAY("Calling destroy with flags 0x%x\n", destroy_flags);
|
|
366
|
+
if (destroy_flags)
|
|
367
|
+
rd_kafka_share_destroy_flags(rkshare, destroy_flags);
|
|
368
|
+
else
|
|
369
|
+
rd_kafka_share_destroy(rkshare);
|
|
370
|
+
TEST_SAY("Successfully destroyed share consumer\n");
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/* Per-thread argument for destroy_watchdog_thread. Atomic `done` lets the
|
|
374
|
+
* main thread poll for completion without locking. */
|
|
375
|
+
typedef struct destroy_watchdog_arg_s {
|
|
376
|
+
rd_kafka_share_t *rkshare;
|
|
377
|
+
int destroy_flags;
|
|
378
|
+
rd_atomic32_t done;
|
|
379
|
+
} destroy_watchdog_arg_t;
|
|
380
|
+
|
|
381
|
+
static int destroy_watchdog_thread(void *p) {
|
|
382
|
+
destroy_watchdog_arg_t *a = p;
|
|
383
|
+
if (a->destroy_flags)
|
|
384
|
+
rd_kafka_share_destroy_flags(a->rkshare, a->destroy_flags);
|
|
385
|
+
else
|
|
386
|
+
rd_kafka_share_destroy(a->rkshare);
|
|
387
|
+
rd_atomic32_set(&a->done, 1);
|
|
388
|
+
return 0;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/**
|
|
392
|
+
* @brief is_fatal_cb hook for test_broker_decommission_with_commit_sync.
|
|
393
|
+
*
|
|
394
|
+
* The decommission of a broker mid-flight produces __TRANSPORT and
|
|
395
|
+
* __ALL_BROKERS_DOWN errors as the connection is dropped and the client
|
|
396
|
+
* tries to reconnect. These are expected and should not fail the test.
|
|
397
|
+
*/
|
|
398
|
+
static int decommission_is_fatal_cb(rd_kafka_t *rk,
|
|
399
|
+
rd_kafka_resp_err_t err,
|
|
400
|
+
const char *reason) {
|
|
401
|
+
if (err == RD_KAFKA_RESP_ERR__TRANSPORT ||
|
|
402
|
+
err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN) {
|
|
403
|
+
TEST_SAY("Ignoring expected error: %s: %s\n",
|
|
404
|
+
rd_kafka_err2name(err), reason);
|
|
405
|
+
return 0;
|
|
406
|
+
}
|
|
407
|
+
return 1;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/****************************************************************************
|
|
411
|
+
* Test Cases
|
|
412
|
+
****************************************************************************/
|
|
413
|
+
|
|
414
|
+
/**
|
|
415
|
+
* @brief This test uses mock brokers to simulate delayed broker responses and
|
|
416
|
+
* makes commit* calls causing acknowledgements to get cached. Eventually, calls
|
|
417
|
+
* destroy() to validate that it does not hang.
|
|
418
|
+
* @param destroy_flags 0 for normal destroy,
|
|
419
|
+
* RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE to skip consumer close.
|
|
420
|
+
*
|
|
421
|
+
* In case of RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE, a __DESTROY error is
|
|
422
|
+
* expected to be returned from the broker thread having requests in-flight
|
|
423
|
+
*/
|
|
424
|
+
static void
|
|
425
|
+
test_destroy_with_cached_acks_and_delayed_broker(int destroy_flags) {
|
|
426
|
+
rd_kafka_mock_cluster_t *mcluster;
|
|
427
|
+
const char *bootstraps;
|
|
428
|
+
rd_kafka_share_t *rkshare;
|
|
429
|
+
rd_kafka_error_t *error;
|
|
430
|
+
rd_kafka_messages_t *rkmessages = NULL;
|
|
431
|
+
rd_kafka_topic_partition_list_t *commit_result = NULL;
|
|
432
|
+
const char *topic = "0179-destroy-cached-acks-delayed-broker";
|
|
433
|
+
const char *group = "0179-destroy-cached-acks";
|
|
434
|
+
const int broker_delay_ms = 5000;
|
|
435
|
+
size_t rcvd = 0;
|
|
436
|
+
int attempts = 0;
|
|
437
|
+
int i;
|
|
438
|
+
rd_ts_t t_start, t_elapsed_ms;
|
|
439
|
+
ack_receipts_t receipts;
|
|
440
|
+
|
|
441
|
+
ack_receipts_init(&receipts);
|
|
442
|
+
|
|
443
|
+
SUB_TEST_QUICK("destroy_flags=0x%x", destroy_flags);
|
|
444
|
+
|
|
445
|
+
mcluster = test_mock_cluster_new(1, &bootstraps);
|
|
446
|
+
rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1);
|
|
447
|
+
|
|
448
|
+
TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 1, 1) ==
|
|
449
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
450
|
+
"Failed to create mock topic");
|
|
451
|
+
|
|
452
|
+
TEST_SAY("Producing %d messages to topic %s\n", TEST_MSGS, topic);
|
|
453
|
+
test_produce_msgs_easy_v(topic, 0, 0, 0, TEST_MSGS, 16,
|
|
454
|
+
"bootstrap.servers", bootstraps, NULL);
|
|
455
|
+
|
|
456
|
+
TEST_SAY("Creating share consumer with explicit ack\n");
|
|
457
|
+
rkshare = new_share_consumer_for_mock_test(bootstraps, group, rd_true,
|
|
458
|
+
&receipts);
|
|
459
|
+
subscribe_topics(rkshare, &topic, 1);
|
|
460
|
+
|
|
461
|
+
TEST_SAY("Consuming messages (up to %d attempts)\n",
|
|
462
|
+
MAX_CONSUME_ATTEMPTS);
|
|
463
|
+
while (rcvd < 10 && attempts < MAX_CONSUME_ATTEMPTS) {
|
|
464
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
465
|
+
rkmessages = NULL;
|
|
466
|
+
error = rd_kafka_share_poll(rkshare, 3000, &rkmessages);
|
|
467
|
+
|
|
468
|
+
if (error) {
|
|
469
|
+
TEST_SAY("Attempt %d: consume error: %s\n", attempts,
|
|
470
|
+
rd_kafka_error_string(error));
|
|
471
|
+
rd_kafka_error_destroy(error);
|
|
472
|
+
} else {
|
|
473
|
+
rcvd = rd_kafka_messages_count(rkmessages);
|
|
474
|
+
if (rcvd > 0)
|
|
475
|
+
TEST_SAY("Attempt %d: consumed %d messages\n",
|
|
476
|
+
attempts, (int)rcvd);
|
|
477
|
+
}
|
|
478
|
+
attempts++;
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
TEST_ASSERT(rcvd >= 10,
|
|
482
|
+
"Expected at least 10 messages after %d attempts, got %d",
|
|
483
|
+
MAX_CONSUME_ATTEMPTS, (int)rcvd);
|
|
484
|
+
TEST_SAY("Successfully consumed %d messages\n", (int)rcvd);
|
|
485
|
+
|
|
486
|
+
/* Inject %dms delay on the next 3 ShareAcknowledge responses on
|
|
487
|
+
* broker 1 (the only broker in this mock cluster). */
|
|
488
|
+
TEST_SAY(
|
|
489
|
+
"Injecting %dms delay on the next 3 ShareAcknowledge "
|
|
490
|
+
"responses on broker 1\n",
|
|
491
|
+
broker_delay_ms);
|
|
492
|
+
TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts(
|
|
493
|
+
mcluster, 1, RD_KAFKAP_ShareAcknowledge, 3,
|
|
494
|
+
RD_KAFKA_RESP_ERR_NO_ERROR, broker_delay_ms,
|
|
495
|
+
RD_KAFKA_RESP_ERR_NO_ERROR, broker_delay_ms,
|
|
496
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
497
|
+
broker_delay_ms) == RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
498
|
+
"Failed to inject ShareAcknowledge delay");
|
|
499
|
+
|
|
500
|
+
/* Step 1: Acknowledge first 2 messages and commit async */
|
|
501
|
+
TEST_SAY(
|
|
502
|
+
"Step 1: Acknowledging messages 0-1 and calling commit_async\n");
|
|
503
|
+
ack_range_logged(rkshare, rkmessages, 0, 2, (int)rcvd);
|
|
504
|
+
/* The below call should keep the broker busy */
|
|
505
|
+
rd_kafka_share_commit_async(rkshare);
|
|
506
|
+
|
|
507
|
+
/* Step 2: Acknowledge next 4 messages and commit async */
|
|
508
|
+
TEST_SAY(
|
|
509
|
+
"Step 2: Acknowledging messages 2-5 and calling commit_async "
|
|
510
|
+
"(should cache)\n");
|
|
511
|
+
ack_range_logged(rkshare, rkmessages, 2, 6, (int)rcvd);
|
|
512
|
+
rd_kafka_share_commit_async(rkshare);
|
|
513
|
+
|
|
514
|
+
/* Step 3: Acknowledge next 4 messages and commit sync */
|
|
515
|
+
TEST_SAY(
|
|
516
|
+
"Step 3: Acknowledging messages 6-9 and calling "
|
|
517
|
+
"commit_sync (should cache)\n");
|
|
518
|
+
ack_range_logged(rkshare, rkmessages, 6, 10, (int)rcvd);
|
|
519
|
+
|
|
520
|
+
rd_kafka_share_commit_sync(rkshare, 1000, &commit_result);
|
|
521
|
+
TEST_ASSERT(commit_result != NULL,
|
|
522
|
+
"Expected non-NULL commit_sync result");
|
|
523
|
+
TEST_SAY("commit_sync returned %d partition result(s)\n",
|
|
524
|
+
commit_result->cnt);
|
|
525
|
+
for (i = 0; i < commit_result->cnt; i++) {
|
|
526
|
+
rd_kafka_topic_partition_t *p = &commit_result->elems[i];
|
|
527
|
+
TEST_SAY(" result[%d] %s [%" PRId32 "] err=%s\n", i, p->topic,
|
|
528
|
+
p->partition, rd_kafka_err2str(p->err));
|
|
529
|
+
TEST_ASSERT(p->err == RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT,
|
|
530
|
+
"Expected REQUEST_TIMED_OUT for %s [%" PRId32
|
|
531
|
+
"], got %s",
|
|
532
|
+
p->topic, p->partition, rd_kafka_err2str(p->err));
|
|
533
|
+
}
|
|
534
|
+
rd_kafka_topic_partition_list_destroy(commit_result);
|
|
535
|
+
|
|
536
|
+
/* Destroy all consumed messages */
|
|
537
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
538
|
+
rkmessages = NULL;
|
|
539
|
+
|
|
540
|
+
TEST_SAY("Calling destroy with flags 0x%x\n", destroy_flags);
|
|
541
|
+
t_start = test_clock();
|
|
542
|
+
destroy_share_consumer(rkshare, destroy_flags);
|
|
543
|
+
t_elapsed_ms = (test_clock() - t_start) / 1000;
|
|
544
|
+
TEST_SAY("Destroy completed in %" PRId64 " ms\n", t_elapsed_ms);
|
|
545
|
+
|
|
546
|
+
if (destroy_flags & RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE) {
|
|
547
|
+
/* NO_CONSUMER_CLOSE: destroy must NOT wait for the
|
|
548
|
+
* delayed broker. */
|
|
549
|
+
TEST_ASSERT(t_elapsed_ms < 2000,
|
|
550
|
+
"Destroy(NO_CONSUMER_CLOSE) took %" PRId64
|
|
551
|
+
" ms, expected < 2000 ms",
|
|
552
|
+
t_elapsed_ms);
|
|
553
|
+
verify_ack_receipts(&receipts, NULL, 0,
|
|
554
|
+
"cached-acks NO_CONSUMER_CLOSE");
|
|
555
|
+
} else {
|
|
556
|
+
/* Full close: destroy waits for the existing commit
|
|
557
|
+
* async request and the session-leave request. Expect
|
|
558
|
+
* roughly 2 * broker_delay_ms plus small overhead. */
|
|
559
|
+
int64_t expected_max_ms = 2 * broker_delay_ms + 2000;
|
|
560
|
+
TEST_ASSERT(t_elapsed_ms <= expected_max_ms,
|
|
561
|
+
"Destroy took %" PRId64 " ms, expected <= %" PRId64
|
|
562
|
+
" ms",
|
|
563
|
+
t_elapsed_ms, expected_max_ms);
|
|
564
|
+
expected_ack_t expected[] = {
|
|
565
|
+
{topic, 0, RD_KAFKA_RESP_ERR_NO_ERROR, 10},
|
|
566
|
+
};
|
|
567
|
+
verify_ack_receipts(&receipts, expected, 1,
|
|
568
|
+
"cached-acks full close");
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
ack_receipts_destroy(&receipts);
|
|
572
|
+
|
|
573
|
+
test_mock_cluster_destroy(mcluster);
|
|
574
|
+
SUB_TEST_PASS();
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
/**
|
|
578
|
+
* @brief Stage a deterministic broker decommission so the next request
|
|
579
|
+
* sent to \p target_broker_id (on ApiKey \p blocked_api_key) is
|
|
580
|
+
* stamped with __DESTROY_BROKER.
|
|
581
|
+
*
|
|
582
|
+
* Caller invariants when calling this:
|
|
583
|
+
* - Cluster has at least target_broker_id and surviving_broker_id.
|
|
584
|
+
* - target_partition's leader in the mock is target_broker_id.
|
|
585
|
+
* - No \p blocked_api_key request is currently in flight to
|
|
586
|
+
* target_broker_id (the injected delay is consumed by the
|
|
587
|
+
* NEXT one).
|
|
588
|
+
*
|
|
589
|
+
* After this returns:
|
|
590
|
+
* - Mock state has target_partition's leader migrated to
|
|
591
|
+
* surviving_broker_id and target_broker_id removed from
|
|
592
|
+
* metadata.
|
|
593
|
+
* - A 2s delay is queued on the next MetadataResponse from
|
|
594
|
+
* BOTH brokers — the client has NOT yet observed the
|
|
595
|
+
* topology change.
|
|
596
|
+
* - A long (30s) delay is queued on the next
|
|
597
|
+
* blocked_api_key response from target_broker_id.
|
|
598
|
+
* - We've slept past one refresh tick, so the periodic
|
|
599
|
+
* MetadataRequest is in flight (held by the mock).
|
|
600
|
+
*
|
|
601
|
+
* Now when the caller invokes the action under test (which
|
|
602
|
+
* will ship a blocked_api_key request to target_broker_id),
|
|
603
|
+
* the timeline is:
|
|
604
|
+
* - The request lands on target_broker_id and parks (30s).
|
|
605
|
+
* - The held MetadataResponse fires (2s in), the client
|
|
606
|
+
* sees target_broker_id is gone, runs
|
|
607
|
+
* rd_kafka_broker_decommission(target_broker_id).
|
|
608
|
+
* - target_broker_id's termination path runs
|
|
609
|
+
* bufq_timeout_scan over its outbufs/waitresps and stamps
|
|
610
|
+
* the parked request with __DESTROY_BROKER.
|
|
611
|
+
*
|
|
612
|
+
* Implementation detail: the Metadata RTT injection only takes
|
|
613
|
+
* effect because of the rd_kafka_mock_next_request_error() call
|
|
614
|
+
* we added to the Metadata handler in rdkafka_mock_handlers.c.
|
|
615
|
+
* Without that call, the mock ignores Metadata RTT injections.
|
|
616
|
+
*/
|
|
617
|
+
static void stage_broker_decommission(rd_kafka_mock_cluster_t *mcluster,
|
|
618
|
+
const char *topic,
|
|
619
|
+
int32_t target_partition,
|
|
620
|
+
int32_t target_broker_id,
|
|
621
|
+
int32_t surviving_broker_id,
|
|
622
|
+
int16_t blocked_api_key) {
|
|
623
|
+
const int metadata_delay_ms = 2000;
|
|
624
|
+
const int blocked_api_delay_ms = 30000;
|
|
625
|
+
const int refresh_settle_ms = 700; /* > 500ms refresh interval */
|
|
626
|
+
|
|
627
|
+
TEST_SAY("Staging decommission: target broker=%" PRId32
|
|
628
|
+
", surviving broker=%" PRId32 ", target partition=%" PRId32
|
|
629
|
+
", blocked ApiKey=%hd\n",
|
|
630
|
+
target_broker_id, surviving_broker_id, target_partition,
|
|
631
|
+
blocked_api_key);
|
|
632
|
+
|
|
633
|
+
/* Delay metadata responses from both brokers so the client
|
|
634
|
+
* doesn't observe the topology change until AFTER the caller's
|
|
635
|
+
* action has shipped its request to the target broker. */
|
|
636
|
+
rd_kafka_mock_broker_push_request_error_rtts(
|
|
637
|
+
mcluster, target_broker_id, RD_KAFKAP_Metadata, 1,
|
|
638
|
+
RD_KAFKA_RESP_ERR_NO_ERROR, metadata_delay_ms);
|
|
639
|
+
rd_kafka_mock_broker_push_request_error_rtts(
|
|
640
|
+
mcluster, surviving_broker_id, RD_KAFKAP_Metadata, 1,
|
|
641
|
+
RD_KAFKA_RESP_ERR_NO_ERROR, metadata_delay_ms);
|
|
642
|
+
|
|
643
|
+
/* Delay the blocked-API response on the target broker so the
|
|
644
|
+
* caller's request stays parked until decommission fires. */
|
|
645
|
+
rd_kafka_mock_broker_push_request_error_rtts(
|
|
646
|
+
mcluster, target_broker_id, blocked_api_key, 1,
|
|
647
|
+
RD_KAFKA_RESP_ERR_NO_ERROR, blocked_api_delay_ms);
|
|
648
|
+
|
|
649
|
+
/* Migrate the target partition's leader, then remove the
|
|
650
|
+
* target broker from metadata. Target broker's TCP connections
|
|
651
|
+
* stay alive; in-flight requests remain parked. */
|
|
652
|
+
TEST_ASSERT(rd_kafka_mock_partition_set_leader(
|
|
653
|
+
mcluster, topic, target_partition,
|
|
654
|
+
surviving_broker_id) == RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
655
|
+
"Failed to migrate partition %" PRId32 " leader",
|
|
656
|
+
target_partition);
|
|
657
|
+
TEST_ASSERT(rd_kafka_mock_broker_remove_from_metadata(
|
|
658
|
+
mcluster, target_broker_id) ==
|
|
659
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
660
|
+
"Failed to remove broker %" PRId32 " from metadata",
|
|
661
|
+
target_broker_id);
|
|
662
|
+
|
|
663
|
+
/* Sleep past one refresh tick so the next periodic refresh
|
|
664
|
+
* fires (and parks behind the metadata delay we just injected). */
|
|
665
|
+
rd_usleep(refresh_settle_ms * 1000, NULL);
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
/**
|
|
669
|
+
* @brief Test that commit_sync returns __DESTROY_BROKER for partitions
|
|
670
|
+
* whose broker is decommissioned mid-flight.
|
|
671
|
+
*
|
|
672
|
+
* Setup: 2-broker mock cluster, 2-partition topic with p0 led
|
|
673
|
+
* by broker 1 and p1 led by broker 2.
|
|
674
|
+
*
|
|
675
|
+
* Flow:
|
|
676
|
+
* 1. Drain both partitions. The last batch's partition tells
|
|
677
|
+
* us which broker is currently holding unacked records —
|
|
678
|
+
* that's the broker we'll decommission.
|
|
679
|
+
* 2. Inject a 2s delay on the target broker's NEXT
|
|
680
|
+
* MetadataResponse.
|
|
681
|
+
* 3. Synchronously update mock state: migrate target
|
|
682
|
+
* partition's leader to the surviving broker, and remove
|
|
683
|
+
* the target broker from metadata. Connection stays alive.
|
|
684
|
+
* 4. Sleep > topic.metadata.refresh.interval.ms (500ms) so the
|
|
685
|
+
* periodic refresh tick fires and queues a MetadataRequest
|
|
686
|
+
* on some broker. If it lands on the target broker, the
|
|
687
|
+
* request sits in front of any subsequent ShareAck on the
|
|
688
|
+
* same connection (response is held 2s by the injection).
|
|
689
|
+
* If it lands on the surviving broker, the response comes
|
|
690
|
+
* back quickly. Either way, the parsed response sees the
|
|
691
|
+
* target broker missing and fires
|
|
692
|
+
* rd_kafka_broker_decommission() against it.
|
|
693
|
+
* 5. commit_sync ships the unacked batch's ShareAck to the
|
|
694
|
+
* target broker. The decommission's OP_TERMINATE handler
|
|
695
|
+
* stamps the in-flight ShareAck with __DESTROY_BROKER.
|
|
696
|
+
*
|
|
697
|
+
* Assertion: commit_sync result has 1 entry for the target
|
|
698
|
+
* partition with err == __DESTROY_BROKER.
|
|
699
|
+
*/
|
|
700
|
+
static void test_broker_decommission_with_commit_sync(int destroy_flags,
|
|
701
|
+
rd_bool_t explicit_ack) {
|
|
702
|
+
rd_kafka_mock_cluster_t *mcluster;
|
|
703
|
+
const char *bootstraps;
|
|
704
|
+
rd_kafka_share_t *rkshare;
|
|
705
|
+
rd_kafka_error_t *error;
|
|
706
|
+
rd_kafka_messages_t *rkmessages = NULL;
|
|
707
|
+
rd_kafka_topic_partition_list_t *result = NULL;
|
|
708
|
+
const char *topic;
|
|
709
|
+
const char *group = "0179-decommission-commit-sync";
|
|
710
|
+
int32_t target_broker_id = -1;
|
|
711
|
+
int32_t surviving_broker_id = -1;
|
|
712
|
+
int32_t target_partition = -1;
|
|
713
|
+
size_t rcvd = 0;
|
|
714
|
+
int attempts = 0;
|
|
715
|
+
int i;
|
|
716
|
+
|
|
717
|
+
SUB_TEST_QUICK("destroy_flags=0x%x ack_mode=%s", destroy_flags,
|
|
718
|
+
explicit_ack ? "explicit" : "implicit");
|
|
719
|
+
|
|
720
|
+
/* Suppress expected __TRANSPORT / __ALL_BROKERS_DOWN errors that
|
|
721
|
+
* fire when the target broker connection is dropped. */
|
|
722
|
+
test_curr->is_fatal_cb = decommission_is_fatal_cb;
|
|
723
|
+
|
|
724
|
+
/* 2-broker mock cluster */
|
|
725
|
+
mcluster = test_mock_cluster_new(2, &bootstraps);
|
|
726
|
+
rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1);
|
|
727
|
+
|
|
728
|
+
/* 2-partition topic: p0 led by broker 1, p1 led by broker 2.
|
|
729
|
+
* Consuming + acking from both partitions establishes UP
|
|
730
|
+
* connections to both brokers. */
|
|
731
|
+
topic = test_mk_topic_name("0179-decommission-commit-sync", 1);
|
|
732
|
+
TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 2, 2) ==
|
|
733
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
734
|
+
"Failed to create mock topic");
|
|
735
|
+
TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 0, 1) ==
|
|
736
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
737
|
+
"Failed to set partition 0 leader to broker 1");
|
|
738
|
+
TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 1, 2) ==
|
|
739
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
740
|
+
"Failed to set partition 1 leader to broker 2");
|
|
741
|
+
|
|
742
|
+
/* Produce TEST_MSGS/2 messages to each partition. */
|
|
743
|
+
TEST_SAY("Producing %d messages to each partition of topic %s\n",
|
|
744
|
+
TEST_MSGS / 2, topic);
|
|
745
|
+
test_produce_msgs_easy_v(topic, 0, 0, 0, TEST_MSGS / 2, 16,
|
|
746
|
+
"bootstrap.servers", bootstraps, NULL);
|
|
747
|
+
test_produce_msgs_easy_v(topic, 0, 1, 0, TEST_MSGS / 2, 16,
|
|
748
|
+
"bootstrap.servers", bootstraps, NULL);
|
|
749
|
+
|
|
750
|
+
TEST_SAY("Creating share consumer (%s ack)\n",
|
|
751
|
+
explicit_ack ? "explicit" : "implicit");
|
|
752
|
+
rkshare = new_share_consumer_for_mock_test(bootstraps, group,
|
|
753
|
+
explicit_ack, NULL);
|
|
754
|
+
subscribe_topics(rkshare, &topic, 1);
|
|
755
|
+
|
|
756
|
+
TEST_SAY("Consuming up to %d messages (max %d attempts)\n", TEST_MSGS,
|
|
757
|
+
MAX_CONSUME_ATTEMPTS);
|
|
758
|
+
while (rcvd < TEST_MSGS && attempts < MAX_CONSUME_ATTEMPTS) {
|
|
759
|
+
size_t batch_rcvd;
|
|
760
|
+
|
|
761
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
762
|
+
rkmessages = NULL;
|
|
763
|
+
error = rd_kafka_share_poll(rkshare, 3000, &rkmessages);
|
|
764
|
+
|
|
765
|
+
if (error) {
|
|
766
|
+
TEST_SAY("Attempt %d: consume error: %s\n", attempts,
|
|
767
|
+
rd_kafka_error_string(error));
|
|
768
|
+
rd_kafka_error_destroy(error);
|
|
769
|
+
attempts++;
|
|
770
|
+
continue;
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
batch_rcvd = rd_kafka_messages_count(rkmessages);
|
|
774
|
+
if (batch_rcvd > 0) {
|
|
775
|
+
TEST_SAY("Attempt %d: consumed %d messages\n", attempts,
|
|
776
|
+
(int)batch_rcvd);
|
|
777
|
+
rcvd += batch_rcvd;
|
|
778
|
+
|
|
779
|
+
if (explicit_ack) {
|
|
780
|
+
size_t k;
|
|
781
|
+
for (k = 0; k < batch_rcvd; k++) {
|
|
782
|
+
rd_kafka_message_t *rkm =
|
|
783
|
+
rd_kafka_messages_get(rkmessages,
|
|
784
|
+
k);
|
|
785
|
+
if (rkm)
|
|
786
|
+
rd_kafka_share_acknowledge(
|
|
787
|
+
rkshare, rkm);
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
attempts++;
|
|
792
|
+
}
|
|
793
|
+
TEST_ASSERT(rcvd == TEST_MSGS,
|
|
794
|
+
"Expected %d messages, got %d after %d attempts", TEST_MSGS,
|
|
795
|
+
(int)rcvd, attempts);
|
|
796
|
+
TEST_SAY("Consumed %d msgs total in %d attempts\n", (int)rcvd,
|
|
797
|
+
attempts);
|
|
798
|
+
|
|
799
|
+
/* The last message of the last batch tells us which broker
|
|
800
|
+
* holds the unacked records that commit_sync will ship to. */
|
|
801
|
+
{
|
|
802
|
+
size_t _last_cnt = rd_kafka_messages_count(rkmessages);
|
|
803
|
+
rd_kafka_message_t *_last_rkm =
|
|
804
|
+
_last_cnt > 0
|
|
805
|
+
? rd_kafka_messages_get(rkmessages, _last_cnt - 1)
|
|
806
|
+
: NULL;
|
|
807
|
+
TEST_ASSERT(_last_rkm != NULL,
|
|
808
|
+
"Expected last batch to be non-empty");
|
|
809
|
+
target_partition = _last_rkm->partition;
|
|
810
|
+
}
|
|
811
|
+
target_broker_id = (target_partition == 0) ? 1 : 2;
|
|
812
|
+
surviving_broker_id = (target_broker_id == 1) ? 2 : 1;
|
|
813
|
+
TEST_SAY("Target partition = %" PRId32 ", target broker = %" PRId32
|
|
814
|
+
", surviving broker = %" PRId32 "\n",
|
|
815
|
+
target_partition, target_broker_id, surviving_broker_id);
|
|
816
|
+
|
|
817
|
+
stage_broker_decommission(mcluster, topic, target_partition,
|
|
818
|
+
target_broker_id, surviving_broker_id,
|
|
819
|
+
RD_KAFKAP_ShareAcknowledge);
|
|
820
|
+
|
|
821
|
+
TEST_SAY("Calling commit_sync (timeout 60s)\n");
|
|
822
|
+
error = rd_kafka_share_commit_sync(rkshare, 60000, &result);
|
|
823
|
+
|
|
824
|
+
if (error) {
|
|
825
|
+
TEST_SAY("commit_sync returned error: %s\n",
|
|
826
|
+
rd_kafka_error_string(error));
|
|
827
|
+
rd_kafka_error_destroy(error);
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
TEST_ASSERT(result != NULL,
|
|
831
|
+
"Expected commit_sync to return a non-NULL results list");
|
|
832
|
+
TEST_SAY("commit_sync returned %d partition result(s)\n", result->cnt);
|
|
833
|
+
|
|
834
|
+
/* Only the last batch's acks reach commit_sync, so we expect
|
|
835
|
+
* exactly one partition result: the target partition. */
|
|
836
|
+
TEST_ASSERT(result->cnt == 1,
|
|
837
|
+
"Expected 1 partition result (partition %" PRId32
|
|
838
|
+
" only), got %d",
|
|
839
|
+
target_partition, result->cnt);
|
|
840
|
+
|
|
841
|
+
for (i = 0; i < result->cnt; i++) {
|
|
842
|
+
rd_kafka_topic_partition_t *p = &result->elems[i];
|
|
843
|
+
TEST_SAY(" result[%d] %s [%" PRId32 "] err=%s\n", i, p->topic,
|
|
844
|
+
p->partition, rd_kafka_err2str(p->err));
|
|
845
|
+
TEST_ASSERT(p->partition == target_partition,
|
|
846
|
+
"Expected partition %" PRId32
|
|
847
|
+
" in result, got %" PRId32,
|
|
848
|
+
target_partition, p->partition);
|
|
849
|
+
/* __DESTROY_BROKER from the broker-thread decommission
|
|
850
|
+
* path is translated to __TRANSPORT at the app-facing
|
|
851
|
+
* funnel (rd_kafka_share_commit_sync_apply_result), to
|
|
852
|
+
* match the Java NetworkException/DisconnectException
|
|
853
|
+
* terminal surface. */
|
|
854
|
+
TEST_ASSERT(p->err == RD_KAFKA_RESP_ERR__TRANSPORT,
|
|
855
|
+
"Expected __TRANSPORT for partition %" PRId32
|
|
856
|
+
" (broker %" PRId32 ", decommissioned), got %s",
|
|
857
|
+
target_partition, target_broker_id,
|
|
858
|
+
rd_kafka_err2str(p->err));
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
rd_kafka_topic_partition_list_destroy(result);
|
|
862
|
+
|
|
863
|
+
/* Cleanup */
|
|
864
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
865
|
+
rkmessages = NULL;
|
|
866
|
+
|
|
867
|
+
destroy_share_consumer(rkshare, destroy_flags);
|
|
868
|
+
test_mock_cluster_destroy(mcluster);
|
|
869
|
+
|
|
870
|
+
/* Restore the default fatal-error handler. */
|
|
871
|
+
test_curr->is_fatal_cb = NULL;
|
|
872
|
+
|
|
873
|
+
SUB_TEST_PASS();
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
|
|
877
|
+
/**
|
|
878
|
+
* @brief Test that a broker decommissioned while a piggybacked
|
|
879
|
+
* ShareAck is in flight is handled gracefully: no app-visible
|
|
880
|
+
* error surfaces from consume_batch.
|
|
881
|
+
*
|
|
882
|
+
* Setup mirrors test_broker_decommission_with_commit_sync —
|
|
883
|
+
* 2-broker mock cluster, 2-partition topic with RF=2, drain
|
|
884
|
+
* both partitions in implicit-ack mode. The last batch's
|
|
885
|
+
* records stay ACQUIRED on whichever broker served them
|
|
886
|
+
* (target broker, picked dynamically from the last batch's
|
|
887
|
+
* partition). The next consume_batch piggybacks an implicit
|
|
888
|
+
* ShareAck onto a ShareFetch sent to the target broker — that
|
|
889
|
+
* request is what we want stamped __DESTROY_BROKER.
|
|
890
|
+
*
|
|
891
|
+
* After staging the decommission, we call consume_batch again.
|
|
892
|
+
* The fanout sends ShareFetch (with piggybacked ack) to the
|
|
893
|
+
* target broker — parked behind the injected ShareAck delay.
|
|
894
|
+
* When the metadata response triggers the decommission, the
|
|
895
|
+
* parked request is stamped __DESTROY_BROKER via
|
|
896
|
+
* bufq_timeout_scan. consume_batch returns no error and no
|
|
897
|
+
* messages.
|
|
898
|
+
*/
|
|
899
|
+
static void test_broker_decommission_with_consume_batch(int destroy_flags) {
|
|
900
|
+
rd_kafka_mock_cluster_t *mcluster;
|
|
901
|
+
const char *bootstraps;
|
|
902
|
+
rd_kafka_share_t *rkshare;
|
|
903
|
+
rd_kafka_error_t *error;
|
|
904
|
+
rd_kafka_messages_t *rkmessages = NULL;
|
|
905
|
+
const char *topic;
|
|
906
|
+
const char *group = "0179-decommission-consume-batch";
|
|
907
|
+
int32_t target_broker_id = -1;
|
|
908
|
+
int32_t surviving_broker_id = -1;
|
|
909
|
+
int32_t target_partition = -1;
|
|
910
|
+
size_t rcvd = 0;
|
|
911
|
+
size_t fetch_rcvd = 0;
|
|
912
|
+
int attempts = 0;
|
|
913
|
+
int32_t surviving_part;
|
|
914
|
+
expected_ack_t expected[2];
|
|
915
|
+
ack_receipts_t receipts;
|
|
916
|
+
|
|
917
|
+
ack_receipts_init(&receipts);
|
|
918
|
+
|
|
919
|
+
SUB_TEST_QUICK();
|
|
920
|
+
|
|
921
|
+
/* Suppress expected __TRANSPORT / __ALL_BROKERS_DOWN errors that
|
|
922
|
+
* fire when the target broker connection is dropped. */
|
|
923
|
+
test_curr->is_fatal_cb = decommission_is_fatal_cb;
|
|
924
|
+
|
|
925
|
+
/* 2-broker mock cluster */
|
|
926
|
+
mcluster = test_mock_cluster_new(2, &bootstraps);
|
|
927
|
+
rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1);
|
|
928
|
+
|
|
929
|
+
/* 2-partition topic, RF=2: p0 led by broker 1, p1 led by broker 2. */
|
|
930
|
+
topic = test_mk_topic_name("0179-decommission-consume-batch", 1);
|
|
931
|
+
TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 2, 2) ==
|
|
932
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
933
|
+
"Failed to create mock topic");
|
|
934
|
+
TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 0, 1) ==
|
|
935
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
936
|
+
"Failed to set partition 0 leader to broker 1");
|
|
937
|
+
TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 1, 2) ==
|
|
938
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
939
|
+
"Failed to set partition 1 leader to broker 2");
|
|
940
|
+
|
|
941
|
+
/* Produce TEST_MSGS/2 messages to each partition. */
|
|
942
|
+
TEST_SAY("Producing %d messages to each partition of topic %s\n",
|
|
943
|
+
TEST_MSGS / 2, topic);
|
|
944
|
+
test_produce_msgs_easy_v(topic, 0, 0, 0, TEST_MSGS / 2, 16,
|
|
945
|
+
"bootstrap.servers", bootstraps, NULL);
|
|
946
|
+
test_produce_msgs_easy_v(topic, 0, 1, 0, TEST_MSGS / 2, 16,
|
|
947
|
+
"bootstrap.servers", bootstraps, NULL);
|
|
948
|
+
|
|
949
|
+
TEST_SAY("Creating share consumer (implicit ack) with ack_cb\n");
|
|
950
|
+
rkshare = new_share_consumer_for_mock_test(bootstraps, group, rd_false,
|
|
951
|
+
&receipts);
|
|
952
|
+
subscribe_topics(rkshare, &topic, 1);
|
|
953
|
+
|
|
954
|
+
/* Drain both partitions in implicit-ack mode. No trailing-empty
|
|
955
|
+
* flush — the last batch's records stay ACQUIRED on whichever
|
|
956
|
+
* broker served them. */
|
|
957
|
+
TEST_SAY("Consuming up to %d messages (max %d attempts)\n", TEST_MSGS,
|
|
958
|
+
MAX_CONSUME_ATTEMPTS);
|
|
959
|
+
while (rcvd < TEST_MSGS && attempts < MAX_CONSUME_ATTEMPTS) {
|
|
960
|
+
size_t batch_rcvd;
|
|
961
|
+
|
|
962
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
963
|
+
rkmessages = NULL;
|
|
964
|
+
error = rd_kafka_share_poll(rkshare, 3000, &rkmessages);
|
|
965
|
+
|
|
966
|
+
if (error) {
|
|
967
|
+
TEST_SAY("Attempt %d: consume error: %s\n", attempts,
|
|
968
|
+
rd_kafka_error_string(error));
|
|
969
|
+
rd_kafka_error_destroy(error);
|
|
970
|
+
attempts++;
|
|
971
|
+
continue;
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
batch_rcvd = rd_kafka_messages_count(rkmessages);
|
|
975
|
+
if (batch_rcvd > 0) {
|
|
976
|
+
TEST_SAY("Attempt %d: consumed %d messages\n", attempts,
|
|
977
|
+
(int)batch_rcvd);
|
|
978
|
+
rcvd += batch_rcvd;
|
|
979
|
+
}
|
|
980
|
+
attempts++;
|
|
981
|
+
}
|
|
982
|
+
TEST_ASSERT(rcvd == TEST_MSGS,
|
|
983
|
+
"Expected %d messages, got %d after %d attempts", TEST_MSGS,
|
|
984
|
+
(int)rcvd, attempts);
|
|
985
|
+
TEST_SAY("Consumed %d msgs total in %d attempts\n", (int)rcvd,
|
|
986
|
+
attempts);
|
|
987
|
+
|
|
988
|
+
/* The last message of the last batch tells us which broker
|
|
989
|
+
* holds the unacked records — that's the broker the next
|
|
990
|
+
* consume_batch's piggybacked ShareAck will target. */
|
|
991
|
+
{
|
|
992
|
+
size_t _last_cnt = rd_kafka_messages_count(rkmessages);
|
|
993
|
+
rd_kafka_message_t *_last_rkm =
|
|
994
|
+
_last_cnt > 0
|
|
995
|
+
? rd_kafka_messages_get(rkmessages, _last_cnt - 1)
|
|
996
|
+
: NULL;
|
|
997
|
+
TEST_ASSERT(_last_rkm != NULL,
|
|
998
|
+
"Expected last batch to be non-empty");
|
|
999
|
+
target_partition = _last_rkm->partition;
|
|
1000
|
+
}
|
|
1001
|
+
target_broker_id = (target_partition == 0) ? 1 : 2;
|
|
1002
|
+
surviving_broker_id = (target_broker_id == 1) ? 2 : 1;
|
|
1003
|
+
TEST_SAY("Target partition = %" PRId32 ", target broker = %" PRId32
|
|
1004
|
+
", surviving broker = %" PRId32 "\n",
|
|
1005
|
+
target_partition, target_broker_id, surviving_broker_id);
|
|
1006
|
+
|
|
1007
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
1008
|
+
rkmessages = NULL;
|
|
1009
|
+
rcvd = 0;
|
|
1010
|
+
|
|
1011
|
+
rd_kafka_mock_broker_push_request_error_rtts(
|
|
1012
|
+
mcluster, target_broker_id, RD_KAFKAP_ShareFetch, 1,
|
|
1013
|
+
RD_KAFKA_RESP_ERR_NO_ERROR, 30000);
|
|
1014
|
+
|
|
1015
|
+
stage_broker_decommission(mcluster, topic, target_partition,
|
|
1016
|
+
target_broker_id, surviving_broker_id,
|
|
1017
|
+
RD_KAFKAP_ShareAcknowledge);
|
|
1018
|
+
|
|
1019
|
+
/* Call consume_batch — whichever request lands on the target
|
|
1020
|
+
* broker (ShareFetch with piggybacked ack OR standalone
|
|
1021
|
+
* ShareAcknowledge) parks behind its injected delay; when the
|
|
1022
|
+
* metadata response triggers the decommission, the parked
|
|
1023
|
+
* request is stamped __DESTROY_BROKER. The surviving broker
|
|
1024
|
+
* has no new messages, so consume_batch times out cleanly
|
|
1025
|
+
* with 0 messages and no app-visible error. */
|
|
1026
|
+
TEST_SAY(
|
|
1027
|
+
"Calling consume_batch — expecting no messages and no "
|
|
1028
|
+
"app-visible error\n");
|
|
1029
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
1030
|
+
rkmessages = NULL;
|
|
1031
|
+
error = rd_kafka_share_poll(rkshare, 3000, &rkmessages);
|
|
1032
|
+
fetch_rcvd = rd_kafka_messages_count(rkmessages);
|
|
1033
|
+
|
|
1034
|
+
TEST_ASSERT(error == NULL,
|
|
1035
|
+
"Expected consume_batch to return NULL error, got %s",
|
|
1036
|
+
error ? rd_kafka_error_string(error) : "(null)");
|
|
1037
|
+
TEST_ASSERT(fetch_rcvd == 0,
|
|
1038
|
+
"Expected 0 messages from consume_batch (target broker "
|
|
1039
|
+
"decommissioned, surviving broker has no new "
|
|
1040
|
+
"messages), got %d",
|
|
1041
|
+
(int)fetch_rcvd);
|
|
1042
|
+
|
|
1043
|
+
drain_ack_callbacks(rkshare);
|
|
1044
|
+
|
|
1045
|
+
/* Assert ack callback receipts.
|
|
1046
|
+
*
|
|
1047
|
+
* Drain loop: each batch from the SURVIVING partition gets
|
|
1048
|
+
* implicitly acked when the next ShareFetch on the surviving
|
|
1049
|
+
* broker piggybacks the ack — those records appear in the
|
|
1050
|
+
* callback with NO_ERROR. The TARGET partition's last batch
|
|
1051
|
+
* stays ACQUIRED (no further piggyback happens on that
|
|
1052
|
+
* broker before we stage the decommission).
|
|
1053
|
+
*
|
|
1054
|
+
* Second consume_batch: the piggybacked ShareAck for the
|
|
1055
|
+
* target partition's ACQUIRED batch parks on the target
|
|
1056
|
+
* broker and gets stamped __DESTROY_BROKER when the
|
|
1057
|
+
* decommission fires; __DESTROY_BROKER is translated to
|
|
1058
|
+
* __TRANSPORT at the app-facing funnel.
|
|
1059
|
+
*
|
|
1060
|
+
* surviving_partition: TEST_MSGS/2 offsets, NO_ERROR
|
|
1061
|
+
* target_partition: TEST_MSGS/2 offsets, __TRANSPORT */
|
|
1062
|
+
surviving_part = (target_partition == 0) ? 1 : 0;
|
|
1063
|
+
expected[0] = (expected_ack_t) {
|
|
1064
|
+
topic, surviving_part, RD_KAFKA_RESP_ERR_NO_ERROR, TEST_MSGS / 2};
|
|
1065
|
+
expected[1] =
|
|
1066
|
+
(expected_ack_t) {topic, target_partition,
|
|
1067
|
+
RD_KAFKA_RESP_ERR__TRANSPORT, TEST_MSGS / 2};
|
|
1068
|
+
verify_ack_receipts(&receipts, expected, 2, "consume_batch");
|
|
1069
|
+
|
|
1070
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
1071
|
+
rkmessages = NULL;
|
|
1072
|
+
|
|
1073
|
+
destroy_share_consumer(rkshare, destroy_flags);
|
|
1074
|
+
test_mock_cluster_destroy(mcluster);
|
|
1075
|
+
|
|
1076
|
+
ack_receipts_destroy(&receipts);
|
|
1077
|
+
|
|
1078
|
+
/* Restore the default fatal-error handler. */
|
|
1079
|
+
test_curr->is_fatal_cb = NULL;
|
|
1080
|
+
|
|
1081
|
+
SUB_TEST_PASS();
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
|
|
1085
|
+
/**
|
|
1086
|
+
* @brief Test that rd_kafka_share_consumer_close() completes gracefully
|
|
1087
|
+
* when one of the brokers it ships acks/leaves to is decommissioned
|
|
1088
|
+
* mid-call.
|
|
1089
|
+
*
|
|
1090
|
+
* Setup mirrors test_broker_decommission_with_commit_sync —
|
|
1091
|
+
* 2-broker mock cluster, 2-partition topic with RF=2, fast
|
|
1092
|
+
* metadata refresh. close() will then ship a ShareAck
|
|
1093
|
+
* for those records to whichever broker served them,
|
|
1094
|
+
* plus session-leave requests to both brokers.
|
|
1095
|
+
*
|
|
1096
|
+
* The target broker (the one holding the unacked batch) gets a
|
|
1097
|
+
* 5s ShareAck rtt delay. While that ShareAck is parked, the
|
|
1098
|
+
* background thread removes the broker from mock metadata; the
|
|
1099
|
+
* client decommissions it; the in-flight ShareAck is purged
|
|
1100
|
+
* with __DESTROY_BROKER.
|
|
1101
|
+
*
|
|
1102
|
+
* Expectation: rd_kafka_share_consumer_close() returns NULL (no
|
|
1103
|
+
* error) and does not hang. destroy() afterwards also completes.
|
|
1104
|
+
*/
|
|
1105
|
+
static void test_broker_decommission_during_close(int destroy_flags,
|
|
1106
|
+
rd_bool_t explicit_ack) {
|
|
1107
|
+
rd_kafka_mock_cluster_t *mcluster;
|
|
1108
|
+
const char *bootstraps;
|
|
1109
|
+
rd_kafka_share_t *rkshare;
|
|
1110
|
+
rd_kafka_error_t *error;
|
|
1111
|
+
rd_kafka_messages_t *rkmessages = NULL;
|
|
1112
|
+
const char *topic;
|
|
1113
|
+
const char *group = "0179-decommission-close";
|
|
1114
|
+
int32_t target_broker_id = -1;
|
|
1115
|
+
int32_t surviving_broker_id = -1;
|
|
1116
|
+
int32_t target_partition = -1;
|
|
1117
|
+
size_t rcvd = 0;
|
|
1118
|
+
int attempts = 0;
|
|
1119
|
+
int32_t surviving_part;
|
|
1120
|
+
expected_ack_t expected[2];
|
|
1121
|
+
ack_receipts_t receipts;
|
|
1122
|
+
|
|
1123
|
+
ack_receipts_init(&receipts);
|
|
1124
|
+
|
|
1125
|
+
SUB_TEST_QUICK("destroy_flags=0x%x ack_mode=%s", destroy_flags,
|
|
1126
|
+
explicit_ack ? "explicit" : "implicit");
|
|
1127
|
+
|
|
1128
|
+
/* Suppress expected __TRANSPORT / __ALL_BROKERS_DOWN errors that
|
|
1129
|
+
* fire when the target broker connection is dropped. */
|
|
1130
|
+
test_curr->is_fatal_cb = decommission_is_fatal_cb;
|
|
1131
|
+
|
|
1132
|
+
/* 2-broker mock cluster */
|
|
1133
|
+
mcluster = test_mock_cluster_new(2, &bootstraps);
|
|
1134
|
+
rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1);
|
|
1135
|
+
|
|
1136
|
+
/* 2-partition topic, RF=2: p0 led by broker 1, p1 led by broker 2. */
|
|
1137
|
+
topic = test_mk_topic_name("0179-decommission-close", 1);
|
|
1138
|
+
TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 2, 2) ==
|
|
1139
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1140
|
+
"Failed to create mock topic");
|
|
1141
|
+
TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 0, 1) ==
|
|
1142
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1143
|
+
"Failed to set partition 0 leader to broker 1");
|
|
1144
|
+
TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 1, 2) ==
|
|
1145
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1146
|
+
"Failed to set partition 1 leader to broker 2");
|
|
1147
|
+
|
|
1148
|
+
/* Produce TEST_MSGS/2 messages to each partition. */
|
|
1149
|
+
TEST_SAY("Producing %d messages to each partition of topic %s\n",
|
|
1150
|
+
TEST_MSGS / 2, topic);
|
|
1151
|
+
test_produce_msgs_easy_v(topic, 0, 0, 0, TEST_MSGS / 2, 16,
|
|
1152
|
+
"bootstrap.servers", bootstraps, NULL);
|
|
1153
|
+
test_produce_msgs_easy_v(topic, 0, 1, 0, TEST_MSGS / 2, 16,
|
|
1154
|
+
"bootstrap.servers", bootstraps, NULL);
|
|
1155
|
+
|
|
1156
|
+
TEST_SAY("Creating share consumer (%s ack)\n",
|
|
1157
|
+
explicit_ack ? "explicit" : "implicit");
|
|
1158
|
+
rkshare = new_share_consumer_for_mock_test(bootstraps, group,
|
|
1159
|
+
explicit_ack, &receipts);
|
|
1160
|
+
subscribe_topics(rkshare, &topic, 1);
|
|
1161
|
+
|
|
1162
|
+
/* Drain both partitions WITHOUT a trailing empty flush — the
|
|
1163
|
+
* last batch's records remain ACQUIRED (unacked). close() will
|
|
1164
|
+
* ship a ShareAck for those records to whichever broker served
|
|
1165
|
+
* them. */
|
|
1166
|
+
TEST_SAY("Consuming up to %d messages (max %d attempts)\n", TEST_MSGS,
|
|
1167
|
+
MAX_CONSUME_ATTEMPTS);
|
|
1168
|
+
while (rcvd < TEST_MSGS && attempts < MAX_CONSUME_ATTEMPTS) {
|
|
1169
|
+
size_t batch_rcvd;
|
|
1170
|
+
|
|
1171
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
1172
|
+
rkmessages = NULL;
|
|
1173
|
+
error = rd_kafka_share_poll(rkshare, 3000, &rkmessages);
|
|
1174
|
+
|
|
1175
|
+
if (error) {
|
|
1176
|
+
TEST_SAY("Attempt %d: consume error: %s\n", attempts,
|
|
1177
|
+
rd_kafka_error_string(error));
|
|
1178
|
+
rd_kafka_error_destroy(error);
|
|
1179
|
+
attempts++;
|
|
1180
|
+
continue;
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1183
|
+
batch_rcvd = rd_kafka_messages_count(rkmessages);
|
|
1184
|
+
if (batch_rcvd > 0) {
|
|
1185
|
+
TEST_SAY("Attempt %d: consumed %d messages\n", attempts,
|
|
1186
|
+
(int)batch_rcvd);
|
|
1187
|
+
rcvd += batch_rcvd;
|
|
1188
|
+
|
|
1189
|
+
if (explicit_ack) {
|
|
1190
|
+
size_t k;
|
|
1191
|
+
for (k = 0; k < batch_rcvd; k++) {
|
|
1192
|
+
rd_kafka_message_t *rkm =
|
|
1193
|
+
rd_kafka_messages_get(rkmessages,
|
|
1194
|
+
k);
|
|
1195
|
+
if (rkm)
|
|
1196
|
+
rd_kafka_share_acknowledge(
|
|
1197
|
+
rkshare, rkm);
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
attempts++;
|
|
1202
|
+
}
|
|
1203
|
+
TEST_ASSERT(rcvd == TEST_MSGS,
|
|
1204
|
+
"Expected %d messages, got %d after %d attempts", TEST_MSGS,
|
|
1205
|
+
(int)rcvd, attempts);
|
|
1206
|
+
TEST_SAY("Consumed %d msgs total in %d attempts\n", (int)rcvd,
|
|
1207
|
+
attempts);
|
|
1208
|
+
|
|
1209
|
+
drain_ack_callbacks(rkshare);
|
|
1210
|
+
/* The last message of the last batch tells us which broker
|
|
1211
|
+
* holds the unacked records — that's the broker close()'s
|
|
1212
|
+
* ShareAck will target. Decommission that broker; the other
|
|
1213
|
+
* survives. */
|
|
1214
|
+
{
|
|
1215
|
+
size_t _last_cnt = rd_kafka_messages_count(rkmessages);
|
|
1216
|
+
rd_kafka_message_t *_last_rkm =
|
|
1217
|
+
_last_cnt > 0
|
|
1218
|
+
? rd_kafka_messages_get(rkmessages, _last_cnt - 1)
|
|
1219
|
+
: NULL;
|
|
1220
|
+
TEST_ASSERT(_last_rkm != NULL,
|
|
1221
|
+
"Expected last batch to be non-empty");
|
|
1222
|
+
target_partition = _last_rkm->partition;
|
|
1223
|
+
}
|
|
1224
|
+
target_broker_id = (target_partition == 0) ? 1 : 2;
|
|
1225
|
+
surviving_broker_id = (target_broker_id == 1) ? 2 : 1;
|
|
1226
|
+
TEST_SAY("Target partition = %" PRId32 ", target broker = %" PRId32
|
|
1227
|
+
", surviving broker = %" PRId32 "\n",
|
|
1228
|
+
target_partition, target_broker_id, surviving_broker_id);
|
|
1229
|
+
|
|
1230
|
+
stage_broker_decommission(mcluster, topic, target_partition,
|
|
1231
|
+
target_broker_id, surviving_broker_id,
|
|
1232
|
+
RD_KAFKAP_ShareAcknowledge);
|
|
1233
|
+
|
|
1234
|
+
/* Call close(). It ships a ShareAck session leave
|
|
1235
|
+
* request for the unacked batch to the target broker.
|
|
1236
|
+
* The target broker's ShareAck is stamped __DESTROY_BROKER
|
|
1237
|
+
* when the decommission fires; close() must not surface this to the
|
|
1238
|
+
* app. */
|
|
1239
|
+
TEST_SAY(
|
|
1240
|
+
"Calling rd_kafka_share_consumer_close() — expecting "
|
|
1241
|
+
"NULL error\n");
|
|
1242
|
+
error = rd_kafka_share_consumer_close(rkshare);
|
|
1243
|
+
|
|
1244
|
+
TEST_ASSERT(error == NULL,
|
|
1245
|
+
"Expected close() to return NULL error, got %s",
|
|
1246
|
+
error ? rd_kafka_error_string(error) : "(null)");
|
|
1247
|
+
|
|
1248
|
+
/* Assert ack callback receipts.
|
|
1249
|
+
*
|
|
1250
|
+
* The first batch on each broker was piggyback-acked on the
|
|
1251
|
+
* next ShareFetch during drain — those records appear in the
|
|
1252
|
+
* callback with NO_ERROR. The LAST batch on each broker is
|
|
1253
|
+
* still ACQUIRED at close time; close ships those acks
|
|
1254
|
+
* differently in implicit vs explicit mode:
|
|
1255
|
+
*
|
|
1256
|
+
* Implicit ack: the close path only flushes already-converted
|
|
1257
|
+
* (non-ACQUIRED) entries. ACQUIRED records on the last batch
|
|
1258
|
+
* are silently dropped — no ShareAck for them goes on the
|
|
1259
|
+
* wire, so no callback fires for target_partition's last
|
|
1260
|
+
* batch (which would have been stamped __DESTROY_BROKER).
|
|
1261
|
+
* Expectation: only surviving_partition NO_ERROR.
|
|
1262
|
+
*
|
|
1263
|
+
* Explicit ack: the drain loop explicitly acked every record.
|
|
1264
|
+
* Those explicit acks are queued and shipped during close.
|
|
1265
|
+
* The target broker's ack request parks behind the injected
|
|
1266
|
+
* delay; when the decommission fires, the parked request is
|
|
1267
|
+
* stamped __DESTROY_BROKER, which the app-facing funnel
|
|
1268
|
+
* translates to __TRANSPORT before the callback fires.
|
|
1269
|
+
* surviving_partition records that weren't piggyback-acked
|
|
1270
|
+
* during drain are also shipped at close and succeed with
|
|
1271
|
+
* NO_ERROR.
|
|
1272
|
+
*/
|
|
1273
|
+
surviving_part = (target_partition == 0) ? 1 : 0;
|
|
1274
|
+
expected[0] = (expected_ack_t) {
|
|
1275
|
+
topic, surviving_part, RD_KAFKA_RESP_ERR_NO_ERROR, TEST_MSGS / 2};
|
|
1276
|
+
if (explicit_ack) {
|
|
1277
|
+
expected[1] = (expected_ack_t) {topic, target_partition,
|
|
1278
|
+
RD_KAFKA_RESP_ERR__TRANSPORT,
|
|
1279
|
+
TEST_MSGS / 2};
|
|
1280
|
+
verify_ack_receipts(&receipts, expected, 2, "close explicit");
|
|
1281
|
+
} else {
|
|
1282
|
+
verify_ack_receipts(&receipts, expected, 1, "close implicit");
|
|
1283
|
+
}
|
|
1284
|
+
|
|
1285
|
+
/* Cleanup */
|
|
1286
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
1287
|
+
rkmessages = NULL;
|
|
1288
|
+
|
|
1289
|
+
destroy_share_consumer(rkshare, destroy_flags);
|
|
1290
|
+
test_mock_cluster_destroy(mcluster);
|
|
1291
|
+
|
|
1292
|
+
ack_receipts_destroy(&receipts);
|
|
1293
|
+
|
|
1294
|
+
/* Restore the default fatal-error handler. */
|
|
1295
|
+
test_curr->is_fatal_cb = NULL;
|
|
1296
|
+
|
|
1297
|
+
SUB_TEST_PASS();
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1300
|
+
|
|
1301
|
+
/**
|
|
1302
|
+
* @brief Test that rd_kafka_share_commit_async() does not hang when the
|
|
1303
|
+
* broker it ships acks to is decommissioned mid-flight.
|
|
1304
|
+
*
|
|
1305
|
+
* Setup mirrors test_broker_decommission_during_close — drain both
|
|
1306
|
+
* partitions without a trailing flush so the last batch's records
|
|
1307
|
+
* stay ACQUIRED. commit_async() then ships a ShareAck for those
|
|
1308
|
+
* records to whichever broker served them. The target broker has a
|
|
1309
|
+
* 5s ShareAck rtt delay; while the request is parked, the
|
|
1310
|
+
* background thread removes the broker from mock metadata and the
|
|
1311
|
+
* client decommissions it, purging the in-flight ShareAck with
|
|
1312
|
+
* __DESTROY_BROKER.
|
|
1313
|
+
*
|
|
1314
|
+
* Expectation: rd_kafka_share_commit_async() returns NULL (no
|
|
1315
|
+
* error) and does not block on the in-flight ack.
|
|
1316
|
+
* destroy() afterwards completes cleanly.
|
|
1317
|
+
*/
|
|
1318
|
+
static void test_broker_decommission_with_commit_async(int destroy_flags,
|
|
1319
|
+
rd_bool_t explicit_ack) {
|
|
1320
|
+
rd_kafka_mock_cluster_t *mcluster;
|
|
1321
|
+
const char *bootstraps;
|
|
1322
|
+
rd_kafka_share_t *rkshare;
|
|
1323
|
+
rd_kafka_error_t *error;
|
|
1324
|
+
rd_kafka_messages_t *rkmessages = NULL;
|
|
1325
|
+
const char *topic;
|
|
1326
|
+
const char *group = "0179-decommission-commit-async";
|
|
1327
|
+
int32_t target_broker_id = -1;
|
|
1328
|
+
int32_t surviving_broker_id = -1;
|
|
1329
|
+
int32_t target_partition = -1;
|
|
1330
|
+
size_t rcvd = 0;
|
|
1331
|
+
int attempts = 0;
|
|
1332
|
+
int32_t surviving_part;
|
|
1333
|
+
expected_ack_t expected[2];
|
|
1334
|
+
ack_receipts_t receipts;
|
|
1335
|
+
|
|
1336
|
+
ack_receipts_init(&receipts);
|
|
1337
|
+
|
|
1338
|
+
SUB_TEST_QUICK("destroy_flags=0x%x ack_mode=%s", destroy_flags,
|
|
1339
|
+
explicit_ack ? "explicit" : "implicit");
|
|
1340
|
+
|
|
1341
|
+
/* Suppress expected __TRANSPORT / __ALL_BROKERS_DOWN errors that
|
|
1342
|
+
* fire when the target broker connection is dropped. */
|
|
1343
|
+
test_curr->is_fatal_cb = decommission_is_fatal_cb;
|
|
1344
|
+
|
|
1345
|
+
/* 2-broker mock cluster */
|
|
1346
|
+
mcluster = test_mock_cluster_new(2, &bootstraps);
|
|
1347
|
+
rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1);
|
|
1348
|
+
|
|
1349
|
+
/* 2-partition topic, RF=2: p0 led by broker 1, p1 led by broker 2. */
|
|
1350
|
+
topic = test_mk_topic_name("0179-decommission-commit-async", 1);
|
|
1351
|
+
TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 2, 2) ==
|
|
1352
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1353
|
+
"Failed to create mock topic");
|
|
1354
|
+
TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 0, 1) ==
|
|
1355
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1356
|
+
"Failed to set partition 0 leader to broker 1");
|
|
1357
|
+
TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 1, 2) ==
|
|
1358
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1359
|
+
"Failed to set partition 1 leader to broker 2");
|
|
1360
|
+
|
|
1361
|
+
/* Produce TEST_MSGS/2 messages to each partition. */
|
|
1362
|
+
TEST_SAY("Producing %d messages to each partition of topic %s\n",
|
|
1363
|
+
TEST_MSGS / 2, topic);
|
|
1364
|
+
test_produce_msgs_easy_v(topic, 0, 0, 0, TEST_MSGS / 2, 16,
|
|
1365
|
+
"bootstrap.servers", bootstraps, NULL);
|
|
1366
|
+
test_produce_msgs_easy_v(topic, 0, 1, 0, TEST_MSGS / 2, 16,
|
|
1367
|
+
"bootstrap.servers", bootstraps, NULL);
|
|
1368
|
+
|
|
1369
|
+
/* Implicit-ack share consumer with fast metadata refresh so the
|
|
1370
|
+
* client picks up the target broker's removal during the 5s
|
|
1371
|
+
* ShareAck delay. */
|
|
1372
|
+
TEST_SAY("Creating share consumer (%s ack) with ack_cb\n",
|
|
1373
|
+
explicit_ack ? "explicit" : "implicit");
|
|
1374
|
+
rkshare = new_share_consumer_for_mock_test(bootstraps, group,
|
|
1375
|
+
explicit_ack, &receipts);
|
|
1376
|
+
subscribe_topics(rkshare, &topic, 1);
|
|
1377
|
+
|
|
1378
|
+
/* Drain both partitions WITHOUT a trailing empty flush — the
|
|
1379
|
+
* last batch's records remain ACQUIRED (unacked).
|
|
1380
|
+
* commit_async() will ship a ShareAck for those records to
|
|
1381
|
+
* whichever broker served them. */
|
|
1382
|
+
TEST_SAY("Consuming up to %d messages (max %d attempts)\n", TEST_MSGS,
|
|
1383
|
+
MAX_CONSUME_ATTEMPTS);
|
|
1384
|
+
while (rcvd < TEST_MSGS && attempts < MAX_CONSUME_ATTEMPTS) {
|
|
1385
|
+
size_t batch_rcvd;
|
|
1386
|
+
|
|
1387
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
1388
|
+
rkmessages = NULL;
|
|
1389
|
+
error = rd_kafka_share_poll(rkshare, 3000, &rkmessages);
|
|
1390
|
+
|
|
1391
|
+
if (error) {
|
|
1392
|
+
TEST_SAY("Attempt %d: consume error: %s\n", attempts,
|
|
1393
|
+
rd_kafka_error_string(error));
|
|
1394
|
+
rd_kafka_error_destroy(error);
|
|
1395
|
+
attempts++;
|
|
1396
|
+
continue;
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
batch_rcvd = rd_kafka_messages_count(rkmessages);
|
|
1400
|
+
if (batch_rcvd > 0) {
|
|
1401
|
+
TEST_SAY("Attempt %d: consumed %d messages\n", attempts,
|
|
1402
|
+
(int)batch_rcvd);
|
|
1403
|
+
rcvd += batch_rcvd;
|
|
1404
|
+
|
|
1405
|
+
if (explicit_ack) {
|
|
1406
|
+
size_t k;
|
|
1407
|
+
for (k = 0; k < batch_rcvd; k++) {
|
|
1408
|
+
rd_kafka_message_t *rkm =
|
|
1409
|
+
rd_kafka_messages_get(rkmessages,
|
|
1410
|
+
k);
|
|
1411
|
+
if (rkm)
|
|
1412
|
+
rd_kafka_share_acknowledge(
|
|
1413
|
+
rkshare, rkm);
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
attempts++;
|
|
1418
|
+
}
|
|
1419
|
+
TEST_ASSERT(rcvd == TEST_MSGS,
|
|
1420
|
+
"Expected %d messages, got %d after %d attempts", TEST_MSGS,
|
|
1421
|
+
(int)rcvd, attempts);
|
|
1422
|
+
TEST_SAY("Consumed %d msgs total in %d attempts\n", (int)rcvd,
|
|
1423
|
+
attempts);
|
|
1424
|
+
|
|
1425
|
+
drain_ack_callbacks(rkshare);
|
|
1426
|
+
/* The last message of the last batch tells us which broker
|
|
1427
|
+
* holds the unacked records — that's the broker the
|
|
1428
|
+
* commit_async ShareAck will target. Decommission that broker;
|
|
1429
|
+
* the other survives. */
|
|
1430
|
+
{
|
|
1431
|
+
size_t _last_cnt = rd_kafka_messages_count(rkmessages);
|
|
1432
|
+
rd_kafka_message_t *_last_rkm =
|
|
1433
|
+
_last_cnt > 0
|
|
1434
|
+
? rd_kafka_messages_get(rkmessages, _last_cnt - 1)
|
|
1435
|
+
: NULL;
|
|
1436
|
+
TEST_ASSERT(_last_rkm != NULL,
|
|
1437
|
+
"Expected last batch to be non-empty");
|
|
1438
|
+
target_partition = _last_rkm->partition;
|
|
1439
|
+
}
|
|
1440
|
+
target_broker_id = (target_partition == 0) ? 1 : 2;
|
|
1441
|
+
surviving_broker_id = (target_broker_id == 1) ? 2 : 1;
|
|
1442
|
+
TEST_SAY("Target partition = %" PRId32 ", target broker = %" PRId32
|
|
1443
|
+
", surviving broker = %" PRId32 "\n",
|
|
1444
|
+
target_partition, target_broker_id, surviving_broker_id);
|
|
1445
|
+
|
|
1446
|
+
stage_broker_decommission(mcluster, topic, target_partition,
|
|
1447
|
+
target_broker_id, surviving_broker_id,
|
|
1448
|
+
RD_KAFKAP_ShareAcknowledge);
|
|
1449
|
+
|
|
1450
|
+
/* Call commit_async(). It should return immediately without
|
|
1451
|
+
* blocking on the in-flight ack. The parked ShareAck on the
|
|
1452
|
+
* target broker will be stamped __DESTROY_BROKER when the
|
|
1453
|
+
* decommission fires shortly after. */
|
|
1454
|
+
TEST_SAY(
|
|
1455
|
+
"Calling rd_kafka_share_commit_async() — expecting "
|
|
1456
|
+
"NULL error and no hang\n");
|
|
1457
|
+
error = rd_kafka_share_commit_async(rkshare);
|
|
1458
|
+
|
|
1459
|
+
TEST_ASSERT(error == NULL,
|
|
1460
|
+
"Expected commit_async() to return NULL error, got %s",
|
|
1461
|
+
error ? rd_kafka_error_string(error) : "(null)");
|
|
1462
|
+
|
|
1463
|
+
/* Cleanup messages before destroy. */
|
|
1464
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
1465
|
+
rkmessages = NULL;
|
|
1466
|
+
|
|
1467
|
+
destroy_share_consumer(rkshare, destroy_flags);
|
|
1468
|
+
test_mock_cluster_destroy(mcluster);
|
|
1469
|
+
|
|
1470
|
+
/* Assert ack callback receipts.
|
|
1471
|
+
*
|
|
1472
|
+
* Full close (destroy_flags=0): destroy waits for in-flight
|
|
1473
|
+
* requests. The metadata-driven decommission fires within ~2s,
|
|
1474
|
+
* stamping the parked ShareAck with __DESTROY_BROKER.
|
|
1475
|
+
*
|
|
1476
|
+
* NO_CONSUMER_CLOSE (destroy_flags=0x8): destroy short-
|
|
1477
|
+
* circuits without waiting for the parked ShareAck. No
|
|
1478
|
+
* callback fires for the target partition's records. Only the
|
|
1479
|
+
* surviving partition's ack — which completed before —
|
|
1480
|
+
* surfaces. */
|
|
1481
|
+
surviving_part = (target_partition == 0) ? 1 : 0;
|
|
1482
|
+
expected[0] = (expected_ack_t) {
|
|
1483
|
+
topic, surviving_part, RD_KAFKA_RESP_ERR_NO_ERROR, TEST_MSGS / 2};
|
|
1484
|
+
if (destroy_flags & RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE) {
|
|
1485
|
+
verify_ack_receipts(&receipts, expected, 1,
|
|
1486
|
+
"commit_async NO_CONSUMER_CLOSE");
|
|
1487
|
+
} else {
|
|
1488
|
+
/* __DESTROY_BROKER stamped on parked acks is translated
|
|
1489
|
+
* to __TRANSPORT at the app-facing funnel. */
|
|
1490
|
+
expected[1] = (expected_ack_t) {topic, target_partition,
|
|
1491
|
+
RD_KAFKA_RESP_ERR__TRANSPORT,
|
|
1492
|
+
TEST_MSGS / 2};
|
|
1493
|
+
verify_ack_receipts(&receipts, expected, 2,
|
|
1494
|
+
"commit_async full close");
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
ack_receipts_destroy(&receipts);
|
|
1498
|
+
|
|
1499
|
+
/* Restore the default fatal-error handler. */
|
|
1500
|
+
test_curr->is_fatal_cb = NULL;
|
|
1501
|
+
|
|
1502
|
+
SUB_TEST_PASS();
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
static void test_leader_migration_mid_session_destroy(int destroy_flags) {
|
|
1506
|
+
rd_kafka_mock_cluster_t *mcluster;
|
|
1507
|
+
const char *bootstraps;
|
|
1508
|
+
rd_kafka_share_t *rkshare;
|
|
1509
|
+
rd_kafka_error_t *error;
|
|
1510
|
+
rd_kafka_messages_t *rkmessages = NULL;
|
|
1511
|
+
const char *topic = "0179-leader-migration-mid-session";
|
|
1512
|
+
const char *group = "0179-leader-migration-mid-session";
|
|
1513
|
+
const int msgs_per_round = 5;
|
|
1514
|
+
size_t rcvd = 0;
|
|
1515
|
+
int attempts = 0;
|
|
1516
|
+
|
|
1517
|
+
SUB_TEST_QUICK("destroy_flags=0x%x", destroy_flags);
|
|
1518
|
+
|
|
1519
|
+
mcluster = test_mock_cluster_new(2, &bootstraps);
|
|
1520
|
+
rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1);
|
|
1521
|
+
|
|
1522
|
+
/* 1 partition, RF=2 so both brokers know about it. Initial
|
|
1523
|
+
* leader = broker 1. */
|
|
1524
|
+
TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 1, 2) ==
|
|
1525
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1526
|
+
"Failed to create mock topic");
|
|
1527
|
+
TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 0, 1) ==
|
|
1528
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1529
|
+
"Failed to set initial leader to broker 1");
|
|
1530
|
+
|
|
1531
|
+
/* Produce a first batch so the consumer can fetch from broker 1. */
|
|
1532
|
+
TEST_SAY("Producing %d messages to broker 1\n", msgs_per_round);
|
|
1533
|
+
test_produce_msgs_easy_v(topic, 0, 0, 0, msgs_per_round, 16,
|
|
1534
|
+
"bootstrap.servers", bootstraps, NULL);
|
|
1535
|
+
|
|
1536
|
+
rkshare = new_share_consumer_for_mock_test(
|
|
1537
|
+
bootstraps, group, rd_false /* implicit */, NULL);
|
|
1538
|
+
subscribe_topics(rkshare, &topic, 1);
|
|
1539
|
+
|
|
1540
|
+
/* Round 1: consume from broker 1. After this, the toppar is in
|
|
1541
|
+
* broker 1's toppars_in_session. */
|
|
1542
|
+
TEST_SAY("Round 1: consume from broker 1 (initial leader)\n");
|
|
1543
|
+
while (rcvd < (size_t)msgs_per_round &&
|
|
1544
|
+
attempts++ < MAX_CONSUME_ATTEMPTS) {
|
|
1545
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
1546
|
+
rkmessages = NULL;
|
|
1547
|
+
error = rd_kafka_share_poll(rkshare, 3000, &rkmessages);
|
|
1548
|
+
if (error)
|
|
1549
|
+
rd_kafka_error_destroy(error);
|
|
1550
|
+
else
|
|
1551
|
+
rcvd += rd_kafka_messages_count(rkmessages);
|
|
1552
|
+
}
|
|
1553
|
+
TEST_ASSERT(rcvd >= (size_t)msgs_per_round,
|
|
1554
|
+
"Round 1: expected %d msgs, got %d", msgs_per_round,
|
|
1555
|
+
(int)rcvd);
|
|
1556
|
+
TEST_SAY("Round 1: consumed %d messages\n", (int)rcvd);
|
|
1557
|
+
|
|
1558
|
+
/* Migrate the partition leader to broker 2. */
|
|
1559
|
+
TEST_SAY("Migrating partition leader from broker 1 to broker 2\n");
|
|
1560
|
+
TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 0, 2) ==
|
|
1561
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1562
|
+
"Failed to migrate leader to broker 2");
|
|
1563
|
+
|
|
1564
|
+
/* Produce a second batch so the consumer has something to fetch
|
|
1565
|
+
* from the new leader. */
|
|
1566
|
+
TEST_SAY("Producing %d more messages (now under broker 2)\n",
|
|
1567
|
+
msgs_per_round);
|
|
1568
|
+
test_produce_msgs_easy_v(topic, 0, 0, 0, msgs_per_round, 16,
|
|
1569
|
+
"bootstrap.servers", bootstraps, NULL);
|
|
1570
|
+
|
|
1571
|
+
/* Round 2: consume from broker 2. The first ShareFetch reply
|
|
1572
|
+
* from broker 2 will add the toppar into broker
|
|
1573
|
+
* 2's toppars_in_session. With the fix, broker 1's list
|
|
1574
|
+
* is cleared by PARTITION_LEAVE before this happens. */
|
|
1575
|
+
TEST_SAY("Round 2: consume from broker 2 (new leader)\n");
|
|
1576
|
+
attempts = 0;
|
|
1577
|
+
while (rcvd < (size_t)(2 * msgs_per_round) &&
|
|
1578
|
+
attempts++ < MAX_CONSUME_ATTEMPTS) {
|
|
1579
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
1580
|
+
rkmessages = NULL;
|
|
1581
|
+
error = rd_kafka_share_poll(rkshare, 3000, &rkmessages);
|
|
1582
|
+
if (error)
|
|
1583
|
+
rd_kafka_error_destroy(error);
|
|
1584
|
+
else
|
|
1585
|
+
rcvd += rd_kafka_messages_count(rkmessages);
|
|
1586
|
+
}
|
|
1587
|
+
TEST_ASSERT(rcvd >= (size_t)(2 * msgs_per_round),
|
|
1588
|
+
"Round 2: expected %d total msgs, got %d",
|
|
1589
|
+
2 * msgs_per_round, (int)rcvd);
|
|
1590
|
+
TEST_SAY("Round 2: consumed %d total messages\n", (int)rcvd);
|
|
1591
|
+
|
|
1592
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
1593
|
+
rkmessages = NULL;
|
|
1594
|
+
|
|
1595
|
+
/* Close + destroy. Without the leader-migration fix, broker 1's
|
|
1596
|
+
* destroy_final would assert on non-empty toppars_in_session. */
|
|
1597
|
+
TEST_SAY("Calling destroy (flags 0x%x)\n", destroy_flags);
|
|
1598
|
+
destroy_share_consumer(rkshare, destroy_flags);
|
|
1599
|
+
|
|
1600
|
+
test_mock_cluster_destroy(mcluster);
|
|
1601
|
+
SUB_TEST_PASS();
|
|
1602
|
+
}
|
|
1603
|
+
|
|
1604
|
+
/**
|
|
1605
|
+
* @brief Destroy while the consumer's cgrp is mid-rebalance.
|
|
1606
|
+
*
|
|
1607
|
+
* Drives the cgrp into an intermediate join-state by:
|
|
1608
|
+
* 1. Subscribing and consuming at least one batch (cgrp now in steady
|
|
1609
|
+
* state with a live assignment).
|
|
1610
|
+
* 2. Injecting a long RTT delay on ShareGroupHeartbeat so the next
|
|
1611
|
+
* heartbeat sent by the cgrp will be stuck in-flight.
|
|
1612
|
+
* 3. Calling unsubscribe, which drives the cgrp into
|
|
1613
|
+
* wait-unassign-call -> wait-incr-unassign-to-complete and ends up
|
|
1614
|
+
* waiting on the (now-blocked) leave heartbeat.
|
|
1615
|
+
* 4. Calling destroy with the given flags. Destroy must NOT wait the
|
|
1616
|
+
* full injected RTT.
|
|
1617
|
+
*/
|
|
1618
|
+
static void test_destroy_during_rebalance(int destroy_flags) {
|
|
1619
|
+
rd_kafka_mock_cluster_t *mcluster;
|
|
1620
|
+
const char *bootstraps;
|
|
1621
|
+
rd_kafka_share_t *rkshare;
|
|
1622
|
+
rd_kafka_error_t *error;
|
|
1623
|
+
rd_kafka_messages_t *rkmessages = NULL;
|
|
1624
|
+
const char *topic = "0179-destroy-during-rebalance";
|
|
1625
|
+
const char *group = "0179-destroy-during-rebalance";
|
|
1626
|
+
const int heartbeat_delay_ms = 10000;
|
|
1627
|
+
size_t rcvd = 0;
|
|
1628
|
+
int attempts = 0;
|
|
1629
|
+
rd_ts_t t_start, t_elapsed_ms;
|
|
1630
|
+
|
|
1631
|
+
SUB_TEST_QUICK("destroy_flags=0x%x", destroy_flags);
|
|
1632
|
+
|
|
1633
|
+
mcluster = test_mock_cluster_new(1, &bootstraps);
|
|
1634
|
+
rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1);
|
|
1635
|
+
|
|
1636
|
+
TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 1, 1) ==
|
|
1637
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1638
|
+
"Failed to create mock topic");
|
|
1639
|
+
|
|
1640
|
+
/* Produce so the consumer's first consume_batch returns a real
|
|
1641
|
+
* message, guaranteeing the cgrp reached up/steady with a live
|
|
1642
|
+
* assignment. */
|
|
1643
|
+
test_produce_msgs_easy_v(topic, 0, 0, 0, 10, 16, "bootstrap.servers",
|
|
1644
|
+
bootstraps, NULL);
|
|
1645
|
+
|
|
1646
|
+
rkshare = new_share_consumer_for_mock_test(
|
|
1647
|
+
bootstraps, group, rd_false /* implicit ack */, NULL);
|
|
1648
|
+
subscribe_topics(rkshare, &topic, 1);
|
|
1649
|
+
|
|
1650
|
+
TEST_SAY("Waiting for first batch (signals assignment is live)\n");
|
|
1651
|
+
while (rcvd == 0 && attempts++ < MAX_CONSUME_ATTEMPTS) {
|
|
1652
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
1653
|
+
rkmessages = NULL;
|
|
1654
|
+
error = rd_kafka_share_poll(rkshare, 3000, &rkmessages);
|
|
1655
|
+
if (error)
|
|
1656
|
+
rd_kafka_error_destroy(error);
|
|
1657
|
+
else
|
|
1658
|
+
rcvd = rd_kafka_messages_count(rkmessages);
|
|
1659
|
+
}
|
|
1660
|
+
TEST_ASSERT(rcvd > 0,
|
|
1661
|
+
"Expected at least 1 msg before forcing rebalance; "
|
|
1662
|
+
"got %d after %d attempts",
|
|
1663
|
+
(int)rcvd, attempts);
|
|
1664
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
1665
|
+
rkmessages = NULL;
|
|
1666
|
+
|
|
1667
|
+
/* Block all subsequent ShareGroupHeartbeats. The next heartbeat
|
|
1668
|
+
* sent by the cgrp (including the leave-group heartbeat) will be
|
|
1669
|
+
* stuck in flight for heartbeat_delay_ms. */
|
|
1670
|
+
TEST_SAY("Injecting %dms RTT delay on ShareGroupHeartbeat\n",
|
|
1671
|
+
heartbeat_delay_ms);
|
|
1672
|
+
TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts(
|
|
1673
|
+
mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1,
|
|
1674
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1675
|
+
heartbeat_delay_ms) == RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1676
|
+
"Failed to inject heartbeat delay");
|
|
1677
|
+
|
|
1678
|
+
/* Drive the cgrp into the leave-flow. */
|
|
1679
|
+
TEST_SAY("Calling unsubscribe to drive cgrp into wait-incr-unassign\n");
|
|
1680
|
+
TEST_ASSERT(!rd_kafka_share_unsubscribe(rkshare), "unsubscribe failed");
|
|
1681
|
+
|
|
1682
|
+
/* Give the cgrp main thread time to process the unsubscribe op
|
|
1683
|
+
* and transition into the intermediate join-state. */
|
|
1684
|
+
rd_sleep(1);
|
|
1685
|
+
|
|
1686
|
+
TEST_SAY("Calling destroy mid-rebalance (flags 0x%x)\n", destroy_flags);
|
|
1687
|
+
t_start = test_clock();
|
|
1688
|
+
destroy_share_consumer(rkshare, destroy_flags);
|
|
1689
|
+
t_elapsed_ms = (test_clock() - t_start) / 1000;
|
|
1690
|
+
TEST_SAY("Destroy completed in %" PRId64 " ms\n", t_elapsed_ms);
|
|
1691
|
+
|
|
1692
|
+
if (destroy_flags & RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE) {
|
|
1693
|
+
/* No-close: destroy must skip the cgrp leave flow and
|
|
1694
|
+
* return promptly, regardless of the heartbeat delay. */
|
|
1695
|
+
TEST_ASSERT(t_elapsed_ms < 2000,
|
|
1696
|
+
"Destroy(NO_CONSUMER_CLOSE) took %" PRId64
|
|
1697
|
+
" ms, expected < 2000 ms",
|
|
1698
|
+
t_elapsed_ms);
|
|
1699
|
+
} else {
|
|
1700
|
+
/* Full close: destroy waits for the cgrp to finish its
|
|
1701
|
+
* leave flow, which is blocked on the heartbeat. It must
|
|
1702
|
+
* still complete in roughly heartbeat_delay_ms (plus
|
|
1703
|
+
* small overhead) and never wait indefinitely. */
|
|
1704
|
+
TEST_ASSERT(t_elapsed_ms < heartbeat_delay_ms + 5000,
|
|
1705
|
+
"Destroy took %" PRId64 " ms, expected < %d ms",
|
|
1706
|
+
t_elapsed_ms, heartbeat_delay_ms + 5000);
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
test_mock_cluster_destroy(mcluster);
|
|
1710
|
+
SUB_TEST_PASS();
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
/**
|
|
1714
|
+
* @brief Test destroying a share consumer after a fatal error.
|
|
1715
|
+
*
|
|
1716
|
+
* Drives the consumer to steady state, acks half the consumed batch,
|
|
1717
|
+
* then injects a 30s RTT delay on the next ShareAcknowledge so a real
|
|
1718
|
+
* close path would have to wait that long. Trigger a fatal error
|
|
1719
|
+
* (which causes destroy_app to promote flags to NO_CONSUMER_CLOSE,
|
|
1720
|
+
* then call destroy.
|
|
1721
|
+
*
|
|
1722
|
+
* Destroy must short-circuit the close path and complete promptly
|
|
1723
|
+
* regardless of the supplied destroy_flags — the delayed ShareAck
|
|
1724
|
+
* never gets sent, and broker threads must exit without waiting.
|
|
1725
|
+
*
|
|
1726
|
+
* @param destroy_flags Destroy flags (0 or
|
|
1727
|
+
* RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE). Both
|
|
1728
|
+
* must complete fast under fatal-error promotion.
|
|
1729
|
+
*/
|
|
1730
|
+
static void test_destroy_with_fatal_error(int destroy_flags) {
|
|
1731
|
+
rd_kafka_mock_cluster_t *mcluster;
|
|
1732
|
+
const char *bootstraps;
|
|
1733
|
+
rd_kafka_share_t *rkshare;
|
|
1734
|
+
rd_kafka_t *rk;
|
|
1735
|
+
rd_kafka_error_t *error;
|
|
1736
|
+
rd_kafka_resp_err_t err;
|
|
1737
|
+
rd_kafka_messages_t *rkmessages = NULL;
|
|
1738
|
+
const char *topic = "0179-destroy-fatal-error";
|
|
1739
|
+
const char *group = "0179-destroy-fatal-error";
|
|
1740
|
+
const int total_msgs = 10;
|
|
1741
|
+
const int share_ack_delay_ms = 30000;
|
|
1742
|
+
size_t rcvd = 0;
|
|
1743
|
+
int attempts = 0;
|
|
1744
|
+
size_t i;
|
|
1745
|
+
rd_ts_t t_start, t_elapsed_ms;
|
|
1746
|
+
|
|
1747
|
+
SUB_TEST_QUICK("destroy_flags=0x%x", destroy_flags);
|
|
1748
|
+
|
|
1749
|
+
mcluster = test_mock_cluster_new(1, &bootstraps);
|
|
1750
|
+
rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1);
|
|
1751
|
+
|
|
1752
|
+
TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 1, 1) ==
|
|
1753
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1754
|
+
"Failed to create mock topic");
|
|
1755
|
+
|
|
1756
|
+
test_produce_msgs_easy_v(topic, 0, 0, 0, total_msgs, 16,
|
|
1757
|
+
"bootstrap.servers", bootstraps, NULL);
|
|
1758
|
+
|
|
1759
|
+
rkshare = new_share_consumer_for_mock_test(
|
|
1760
|
+
bootstraps, group, rd_true /* explicit */, NULL);
|
|
1761
|
+
rk = test_share_consumer_get_rk(rkshare);
|
|
1762
|
+
subscribe_topics(rkshare, &topic, 1);
|
|
1763
|
+
|
|
1764
|
+
TEST_SAY("Consuming %d msgs\n", total_msgs);
|
|
1765
|
+
while (rcvd < (size_t)total_msgs && attempts++ < MAX_CONSUME_ATTEMPTS) {
|
|
1766
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
1767
|
+
rkmessages = NULL;
|
|
1768
|
+
error = rd_kafka_share_poll(rkshare, 3000, &rkmessages);
|
|
1769
|
+
if (error)
|
|
1770
|
+
rd_kafka_error_destroy(error);
|
|
1771
|
+
else
|
|
1772
|
+
rcvd = rd_kafka_messages_count(rkmessages);
|
|
1773
|
+
}
|
|
1774
|
+
TEST_ASSERT(rcvd >= (size_t)total_msgs,
|
|
1775
|
+
"Expected %d msgs, got %d after %d attempts", total_msgs,
|
|
1776
|
+
(int)rcvd, attempts);
|
|
1777
|
+
|
|
1778
|
+
TEST_SAY("Acknowledging %d/%d messages (no commit)\n", (int)(rcvd / 2),
|
|
1779
|
+
(int)rcvd);
|
|
1780
|
+
for (i = 0; i < rcvd / 2; i++) {
|
|
1781
|
+
rd_kafka_message_t *rkm = rd_kafka_messages_get(rkmessages, i);
|
|
1782
|
+
if (rkm)
|
|
1783
|
+
rd_kafka_share_acknowledge(rkshare, rkm);
|
|
1784
|
+
}
|
|
1785
|
+
|
|
1786
|
+
/* Inject a long delay on the next ShareAcknowledge so a normal
|
|
1787
|
+
* close path would block on it. The fatal-error path must NOT
|
|
1788
|
+
* wait. */
|
|
1789
|
+
TEST_SAY("Injecting %dms RTT delay on ShareAcknowledge\n",
|
|
1790
|
+
share_ack_delay_ms);
|
|
1791
|
+
TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts(
|
|
1792
|
+
mcluster, 1, RD_KAFKAP_ShareAcknowledge, 1,
|
|
1793
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1794
|
+
share_ack_delay_ms) == RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1795
|
+
"Failed to inject ShareAcknowledge delay");
|
|
1796
|
+
|
|
1797
|
+
TEST_SAY("Injecting fatal error\n");
|
|
1798
|
+
err = rd_kafka_test_fatal_error(
|
|
1799
|
+
rk, RD_KAFKA_RESP_ERR__FATAL,
|
|
1800
|
+
"0179: injected fatal error before destroy");
|
|
1801
|
+
TEST_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1802
|
+
"test_fatal_error returned %s", rd_kafka_err2name(err));
|
|
1803
|
+
|
|
1804
|
+
/* Destroy all consumed messages before share_destroy. */
|
|
1805
|
+
rd_kafka_messages_destroy(rkmessages);
|
|
1806
|
+
rkmessages = NULL;
|
|
1807
|
+
|
|
1808
|
+
TEST_SAY("Calling destroy with flags 0x%x (fatal-error path)\n",
|
|
1809
|
+
destroy_flags);
|
|
1810
|
+
t_start = test_clock();
|
|
1811
|
+
destroy_share_consumer(rkshare, destroy_flags);
|
|
1812
|
+
t_elapsed_ms = (test_clock() - t_start) / 1000;
|
|
1813
|
+
TEST_SAY("Destroy completed in %" PRId64 " ms\n", t_elapsed_ms);
|
|
1814
|
+
|
|
1815
|
+
/* Fatal error short-circuits close regardless of supplied flag,
|
|
1816
|
+
* so destroy must be prompt and never wait the injected RTT. */
|
|
1817
|
+
TEST_ASSERT(t_elapsed_ms < 2000,
|
|
1818
|
+
"Destroy took %" PRId64 " ms, expected < 2000 ms",
|
|
1819
|
+
t_elapsed_ms);
|
|
1820
|
+
|
|
1821
|
+
test_mock_cluster_destroy(mcluster);
|
|
1822
|
+
SUB_TEST_PASS();
|
|
1823
|
+
}
|
|
1824
|
+
|
|
1825
|
+
|
|
1826
|
+
/**
|
|
1827
|
+
* @brief Test destroying share consumer with explicit-ack mode.
|
|
1828
|
+
*
|
|
1829
|
+
* Consumes a batch of messages, explicitly acknowledges some/all of
|
|
1830
|
+
* them (no commit), then destroys the consumer. Verifies that destroy
|
|
1831
|
+
* handles pending acks correctly.
|
|
1832
|
+
*
|
|
1833
|
+
* @param destroy_flags Destroy flags (0 or
|
|
1834
|
+
* RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE).
|
|
1835
|
+
* @param ack_half If true, ack only the first half of the
|
|
1836
|
+
* consumed batch (the rest are left unacked). If
|
|
1837
|
+
* false, ack the full batch.
|
|
1838
|
+
*/
|
|
1839
|
+
static void do_test_destroy_with_explicit_ack(int destroy_flags,
|
|
1840
|
+
rd_bool_t ack_half) {
|
|
1841
|
+
const char *topic;
|
|
1842
|
+
const char *group = "0179-destroy-explicit-ack";
|
|
1843
|
+
rd_kafka_share_t *rkshare;
|
|
1844
|
+
rd_kafka_error_t *error;
|
|
1845
|
+
rd_kafka_messages_t *first_batch = NULL;
|
|
1846
|
+
rd_kafka_messages_t *second_batch = NULL;
|
|
1847
|
+
size_t rcvd = 0;
|
|
1848
|
+
size_t first_batch_cnt = 0;
|
|
1849
|
+
int32_t first_batch_part = -1;
|
|
1850
|
+
int32_t second_batch_part = -1;
|
|
1851
|
+
int attempts = 0;
|
|
1852
|
+
int ack_cnt;
|
|
1853
|
+
expected_ack_t expected[2];
|
|
1854
|
+
int second_part_cnt;
|
|
1855
|
+
ack_receipts_t receipts;
|
|
1856
|
+
|
|
1857
|
+
ack_receipts_init(&receipts);
|
|
1858
|
+
|
|
1859
|
+
SUB_TEST("destroy_flags=0x%x ack_half=%s", destroy_flags,
|
|
1860
|
+
ack_half ? "true" : "false");
|
|
1861
|
+
|
|
1862
|
+
topic = test_mk_topic_name("0179-destroy-explicit-ack", 1);
|
|
1863
|
+
TEST_SAY("Creating 2-partition topic %s\n", topic);
|
|
1864
|
+
test_create_topic_wait_exists(common_admin, topic, 2, -1, 60 * 1000);
|
|
1865
|
+
|
|
1866
|
+
TEST_SAY("Producing %d messages to each partition of topic %s\n",
|
|
1867
|
+
TEST_MSGS / 2, topic);
|
|
1868
|
+
test_produce_msgs_simple(common_producer, topic, 0, TEST_MSGS / 2);
|
|
1869
|
+
test_produce_msgs_simple(common_producer, topic, 1, TEST_MSGS / 2);
|
|
1870
|
+
|
|
1871
|
+
test_share_set_auto_offset_reset(group, "earliest");
|
|
1872
|
+
|
|
1873
|
+
TEST_SAY("Creating share consumer (explicit ack mode) with ack_cb\n");
|
|
1874
|
+
rkshare =
|
|
1875
|
+
new_share_consumer_for_real_test(group, "explicit", &receipts);
|
|
1876
|
+
subscribe_topics(rkshare, &topic, 1);
|
|
1877
|
+
|
|
1878
|
+
/* Consume + ack loop. Explicit ack mode requires acking the
|
|
1879
|
+
* prior batch before the next consume_batch (otherwise it
|
|
1880
|
+
* errors), so we ack inside the loop.
|
|
1881
|
+
*
|
|
1882
|
+
* 2-partition setup with TEST_MSGS/2 per partition: each
|
|
1883
|
+
* non-empty consume_batch returns one partition's full
|
|
1884
|
+
* payload (TEST_MSGS/2 records). The 1st non-empty batch is
|
|
1885
|
+
* always acked in full; we break out once we've consumed all
|
|
1886
|
+
* TEST_MSGS records (i.e., after the 2nd batch arrives —
|
|
1887
|
+
* which we then ack outside the loop based on ack_half). */
|
|
1888
|
+
TEST_SAY("Consuming %d messages (up to %d attempts)\n", TEST_MSGS,
|
|
1889
|
+
MAX_CONSUME_ATTEMPTS);
|
|
1890
|
+
while (rcvd < TEST_MSGS && attempts < MAX_CONSUME_ATTEMPTS) {
|
|
1891
|
+
rd_kafka_messages_t *batch = NULL;
|
|
1892
|
+
size_t batch_rcvd;
|
|
1893
|
+
error = rd_kafka_share_poll(rkshare, 3000, &batch);
|
|
1894
|
+
|
|
1895
|
+
if (error) {
|
|
1896
|
+
TEST_SAY("Attempt %d: consume error: %s\n", attempts,
|
|
1897
|
+
rd_kafka_error_string(error));
|
|
1898
|
+
rd_kafka_error_destroy(error);
|
|
1899
|
+
rd_kafka_messages_destroy(batch);
|
|
1900
|
+
attempts++;
|
|
1901
|
+
continue;
|
|
1902
|
+
}
|
|
1903
|
+
|
|
1904
|
+
batch_rcvd = rd_kafka_messages_count(batch);
|
|
1905
|
+
if (batch_rcvd > 0) {
|
|
1906
|
+
TEST_SAY("Attempt %d: consumed %d messages\n", attempts,
|
|
1907
|
+
(int)batch_rcvd);
|
|
1908
|
+
if (first_batch == NULL) {
|
|
1909
|
+
size_t k;
|
|
1910
|
+
/* First non-empty batch: ack ALL. The
|
|
1911
|
+
* next consume_batch will piggyback
|
|
1912
|
+
* this ack on a ShareFetch. */
|
|
1913
|
+
first_batch = batch;
|
|
1914
|
+
first_batch_cnt = batch_rcvd;
|
|
1915
|
+
for (k = 0; k < batch_rcvd; k++) {
|
|
1916
|
+
rd_kafka_message_t *rkm =
|
|
1917
|
+
rd_kafka_messages_get(batch, k);
|
|
1918
|
+
if (rkm)
|
|
1919
|
+
rd_kafka_share_acknowledge(
|
|
1920
|
+
rkshare, rkm);
|
|
1921
|
+
}
|
|
1922
|
+
} else {
|
|
1923
|
+
/* Second non-empty batch — keep it for
|
|
1924
|
+
* out-of-loop acks below. */
|
|
1925
|
+
rd_kafka_messages_destroy(second_batch);
|
|
1926
|
+
second_batch = batch;
|
|
1927
|
+
}
|
|
1928
|
+
rcvd += batch_rcvd;
|
|
1929
|
+
} else {
|
|
1930
|
+
rd_kafka_messages_destroy(batch);
|
|
1931
|
+
}
|
|
1932
|
+
attempts++;
|
|
1933
|
+
}
|
|
1934
|
+
|
|
1935
|
+
TEST_ASSERT(rcvd == TEST_MSGS,
|
|
1936
|
+
"Expected %d messages after %d attempts, got %d", TEST_MSGS,
|
|
1937
|
+
MAX_CONSUME_ATTEMPTS, (int)rcvd);
|
|
1938
|
+
TEST_ASSERT(first_batch_cnt == TEST_MSGS / 2,
|
|
1939
|
+
"Expected first batch == TEST_MSGS/2 (%d), got %d",
|
|
1940
|
+
TEST_MSGS / 2, (int)first_batch_cnt);
|
|
1941
|
+
TEST_ASSERT(second_batch != NULL, "Expected a non-NULL second batch");
|
|
1942
|
+
|
|
1943
|
+
drain_ack_callbacks(rkshare);
|
|
1944
|
+
{
|
|
1945
|
+
rd_kafka_message_t *first_rkm =
|
|
1946
|
+
rd_kafka_messages_get(first_batch, 0);
|
|
1947
|
+
TEST_ASSERT(first_rkm != NULL,
|
|
1948
|
+
"Expected non-NULL first batch message");
|
|
1949
|
+
first_batch_part = first_rkm->partition;
|
|
1950
|
+
}
|
|
1951
|
+
second_batch_part = (first_batch_part == 0) ? 1 : 0;
|
|
1952
|
+
TEST_SAY("Consumed %d messages (first batch on partition %" PRId32
|
|
1953
|
+
", second batch on partition %" PRId32 ")\n",
|
|
1954
|
+
(int)rcvd, first_batch_part, second_batch_part);
|
|
1955
|
+
|
|
1956
|
+
/* Ack the second batch:
|
|
1957
|
+
* ack_half=false: ack all TEST_MSGS/2 records.
|
|
1958
|
+
* ack_half=true: ack TEST_MSGS/4 records (the first half).
|
|
1959
|
+
* These acks won't be piggyback-flushed close runs — which is
|
|
1960
|
+
* exactly what destroy_share_consumer does (with destroy_flags=0).
|
|
1961
|
+
* With destroy_flags=0x8 these acks are dropped. */
|
|
1962
|
+
{
|
|
1963
|
+
int second_to_ack =
|
|
1964
|
+
ack_half ? (TEST_MSGS / 4) : (TEST_MSGS / 2);
|
|
1965
|
+
int k;
|
|
1966
|
+
for (k = 0; k < second_to_ack; k++) {
|
|
1967
|
+
rd_kafka_message_t *rkm =
|
|
1968
|
+
rd_kafka_messages_get(second_batch, k);
|
|
1969
|
+
if (rkm)
|
|
1970
|
+
rd_kafka_share_acknowledge(rkshare, rkm);
|
|
1971
|
+
}
|
|
1972
|
+
ack_cnt = (int)first_batch_cnt + second_to_ack;
|
|
1973
|
+
TEST_SAY(
|
|
1974
|
+
"Acked first batch (%d) + second batch (%d) = %d / "
|
|
1975
|
+
"%d total\n",
|
|
1976
|
+
(int)first_batch_cnt, second_to_ack, ack_cnt, (int)rcvd);
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1979
|
+
/* Destroy all consumed messages (acked and unacked alike). */
|
|
1980
|
+
rd_kafka_messages_destroy(first_batch);
|
|
1981
|
+
first_batch = NULL;
|
|
1982
|
+
rd_kafka_messages_destroy(second_batch);
|
|
1983
|
+
second_batch = NULL;
|
|
1984
|
+
|
|
1985
|
+
destroy_share_consumer(rkshare, destroy_flags);
|
|
1986
|
+
|
|
1987
|
+
/* Assert ack callback receipts.
|
|
1988
|
+
*
|
|
1989
|
+
* Acks fired:
|
|
1990
|
+
* - First batch: always all TEST_MSGS/2 records on
|
|
1991
|
+
* first_batch_part (acked inside the loop, piggybacked on
|
|
1992
|
+
* the next ShareFetch — delivered before destroy).
|
|
1993
|
+
* - Second batch: TEST_MSGS/2 (if !ack_half) or TEST_MSGS/4
|
|
1994
|
+
* (if ack_half) records on second_batch_part. These acks
|
|
1995
|
+
* are queued internally and only flushed during close.
|
|
1996
|
+
*
|
|
1997
|
+
* destroy_flags=0 (full close): close flushes the second
|
|
1998
|
+
* batch's acks. Both partitions appear.
|
|
1999
|
+
*
|
|
2000
|
+
* destroy_flags=0x8 (NO_CONSUMER_CLOSE): destroy short-
|
|
2001
|
+
* circuits, so second-batch acks are dropped. Only
|
|
2002
|
+
* first_batch_part appears with TEST_MSGS/2. */
|
|
2003
|
+
expected[0] = (expected_ack_t) {
|
|
2004
|
+
topic, first_batch_part, RD_KAFKA_RESP_ERR_NO_ERROR, TEST_MSGS / 2};
|
|
2005
|
+
if (destroy_flags & RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE) {
|
|
2006
|
+
verify_ack_receipts(&receipts, expected, 1,
|
|
2007
|
+
"explicit-ack NO_CONSUMER_CLOSE");
|
|
2008
|
+
} else {
|
|
2009
|
+
second_part_cnt = ack_half ? (TEST_MSGS / 4) : (TEST_MSGS / 2);
|
|
2010
|
+
expected[1] = (expected_ack_t) {topic, second_batch_part,
|
|
2011
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
2012
|
+
second_part_cnt};
|
|
2013
|
+
verify_ack_receipts(&receipts, expected, 2,
|
|
2014
|
+
"explicit-ack full close");
|
|
2015
|
+
}
|
|
2016
|
+
|
|
2017
|
+
ack_receipts_destroy(&receipts);
|
|
2018
|
+
|
|
2019
|
+
SUB_TEST_PASS();
|
|
2020
|
+
}
|
|
2021
|
+
|
|
2022
|
+
|
|
2023
|
+
/**
|
|
2024
|
+
* @brief Test destroying share consumer with implicit-ack mode.
|
|
2025
|
+
*
|
|
2026
|
+
* Consumes a batch of messages in implicit-ack mode but never makes a
|
|
2027
|
+
* follow-up poll (which is what would trigger the implicit ack), then
|
|
2028
|
+
* destroys the consumer directly. Verifies that destroy handles
|
|
2029
|
+
* un-acked records gracefully.
|
|
2030
|
+
*
|
|
2031
|
+
* @param destroy_flags Destroy flags (0 or
|
|
2032
|
+
* RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE).
|
|
2033
|
+
*/
|
|
2034
|
+
static void do_test_destroy_with_implicit_ack(int destroy_flags) {
|
|
2035
|
+
const char *topic;
|
|
2036
|
+
const char *group = "0179-destroy-implicit-ack";
|
|
2037
|
+
rd_kafka_share_t *rkshare;
|
|
2038
|
+
rd_kafka_error_t *error;
|
|
2039
|
+
rd_kafka_messages_t *first_batch = NULL;
|
|
2040
|
+
rd_kafka_messages_t *second_batch = NULL;
|
|
2041
|
+
size_t rcvd = 0;
|
|
2042
|
+
size_t first_batch_cnt = 0;
|
|
2043
|
+
int32_t first_batch_part = -1;
|
|
2044
|
+
int32_t second_batch_part = -1;
|
|
2045
|
+
int attempts = 0;
|
|
2046
|
+
expected_ack_t expected[1];
|
|
2047
|
+
ack_receipts_t receipts;
|
|
2048
|
+
|
|
2049
|
+
ack_receipts_init(&receipts);
|
|
2050
|
+
|
|
2051
|
+
SUB_TEST("destroy_flags=0x%x", destroy_flags);
|
|
2052
|
+
|
|
2053
|
+
topic = test_mk_topic_name("0179-destroy-implicit-ack", 1);
|
|
2054
|
+
TEST_SAY("Creating 2-partition topic %s\n", topic);
|
|
2055
|
+
test_create_topic_wait_exists(common_admin, topic, 2, -1, 60 * 1000);
|
|
2056
|
+
|
|
2057
|
+
/* Explicit produce: TEST_MSGS/2 to each partition. */
|
|
2058
|
+
TEST_SAY("Producing %d messages to each partition of topic %s\n",
|
|
2059
|
+
TEST_MSGS / 2, topic);
|
|
2060
|
+
test_produce_msgs_simple(common_producer, topic, 0, TEST_MSGS / 2);
|
|
2061
|
+
test_produce_msgs_simple(common_producer, topic, 1, TEST_MSGS / 2);
|
|
2062
|
+
|
|
2063
|
+
test_share_set_auto_offset_reset(group, "earliest");
|
|
2064
|
+
|
|
2065
|
+
TEST_SAY("Creating share consumer (implicit ack mode) with ack_cb\n");
|
|
2066
|
+
rkshare =
|
|
2067
|
+
new_share_consumer_for_real_test(group, "implicit", &receipts);
|
|
2068
|
+
subscribe_topics(rkshare, &topic, 1);
|
|
2069
|
+
|
|
2070
|
+
/* Consume loop. Implicit ack mode: records are auto-acked by
|
|
2071
|
+
* the consumer when the next consume_batch's ShareFetch
|
|
2072
|
+
* piggybacks them. The 2-partition setup with TEST_MSGS/2 per
|
|
2073
|
+
* partition means each non-empty consume_batch returns one
|
|
2074
|
+
* partition's full payload. After the 2nd consume_batch
|
|
2075
|
+
* returns, we've consumed all TEST_MSGS records and the 1st
|
|
2076
|
+
* batch's records' implicit-acks have already been
|
|
2077
|
+
* piggybacked. The 2nd batch's records are still ACQUIRED. */
|
|
2078
|
+
TEST_SAY("Consuming %d messages (up to %d attempts)\n", TEST_MSGS,
|
|
2079
|
+
MAX_CONSUME_ATTEMPTS);
|
|
2080
|
+
while (rcvd < TEST_MSGS && attempts < MAX_CONSUME_ATTEMPTS) {
|
|
2081
|
+
rd_kafka_messages_t *batch = NULL;
|
|
2082
|
+
size_t batch_rcvd;
|
|
2083
|
+
error = rd_kafka_share_poll(rkshare, 3000, &batch);
|
|
2084
|
+
|
|
2085
|
+
if (error) {
|
|
2086
|
+
TEST_SAY("Attempt %d: consume error: %s\n", attempts,
|
|
2087
|
+
rd_kafka_error_string(error));
|
|
2088
|
+
rd_kafka_error_destroy(error);
|
|
2089
|
+
rd_kafka_messages_destroy(batch);
|
|
2090
|
+
attempts++;
|
|
2091
|
+
continue;
|
|
2092
|
+
}
|
|
2093
|
+
|
|
2094
|
+
batch_rcvd = rd_kafka_messages_count(batch);
|
|
2095
|
+
if (batch_rcvd > 0) {
|
|
2096
|
+
TEST_SAY("Attempt %d: consumed %d messages\n", attempts,
|
|
2097
|
+
(int)batch_rcvd);
|
|
2098
|
+
if (first_batch == NULL) {
|
|
2099
|
+
first_batch = batch;
|
|
2100
|
+
first_batch_cnt = batch_rcvd;
|
|
2101
|
+
} else {
|
|
2102
|
+
rd_kafka_messages_destroy(second_batch);
|
|
2103
|
+
second_batch = batch;
|
|
2104
|
+
}
|
|
2105
|
+
rcvd += batch_rcvd;
|
|
2106
|
+
} else {
|
|
2107
|
+
rd_kafka_messages_destroy(batch);
|
|
2108
|
+
}
|
|
2109
|
+
attempts++;
|
|
2110
|
+
}
|
|
2111
|
+
|
|
2112
|
+
TEST_ASSERT(rcvd == TEST_MSGS,
|
|
2113
|
+
"Expected %d messages after %d attempts, got %d", TEST_MSGS,
|
|
2114
|
+
MAX_CONSUME_ATTEMPTS, (int)rcvd);
|
|
2115
|
+
TEST_ASSERT(first_batch_cnt == TEST_MSGS / 2,
|
|
2116
|
+
"Expected first batch == TEST_MSGS/2 (%d), got %d",
|
|
2117
|
+
TEST_MSGS / 2, (int)first_batch_cnt);
|
|
2118
|
+
{
|
|
2119
|
+
rd_kafka_message_t *first_rkm =
|
|
2120
|
+
rd_kafka_messages_get(first_batch, 0);
|
|
2121
|
+
TEST_ASSERT(first_rkm != NULL,
|
|
2122
|
+
"Expected non-NULL first batch message");
|
|
2123
|
+
first_batch_part = first_rkm->partition;
|
|
2124
|
+
}
|
|
2125
|
+
second_batch_part = (first_batch_part == 0) ? 1 : 0;
|
|
2126
|
+
TEST_SAY("Consumed %d messages (first batch on partition %" PRId32
|
|
2127
|
+
", second batch on partition %" PRId32 ")\n",
|
|
2128
|
+
(int)rcvd, first_batch_part, second_batch_part);
|
|
2129
|
+
|
|
2130
|
+
/* Drain pending ack callbacks: the first batch's piggybacked
|
|
2131
|
+
* acks may still be in flight. */
|
|
2132
|
+
drain_ack_callbacks(rkshare);
|
|
2133
|
+
|
|
2134
|
+
/* Destroy all consumed messages before share_destroy. */
|
|
2135
|
+
rd_kafka_messages_destroy(first_batch);
|
|
2136
|
+
first_batch = NULL;
|
|
2137
|
+
rd_kafka_messages_destroy(second_batch);
|
|
2138
|
+
second_batch = NULL;
|
|
2139
|
+
|
|
2140
|
+
destroy_share_consumer(rkshare, destroy_flags);
|
|
2141
|
+
|
|
2142
|
+
/* Assert ack callback receipts.
|
|
2143
|
+
*
|
|
2144
|
+
* In implicit-ack mode, the 2nd batch's records stay ACQUIRED
|
|
2145
|
+
* at close/destroy time and are NOT shipped by either close
|
|
2146
|
+
* path (full or NO_CONSUMER_CLOSE) — so they never surface
|
|
2147
|
+
* in the callback. Only the 1st batch's piggybacked acks
|
|
2148
|
+
* (flushed via the 2nd consume_batch's ShareFetch) reach the
|
|
2149
|
+
* callback. Expectation is identical for both destroy_flags
|
|
2150
|
+
* variants: TEST_MSGS/2 NO_ERROR on first_batch_part only. */
|
|
2151
|
+
(void)second_batch_part;
|
|
2152
|
+
expected[0] = (expected_ack_t) {
|
|
2153
|
+
topic, first_batch_part, RD_KAFKA_RESP_ERR_NO_ERROR, TEST_MSGS / 2};
|
|
2154
|
+
verify_ack_receipts(
|
|
2155
|
+
&receipts, expected, 1,
|
|
2156
|
+
(destroy_flags & RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE)
|
|
2157
|
+
? "implicit-ack NO_CONSUMER_CLOSE"
|
|
2158
|
+
: "implicit-ack full close");
|
|
2159
|
+
|
|
2160
|
+
ack_receipts_destroy(&receipts);
|
|
2161
|
+
|
|
2162
|
+
SUB_TEST_PASS();
|
|
2163
|
+
}
|
|
2164
|
+
|
|
2165
|
+
|
|
2166
|
+
/**
|
|
2167
|
+
* @brief Test destroying share consumer after subscribe/unsubscribe.
|
|
2168
|
+
*
|
|
2169
|
+
* This test creates a share consumer, optionally subscribes to topics,
|
|
2170
|
+
* optionally unsubscribes, then destroys it without consuming any messages.
|
|
2171
|
+
* Tests various combinations similar to 0116-kafkaconsumer_close.
|
|
2172
|
+
*
|
|
2173
|
+
* @param do_subscribe Whether to subscribe to topics
|
|
2174
|
+
* @param do_unsubscribe Whether to unsubscribe before destroy
|
|
2175
|
+
* @param destroy_flags Destroy flags (0 or
|
|
2176
|
+
* RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE)
|
|
2177
|
+
*/
|
|
2178
|
+
static void do_test_destroy_with_subscribe_unsubscribe(int do_subscribe,
|
|
2179
|
+
int do_unsubscribe,
|
|
2180
|
+
int destroy_flags) {
|
|
2181
|
+
rd_kafka_share_t *consumer;
|
|
2182
|
+
rd_kafka_topic_partition_list_t *topics;
|
|
2183
|
+
const char *topic = "0179-test-destroy-sub-unsub";
|
|
2184
|
+
rd_kafka_resp_err_t err;
|
|
2185
|
+
|
|
2186
|
+
SUB_TEST("subscribe=%d, unsubscribe=%d, destroy_flags=0x%x",
|
|
2187
|
+
do_subscribe, do_unsubscribe, destroy_flags);
|
|
2188
|
+
|
|
2189
|
+
TEST_SAY("Creating share consumer\n");
|
|
2190
|
+
consumer = test_create_share_consumer("0179-sub-unsub-destroy-test",
|
|
2191
|
+
"explicit");
|
|
2192
|
+
|
|
2193
|
+
if (do_subscribe) {
|
|
2194
|
+
TEST_SAY("Subscribing to topic: %s\n", topic);
|
|
2195
|
+
topics = rd_kafka_topic_partition_list_new(1);
|
|
2196
|
+
rd_kafka_topic_partition_list_add(topics, topic,
|
|
2197
|
+
RD_KAFKA_PARTITION_UA);
|
|
2198
|
+
err = rd_kafka_share_subscribe(consumer, topics);
|
|
2199
|
+
TEST_ASSERT(!err, "Subscribe failed: %s",
|
|
2200
|
+
rd_kafka_err2str(err));
|
|
2201
|
+
rd_kafka_topic_partition_list_destroy(topics);
|
|
2202
|
+
}
|
|
2203
|
+
|
|
2204
|
+
if (do_unsubscribe) {
|
|
2205
|
+
TEST_SAY("Unsubscribing from all topics\n");
|
|
2206
|
+
err = rd_kafka_share_unsubscribe(consumer);
|
|
2207
|
+
TEST_ASSERT(!err, "Unsubscribe failed: %s",
|
|
2208
|
+
rd_kafka_err2str(err));
|
|
2209
|
+
}
|
|
2210
|
+
|
|
2211
|
+
destroy_share_consumer(consumer, destroy_flags);
|
|
2212
|
+
SUB_TEST_PASS();
|
|
2213
|
+
}
|
|
2214
|
+
|
|
2215
|
+
|
|
2216
|
+
/**
|
|
2217
|
+
* @brief Destroy a consumer that drove a mixed-ack chaos workload —
|
|
2218
|
+
* ACCEPT/RELEASE/REJECT round-robin acks, with commit_async every
|
|
2219
|
+
* 5 acks and commit_sync every 10 acks. destroy runs on a
|
|
2220
|
+
* watchdog thread so a hung destroy fails the test loudly instead
|
|
2221
|
+
* of wedging the binary.
|
|
2222
|
+
*/
|
|
2223
|
+
static void do_test_destroy_with_mixed_acks_and_commits(int destroy_flags) {
|
|
2224
|
+
const char *topic;
|
|
2225
|
+
const char *group = "0179-destroy-mixed";
|
|
2226
|
+
rd_kafka_share_t *rkshare;
|
|
2227
|
+
rd_kafka_topic_partition_list_t *partitions = NULL;
|
|
2228
|
+
rd_kafka_messages_t *batch = NULL;
|
|
2229
|
+
rd_kafka_error_t *close_err;
|
|
2230
|
+
thrd_t destroy_thr;
|
|
2231
|
+
destroy_watchdog_arg_t arg;
|
|
2232
|
+
ack_receipts_t receipts;
|
|
2233
|
+
const int produce_cnt = 40;
|
|
2234
|
+
const int target = 30;
|
|
2235
|
+
const int DESTROY_DEADLINE_MS = 15000;
|
|
2236
|
+
rd_ts_t t_start;
|
|
2237
|
+
int64_t t_close_ms = 0;
|
|
2238
|
+
int64_t t_destroy_ms;
|
|
2239
|
+
size_t rcvd;
|
|
2240
|
+
size_t j;
|
|
2241
|
+
int acked = 0;
|
|
2242
|
+
int sync_cnt = 0;
|
|
2243
|
+
int async_cnt = 0;
|
|
2244
|
+
int attempts = 0;
|
|
2245
|
+
int wait_ok;
|
|
2246
|
+
rd_bool_t do_close =
|
|
2247
|
+
!(destroy_flags & RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE);
|
|
2248
|
+
|
|
2249
|
+
SUB_TEST("destroy_flags=0x%x", destroy_flags);
|
|
2250
|
+
|
|
2251
|
+
ack_receipts_init(&receipts);
|
|
2252
|
+
|
|
2253
|
+
topic = test_mk_topic_name("0179-destroy-mixed", 1);
|
|
2254
|
+
test_create_topic_wait_exists(common_admin, topic, 1, -1, 60 * 1000);
|
|
2255
|
+
test_produce_msgs_simple(common_producer, topic, 0, produce_cnt);
|
|
2256
|
+
|
|
2257
|
+
test_share_set_auto_offset_reset(group, "earliest");
|
|
2258
|
+
rkshare =
|
|
2259
|
+
new_share_consumer_for_real_test(group, "explicit", &receipts);
|
|
2260
|
+
subscribe_topics(rkshare, &topic, 1);
|
|
2261
|
+
|
|
2262
|
+
/* Drive chaos: round-robin ACCEPT/RELEASE/REJECT acks with
|
|
2263
|
+
* interleaved commits. Past `target` we drop each batch's
|
|
2264
|
+
* message handles without acking — close/destroy must cope
|
|
2265
|
+
* with whatever inflight state that leaves the lib in. */
|
|
2266
|
+
while (acked < target && attempts++ < 80) {
|
|
2267
|
+
rd_kafka_error_t *err;
|
|
2268
|
+
rd_kafka_messages_destroy(batch);
|
|
2269
|
+
batch = NULL;
|
|
2270
|
+
err = rd_kafka_share_poll(rkshare, 3000, &batch);
|
|
2271
|
+
if (err) {
|
|
2272
|
+
rd_kafka_error_destroy(err);
|
|
2273
|
+
continue;
|
|
2274
|
+
}
|
|
2275
|
+
rcvd = rd_kafka_messages_count(batch);
|
|
2276
|
+
for (j = 0; j < rcvd; j++) {
|
|
2277
|
+
rd_kafka_share_AcknowledgeType_t at;
|
|
2278
|
+
rd_kafka_resp_err_t aerr;
|
|
2279
|
+
rd_kafka_error_t *cerr;
|
|
2280
|
+
rd_kafka_message_t *rkm =
|
|
2281
|
+
rd_kafka_messages_get(batch, j);
|
|
2282
|
+
|
|
2283
|
+
if (!rkm)
|
|
2284
|
+
continue;
|
|
2285
|
+
|
|
2286
|
+
if (rkm->err)
|
|
2287
|
+
continue;
|
|
2288
|
+
|
|
2289
|
+
if (acked >= target)
|
|
2290
|
+
continue;
|
|
2291
|
+
|
|
2292
|
+
switch (acked % 3) {
|
|
2293
|
+
case 0:
|
|
2294
|
+
at = RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT;
|
|
2295
|
+
break;
|
|
2296
|
+
case 1:
|
|
2297
|
+
at = RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE;
|
|
2298
|
+
break;
|
|
2299
|
+
default:
|
|
2300
|
+
at = RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT;
|
|
2301
|
+
break;
|
|
2302
|
+
}
|
|
2303
|
+
|
|
2304
|
+
aerr =
|
|
2305
|
+
rd_kafka_share_acknowledge_type(rkshare, rkm, at);
|
|
2306
|
+
TEST_ASSERT(!aerr, "acknowledge_type(%d) failed: %s",
|
|
2307
|
+
(int)at, rd_kafka_err2str(aerr));
|
|
2308
|
+
acked++;
|
|
2309
|
+
|
|
2310
|
+
if (acked % 5 == 0) {
|
|
2311
|
+
cerr = rd_kafka_share_commit_async(rkshare);
|
|
2312
|
+
TEST_ASSERT(!cerr, "commit_async: %s",
|
|
2313
|
+
cerr ? rd_kafka_error_string(cerr)
|
|
2314
|
+
: "");
|
|
2315
|
+
async_cnt++;
|
|
2316
|
+
}
|
|
2317
|
+
if (acked % 10 == 0) {
|
|
2318
|
+
cerr = rd_kafka_share_commit_sync(
|
|
2319
|
+
rkshare, 30000, &partitions);
|
|
2320
|
+
TEST_ASSERT(!cerr, "commit_sync: %s",
|
|
2321
|
+
cerr ? rd_kafka_error_string(cerr)
|
|
2322
|
+
: "");
|
|
2323
|
+
RD_IF_FREE(
|
|
2324
|
+
partitions,
|
|
2325
|
+
rd_kafka_topic_partition_list_destroy);
|
|
2326
|
+
partitions = NULL;
|
|
2327
|
+
sync_cnt++;
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
}
|
|
2331
|
+
rd_kafka_messages_destroy(batch);
|
|
2332
|
+
batch = NULL;
|
|
2333
|
+
TEST_ASSERT(acked >= target, "expected to ack %d records, got %d",
|
|
2334
|
+
target, acked);
|
|
2335
|
+
TEST_SAY(
|
|
2336
|
+
"acked=%d sync=%d async=%d ack_cb_invocations=%d. "
|
|
2337
|
+
"Calling close.\n",
|
|
2338
|
+
acked, sync_cnt, async_cnt, receipts.callback_invocations);
|
|
2339
|
+
|
|
2340
|
+
if (do_close) {
|
|
2341
|
+
t_start = test_clock();
|
|
2342
|
+
close_err = rd_kafka_share_consumer_close(rkshare);
|
|
2343
|
+
t_close_ms = (test_clock() - t_start) / 1000;
|
|
2344
|
+
TEST_SAY("close returned in %" PRId64
|
|
2345
|
+
" ms (err=%s) "
|
|
2346
|
+
"ack_cb_invocations=%d\n",
|
|
2347
|
+
t_close_ms,
|
|
2348
|
+
close_err ? rd_kafka_error_string(close_err) : "NULL",
|
|
2349
|
+
receipts.callback_invocations);
|
|
2350
|
+
if (close_err)
|
|
2351
|
+
rd_kafka_error_destroy(close_err);
|
|
2352
|
+
} else {
|
|
2353
|
+
TEST_SAY(
|
|
2354
|
+
"Skipping consumer_close (destroy_flags has "
|
|
2355
|
+
"NO_CONSUMER_CLOSE)\n");
|
|
2356
|
+
}
|
|
2357
|
+
|
|
2358
|
+
/* Run destroy on the watchdog thread; fail if it hangs past
|
|
2359
|
+
* DESTROY_DEADLINE_MS instead of wedging the binary. */
|
|
2360
|
+
arg.rkshare = rkshare;
|
|
2361
|
+
arg.destroy_flags = destroy_flags;
|
|
2362
|
+
rd_atomic32_init(&arg.done, 0);
|
|
2363
|
+
|
|
2364
|
+
TEST_ASSERT(thrd_create(&destroy_thr, destroy_watchdog_thread, &arg) ==
|
|
2365
|
+
thrd_success,
|
|
2366
|
+
"Failed to spawn destroy watchdog thread");
|
|
2367
|
+
|
|
2368
|
+
t_start = test_clock();
|
|
2369
|
+
wait_ok = 1;
|
|
2370
|
+
while (!rd_atomic32_get(&arg.done)) {
|
|
2371
|
+
if ((test_clock() - t_start) / 1000 >= DESTROY_DEADLINE_MS) {
|
|
2372
|
+
wait_ok = 0;
|
|
2373
|
+
break;
|
|
2374
|
+
}
|
|
2375
|
+
rd_usleep(100 * 1000, NULL);
|
|
2376
|
+
}
|
|
2377
|
+
t_destroy_ms = (test_clock() - t_start) / 1000;
|
|
2378
|
+
|
|
2379
|
+
if (!wait_ok) {
|
|
2380
|
+
/* Detach so the OS reaps the still-running thread at
|
|
2381
|
+
* process exit (rkshare permanently leaked). */
|
|
2382
|
+
thrd_detach(destroy_thr);
|
|
2383
|
+
TEST_FAIL(
|
|
2384
|
+
"rd_kafka_share_destroy hung past %d ms after close "
|
|
2385
|
+
"(close took %" PRId64
|
|
2386
|
+
" ms). acked=%d sync=%d async=%d "
|
|
2387
|
+
"ack_cb_invocations=%d (expected >= %d).",
|
|
2388
|
+
DESTROY_DEADLINE_MS, t_close_ms, acked, sync_cnt, async_cnt,
|
|
2389
|
+
receipts.callback_invocations, async_cnt + sync_cnt);
|
|
2390
|
+
}
|
|
2391
|
+
|
|
2392
|
+
thrd_join(destroy_thr, NULL);
|
|
2393
|
+
|
|
2394
|
+
TEST_SAY("destroy returned in %" PRId64 " ms (close was %" PRId64
|
|
2395
|
+
" ms); final ack_cb_invocations=%d\n",
|
|
2396
|
+
t_destroy_ms, t_close_ms, receipts.callback_invocations);
|
|
2397
|
+
|
|
2398
|
+
ack_receipts_destroy(&receipts);
|
|
2399
|
+
|
|
2400
|
+
SUB_TEST_PASS();
|
|
2401
|
+
}
|
|
2402
|
+
|
|
2403
|
+
|
|
2404
|
+
int main_0179_share_consumer_destroy(int argc, char **argv) {
|
|
2405
|
+
/* Real broker tests */
|
|
2406
|
+
test_timeout_set(120);
|
|
2407
|
+
|
|
2408
|
+
common_producer = test_create_producer();
|
|
2409
|
+
common_admin = test_create_producer();
|
|
2410
|
+
|
|
2411
|
+
do_test_destroy_with_subscribe_unsubscribe(0, 0, 0);
|
|
2412
|
+
do_test_destroy_with_subscribe_unsubscribe(
|
|
2413
|
+
1, 0, 0); /* subscribe, no unsubscribe */
|
|
2414
|
+
do_test_destroy_with_subscribe_unsubscribe(
|
|
2415
|
+
1, 1, 0); /* subscribe then unsubscribe */
|
|
2416
|
+
do_test_destroy_with_explicit_ack(0, rd_false /* ack all */);
|
|
2417
|
+
do_test_destroy_with_explicit_ack(0, rd_true /* ack half */);
|
|
2418
|
+
do_test_destroy_with_implicit_ack(0);
|
|
2419
|
+
|
|
2420
|
+
do_test_destroy_with_subscribe_unsubscribe(
|
|
2421
|
+
1, 0, RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE);
|
|
2422
|
+
do_test_destroy_with_subscribe_unsubscribe(
|
|
2423
|
+
1, 1, RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE);
|
|
2424
|
+
do_test_destroy_with_subscribe_unsubscribe(
|
|
2425
|
+
0, 0, RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE);
|
|
2426
|
+
do_test_destroy_with_explicit_ack(RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE,
|
|
2427
|
+
rd_false);
|
|
2428
|
+
do_test_destroy_with_explicit_ack(RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE,
|
|
2429
|
+
rd_true);
|
|
2430
|
+
do_test_destroy_with_implicit_ack(RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE);
|
|
2431
|
+
|
|
2432
|
+
do_test_destroy_with_mixed_acks_and_commits(0);
|
|
2433
|
+
do_test_destroy_with_mixed_acks_and_commits(
|
|
2434
|
+
RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE);
|
|
2435
|
+
|
|
2436
|
+
rd_kafka_destroy(common_admin);
|
|
2437
|
+
rd_kafka_destroy(common_producer);
|
|
2438
|
+
|
|
2439
|
+
return 0;
|
|
2440
|
+
}
|
|
2441
|
+
|
|
2442
|
+
int main_0179_share_consumer_destroy_local(int argc, char **argv) {
|
|
2443
|
+
/* Mock broker tests only (no real broker needed) */
|
|
2444
|
+
TEST_SKIP_MOCK_CLUSTER(0);
|
|
2445
|
+
test_timeout_set(480);
|
|
2446
|
+
|
|
2447
|
+
test_destroy_with_fatal_error(0);
|
|
2448
|
+
test_destroy_with_cached_acks_and_delayed_broker(0);
|
|
2449
|
+
test_broker_decommission_with_commit_sync(0, rd_false);
|
|
2450
|
+
test_broker_decommission_with_commit_sync(0, rd_true);
|
|
2451
|
+
test_broker_decommission_with_consume_batch(0);
|
|
2452
|
+
test_broker_decommission_during_close(0, rd_false);
|
|
2453
|
+
test_broker_decommission_during_close(0, rd_true);
|
|
2454
|
+
test_broker_decommission_with_commit_async(0, rd_false);
|
|
2455
|
+
test_broker_decommission_with_commit_async(0, rd_true);
|
|
2456
|
+
test_destroy_during_rebalance(0);
|
|
2457
|
+
|
|
2458
|
+
test_destroy_with_fatal_error(RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE);
|
|
2459
|
+
test_destroy_with_cached_acks_and_delayed_broker(
|
|
2460
|
+
RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE);
|
|
2461
|
+
test_broker_decommission_with_commit_sync(
|
|
2462
|
+
RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE, rd_false);
|
|
2463
|
+
test_broker_decommission_with_commit_sync(
|
|
2464
|
+
RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE, rd_true);
|
|
2465
|
+
test_broker_decommission_with_consume_batch(
|
|
2466
|
+
RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE);
|
|
2467
|
+
test_broker_decommission_during_close(
|
|
2468
|
+
RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE, rd_false);
|
|
2469
|
+
test_broker_decommission_during_close(
|
|
2470
|
+
RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE, rd_true);
|
|
2471
|
+
test_broker_decommission_with_commit_async(
|
|
2472
|
+
RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE, rd_false);
|
|
2473
|
+
test_broker_decommission_with_commit_async(
|
|
2474
|
+
RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE, rd_true);
|
|
2475
|
+
test_destroy_during_rebalance(RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE);
|
|
2476
|
+
|
|
2477
|
+
test_leader_migration_mid_session_destroy(0);
|
|
2478
|
+
test_leader_migration_mid_session_destroy(
|
|
2479
|
+
RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE);
|
|
2480
|
+
|
|
2481
|
+
return 0;
|
|
2482
|
+
}
|