@ohos-ports/confluentinc-kafka-javascript 1.10.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +457 -0
- package/LICENSE.kafkajs +31 -0
- package/LICENSE.node-rdkafka +25 -0
- package/LICENSE.txt +20 -0
- package/README.md +133 -0
- package/binding.gyp +184 -0
- package/build/Release/confluent-kafka-javascript.node +0 -0
- package/build/deps/librdkafka++.so.1 +0 -0
- package/build/deps/librdkafka.Makefile +6 -0
- package/build/deps/librdkafka.so.1 +0 -0
- package/build/deps/librdkafka.target.mk +55 -0
- package/deps/librdkafka/.clang-format +136 -0
- package/deps/librdkafka/.clang-format-cpp +103 -0
- package/deps/librdkafka/.dir-locals.el +10 -0
- package/deps/librdkafka/.formatignore +35 -0
- package/deps/librdkafka/.gdbmacros +19 -0
- package/deps/librdkafka/CHANGELOG.md +2477 -0
- package/deps/librdkafka/CMakeLists.txt +291 -0
- package/deps/librdkafka/CODE_OF_CONDUCT.md +46 -0
- package/deps/librdkafka/CONFIGURATION.md +212 -0
- package/deps/librdkafka/CONTRIBUTING.md +431 -0
- package/deps/librdkafka/Doxyfile +2375 -0
- package/deps/librdkafka/INTRODUCTION.md +2752 -0
- package/deps/librdkafka/LICENSE +26 -0
- package/deps/librdkafka/LICENSE.cjson +22 -0
- package/deps/librdkafka/LICENSE.crc32c +28 -0
- package/deps/librdkafka/LICENSE.fnv1a +18 -0
- package/deps/librdkafka/LICENSE.hdrhistogram +27 -0
- package/deps/librdkafka/LICENSE.lz4 +26 -0
- package/deps/librdkafka/LICENSE.murmur2 +25 -0
- package/deps/librdkafka/LICENSE.nanopb +22 -0
- package/deps/librdkafka/LICENSE.opentelemetry +203 -0
- package/deps/librdkafka/LICENSE.pycrc +23 -0
- package/deps/librdkafka/LICENSE.queue +31 -0
- package/deps/librdkafka/LICENSE.regexp +5 -0
- package/deps/librdkafka/LICENSE.snappy +36 -0
- package/deps/librdkafka/LICENSE.tinycthread +26 -0
- package/deps/librdkafka/LICENSE.wingetopt +49 -0
- package/deps/librdkafka/LICENSES.txt +625 -0
- package/deps/librdkafka/Makefile +125 -0
- package/deps/librdkafka/README.md +201 -0
- package/deps/librdkafka/README.win32 +26 -0
- package/deps/librdkafka/STATISTICS.md +624 -0
- package/deps/librdkafka/configure +214 -0
- package/deps/librdkafka/configure.self +341 -0
- package/deps/librdkafka/dev-conf.sh +123 -0
- package/deps/librdkafka/lds-gen.py +73 -0
- package/deps/librdkafka/mainpage.doxy +40 -0
- package/deps/librdkafka/mklove/Makefile.base +329 -0
- package/deps/librdkafka/mklove/modules/configure.atomics +144 -0
- package/deps/librdkafka/mklove/modules/configure.base +2484 -0
- package/deps/librdkafka/mklove/modules/configure.builtin +70 -0
- package/deps/librdkafka/mklove/modules/configure.cc +186 -0
- package/deps/librdkafka/mklove/modules/configure.cxx +8 -0
- package/deps/librdkafka/mklove/modules/configure.fileversion +65 -0
- package/deps/librdkafka/mklove/modules/configure.gitversion +29 -0
- package/deps/librdkafka/mklove/modules/configure.good_cflags +18 -0
- package/deps/librdkafka/mklove/modules/configure.host +132 -0
- package/deps/librdkafka/mklove/modules/configure.lib +49 -0
- package/deps/librdkafka/mklove/modules/configure.libcurl +99 -0
- package/deps/librdkafka/mklove/modules/configure.libsasl2 +36 -0
- package/deps/librdkafka/mklove/modules/configure.libssl +147 -0
- package/deps/librdkafka/mklove/modules/configure.libzstd +58 -0
- package/deps/librdkafka/mklove/modules/configure.parseversion +95 -0
- package/deps/librdkafka/mklove/modules/configure.pic +16 -0
- package/deps/librdkafka/mklove/modules/configure.socket +20 -0
- package/deps/librdkafka/mklove/modules/configure.zlib +61 -0
- package/deps/librdkafka/mklove/modules/patches/README.md +8 -0
- package/deps/librdkafka/mklove/modules/patches/libcurl.0000-no-runtime-linking-check.patch +11 -0
- package/deps/librdkafka/mklove/modules/patches/libssl.0000-osx-rand-include-fix-OpenSSL-PR16409.patch +56 -0
- package/deps/librdkafka/service.yml +182 -0
- package/deps/librdkafka/src/CMakeLists.txt +379 -0
- package/deps/librdkafka/src/Makefile +105 -0
- package/deps/librdkafka/src/README.lz4.md +30 -0
- package/deps/librdkafka/src/cJSON.c +3191 -0
- package/deps/librdkafka/src/cJSON.h +306 -0
- package/deps/librdkafka/src/crc32c.c +430 -0
- package/deps/librdkafka/src/crc32c.h +38 -0
- package/deps/librdkafka/src/generate_proto.sh +66 -0
- package/deps/librdkafka/src/librdkafka_cgrp_synch.png +0 -0
- package/deps/librdkafka/src/lz4.c +2727 -0
- package/deps/librdkafka/src/lz4.h +842 -0
- package/deps/librdkafka/src/lz4frame.c +2078 -0
- package/deps/librdkafka/src/lz4frame.h +692 -0
- package/deps/librdkafka/src/lz4frame_static.h +47 -0
- package/deps/librdkafka/src/lz4hc.c +1631 -0
- package/deps/librdkafka/src/lz4hc.h +413 -0
- package/deps/librdkafka/src/nanopb/pb.h +917 -0
- package/deps/librdkafka/src/nanopb/pb_common.c +388 -0
- package/deps/librdkafka/src/nanopb/pb_common.h +49 -0
- package/deps/librdkafka/src/nanopb/pb_decode.c +1727 -0
- package/deps/librdkafka/src/nanopb/pb_decode.h +193 -0
- package/deps/librdkafka/src/nanopb/pb_encode.c +1000 -0
- package/deps/librdkafka/src/nanopb/pb_encode.h +185 -0
- package/deps/librdkafka/src/opentelemetry/common.pb.c +32 -0
- package/deps/librdkafka/src/opentelemetry/common.pb.h +170 -0
- package/deps/librdkafka/src/opentelemetry/metrics.options +2 -0
- package/deps/librdkafka/src/opentelemetry/metrics.pb.c +67 -0
- package/deps/librdkafka/src/opentelemetry/metrics.pb.h +966 -0
- package/deps/librdkafka/src/opentelemetry/resource.pb.c +12 -0
- package/deps/librdkafka/src/opentelemetry/resource.pb.h +58 -0
- package/deps/librdkafka/src/queue.h +850 -0
- package/deps/librdkafka/src/rd.h +592 -0
- package/deps/librdkafka/src/rdaddr.c +255 -0
- package/deps/librdkafka/src/rdaddr.h +202 -0
- package/deps/librdkafka/src/rdatomic.h +262 -0
- package/deps/librdkafka/src/rdavg.h +260 -0
- package/deps/librdkafka/src/rdavl.c +210 -0
- package/deps/librdkafka/src/rdavl.h +250 -0
- package/deps/librdkafka/src/rdbase64.c +200 -0
- package/deps/librdkafka/src/rdbase64.h +43 -0
- package/deps/librdkafka/src/rdbuf.c +1884 -0
- package/deps/librdkafka/src/rdbuf.h +375 -0
- package/deps/librdkafka/src/rdcrc32.c +114 -0
- package/deps/librdkafka/src/rdcrc32.h +170 -0
- package/deps/librdkafka/src/rddl.c +179 -0
- package/deps/librdkafka/src/rddl.h +43 -0
- package/deps/librdkafka/src/rdendian.h +175 -0
- package/deps/librdkafka/src/rdfloat.h +67 -0
- package/deps/librdkafka/src/rdfnv1a.c +113 -0
- package/deps/librdkafka/src/rdfnv1a.h +35 -0
- package/deps/librdkafka/src/rdgz.c +120 -0
- package/deps/librdkafka/src/rdgz.h +46 -0
- package/deps/librdkafka/src/rdhdrhistogram.c +721 -0
- package/deps/librdkafka/src/rdhdrhistogram.h +87 -0
- package/deps/librdkafka/src/rdhttp.c +831 -0
- package/deps/librdkafka/src/rdhttp.h +101 -0
- package/deps/librdkafka/src/rdinterval.h +177 -0
- package/deps/librdkafka/src/rdkafka.c +7287 -0
- package/deps/librdkafka/src/rdkafka.h +11795 -0
- package/deps/librdkafka/src/rdkafka_admin.c +10165 -0
- package/deps/librdkafka/src/rdkafka_admin.h +658 -0
- package/deps/librdkafka/src/rdkafka_assignment.c +1282 -0
- package/deps/librdkafka/src/rdkafka_assignment.h +73 -0
- package/deps/librdkafka/src/rdkafka_assignor.c +1800 -0
- package/deps/librdkafka/src/rdkafka_assignor.h +402 -0
- package/deps/librdkafka/src/rdkafka_aux.c +409 -0
- package/deps/librdkafka/src/rdkafka_aux.h +174 -0
- package/deps/librdkafka/src/rdkafka_background.c +222 -0
- package/deps/librdkafka/src/rdkafka_broker.c +6736 -0
- package/deps/librdkafka/src/rdkafka_broker.h +851 -0
- package/deps/librdkafka/src/rdkafka_buf.c +543 -0
- package/deps/librdkafka/src/rdkafka_buf.h +1525 -0
- package/deps/librdkafka/src/rdkafka_cert.c +576 -0
- package/deps/librdkafka/src/rdkafka_cert.h +62 -0
- package/deps/librdkafka/src/rdkafka_cgrp.c +8310 -0
- package/deps/librdkafka/src/rdkafka_cgrp.h +589 -0
- package/deps/librdkafka/src/rdkafka_conf.c +5177 -0
- package/deps/librdkafka/src/rdkafka_conf.h +746 -0
- package/deps/librdkafka/src/rdkafka_confval.h +97 -0
- package/deps/librdkafka/src/rdkafka_coord.c +623 -0
- package/deps/librdkafka/src/rdkafka_coord.h +132 -0
- package/deps/librdkafka/src/rdkafka_error.c +228 -0
- package/deps/librdkafka/src/rdkafka_error.h +80 -0
- package/deps/librdkafka/src/rdkafka_event.c +502 -0
- package/deps/librdkafka/src/rdkafka_event.h +126 -0
- package/deps/librdkafka/src/rdkafka_feature.c +898 -0
- package/deps/librdkafka/src/rdkafka_feature.h +104 -0
- package/deps/librdkafka/src/rdkafka_fetcher.c +3820 -0
- package/deps/librdkafka/src/rdkafka_fetcher.h +62 -0
- package/deps/librdkafka/src/rdkafka_header.c +220 -0
- package/deps/librdkafka/src/rdkafka_header.h +76 -0
- package/deps/librdkafka/src/rdkafka_idempotence.c +807 -0
- package/deps/librdkafka/src/rdkafka_idempotence.h +144 -0
- package/deps/librdkafka/src/rdkafka_int.h +1505 -0
- package/deps/librdkafka/src/rdkafka_interceptor.c +842 -0
- package/deps/librdkafka/src/rdkafka_interceptor.h +104 -0
- package/deps/librdkafka/src/rdkafka_lz4.c +450 -0
- package/deps/librdkafka/src/rdkafka_lz4.h +49 -0
- package/deps/librdkafka/src/rdkafka_metadata.c +2347 -0
- package/deps/librdkafka/src/rdkafka_metadata.h +350 -0
- package/deps/librdkafka/src/rdkafka_metadata_cache.c +1183 -0
- package/deps/librdkafka/src/rdkafka_mock.c +4514 -0
- package/deps/librdkafka/src/rdkafka_mock.h +867 -0
- package/deps/librdkafka/src/rdkafka_mock_cgrp.c +1877 -0
- package/deps/librdkafka/src/rdkafka_mock_handlers.c +5255 -0
- package/deps/librdkafka/src/rdkafka_mock_int.h +1005 -0
- package/deps/librdkafka/src/rdkafka_mock_sharegrp.c +1207 -0
- package/deps/librdkafka/src/rdkafka_msg.c +2619 -0
- package/deps/librdkafka/src/rdkafka_msg.h +629 -0
- package/deps/librdkafka/src/rdkafka_msgbatch.h +62 -0
- package/deps/librdkafka/src/rdkafka_msgset.h +105 -0
- package/deps/librdkafka/src/rdkafka_msgset_reader.c +2099 -0
- package/deps/librdkafka/src/rdkafka_msgset_writer.c +1474 -0
- package/deps/librdkafka/src/rdkafka_offset.c +1565 -0
- package/deps/librdkafka/src/rdkafka_offset.h +150 -0
- package/deps/librdkafka/src/rdkafka_op.c +1219 -0
- package/deps/librdkafka/src/rdkafka_op.h +1065 -0
- package/deps/librdkafka/src/rdkafka_partition.c +5109 -0
- package/deps/librdkafka/src/rdkafka_partition.h +1215 -0
- package/deps/librdkafka/src/rdkafka_pattern.c +228 -0
- package/deps/librdkafka/src/rdkafka_pattern.h +70 -0
- package/deps/librdkafka/src/rdkafka_plugin.c +213 -0
- package/deps/librdkafka/src/rdkafka_plugin.h +41 -0
- package/deps/librdkafka/src/rdkafka_proto.h +739 -0
- package/deps/librdkafka/src/rdkafka_protocol.h +131 -0
- package/deps/librdkafka/src/rdkafka_queue.c +1351 -0
- package/deps/librdkafka/src/rdkafka_queue.h +1235 -0
- package/deps/librdkafka/src/rdkafka_range_assignor.c +1748 -0
- package/deps/librdkafka/src/rdkafka_request.c +7298 -0
- package/deps/librdkafka/src/rdkafka_request.h +761 -0
- package/deps/librdkafka/src/rdkafka_roundrobin_assignor.c +123 -0
- package/deps/librdkafka/src/rdkafka_sasl.c +589 -0
- package/deps/librdkafka/src/rdkafka_sasl.h +63 -0
- package/deps/librdkafka/src/rdkafka_sasl_cyrus.c +722 -0
- package/deps/librdkafka/src/rdkafka_sasl_int.h +89 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer.c +1833 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer.h +52 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer_oidc.c +1827 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer_oidc.h +52 -0
- package/deps/librdkafka/src/rdkafka_sasl_plain.c +142 -0
- package/deps/librdkafka/src/rdkafka_sasl_scram.c +858 -0
- package/deps/librdkafka/src/rdkafka_sasl_win32.c +550 -0
- package/deps/librdkafka/src/rdkafka_share_acknowledgement.c +1445 -0
- package/deps/librdkafka/src/rdkafka_share_acknowledgement.h +373 -0
- package/deps/librdkafka/src/rdkafka_ssl.c +2235 -0
- package/deps/librdkafka/src/rdkafka_ssl.h +86 -0
- package/deps/librdkafka/src/rdkafka_sticky_assignor.c +4785 -0
- package/deps/librdkafka/src/rdkafka_subscription.c +411 -0
- package/deps/librdkafka/src/rdkafka_telemetry.c +761 -0
- package/deps/librdkafka/src/rdkafka_telemetry.h +52 -0
- package/deps/librdkafka/src/rdkafka_telemetry_decode.c +1307 -0
- package/deps/librdkafka/src/rdkafka_telemetry_decode.h +59 -0
- package/deps/librdkafka/src/rdkafka_telemetry_encode.c +1309 -0
- package/deps/librdkafka/src/rdkafka_telemetry_encode.h +459 -0
- package/deps/librdkafka/src/rdkafka_timer.c +407 -0
- package/deps/librdkafka/src/rdkafka_timer.h +117 -0
- package/deps/librdkafka/src/rdkafka_topic.c +2382 -0
- package/deps/librdkafka/src/rdkafka_topic.h +369 -0
- package/deps/librdkafka/src/rdkafka_transport.c +1309 -0
- package/deps/librdkafka/src/rdkafka_transport.h +99 -0
- package/deps/librdkafka/src/rdkafka_transport_int.h +100 -0
- package/deps/librdkafka/src/rdkafka_txnmgr.c +3256 -0
- package/deps/librdkafka/src/rdkafka_txnmgr.h +171 -0
- package/deps/librdkafka/src/rdkafka_zstd.c +226 -0
- package/deps/librdkafka/src/rdkafka_zstd.h +57 -0
- package/deps/librdkafka/src/rdlist.c +593 -0
- package/deps/librdkafka/src/rdlist.h +444 -0
- package/deps/librdkafka/src/rdlog.c +89 -0
- package/deps/librdkafka/src/rdlog.h +41 -0
- package/deps/librdkafka/src/rdmap.c +508 -0
- package/deps/librdkafka/src/rdmap.h +492 -0
- package/deps/librdkafka/src/rdmurmur2.c +167 -0
- package/deps/librdkafka/src/rdmurmur2.h +35 -0
- package/deps/librdkafka/src/rdports.c +61 -0
- package/deps/librdkafka/src/rdports.h +38 -0
- package/deps/librdkafka/src/rdposix.h +250 -0
- package/deps/librdkafka/src/rdrand.c +151 -0
- package/deps/librdkafka/src/rdrand.h +72 -0
- package/deps/librdkafka/src/rdregex.c +156 -0
- package/deps/librdkafka/src/rdregex.h +43 -0
- package/deps/librdkafka/src/rdsignal.h +57 -0
- package/deps/librdkafka/src/rdstring.c +645 -0
- package/deps/librdkafka/src/rdstring.h +98 -0
- package/deps/librdkafka/src/rdsysqueue.h +404 -0
- package/deps/librdkafka/src/rdtime.h +356 -0
- package/deps/librdkafka/src/rdtypes.h +86 -0
- package/deps/librdkafka/src/rdunittest.c +564 -0
- package/deps/librdkafka/src/rdunittest.h +232 -0
- package/deps/librdkafka/src/rdunittest_acknowledge.c +1177 -0
- package/deps/librdkafka/src/rdunittest_fetcher.c +1211 -0
- package/deps/librdkafka/src/rdunittest_msgset_errors.c +1521 -0
- package/deps/librdkafka/src/rdvarint.c +134 -0
- package/deps/librdkafka/src/rdvarint.h +165 -0
- package/deps/librdkafka/src/rdwin32.h +382 -0
- package/deps/librdkafka/src/rdxxhash.c +1030 -0
- package/deps/librdkafka/src/rdxxhash.h +328 -0
- package/deps/librdkafka/src/regexp.c +1352 -0
- package/deps/librdkafka/src/regexp.h +41 -0
- package/deps/librdkafka/src/snappy.c +1866 -0
- package/deps/librdkafka/src/snappy.h +62 -0
- package/deps/librdkafka/src/snappy_compat.h +138 -0
- package/deps/librdkafka/src/statistics_schema.json +444 -0
- package/deps/librdkafka/src/tinycthread.c +932 -0
- package/deps/librdkafka/src/tinycthread.h +503 -0
- package/deps/librdkafka/src/tinycthread_extra.c +210 -0
- package/deps/librdkafka/src/tinycthread_extra.h +218 -0
- package/deps/librdkafka/src/win32_config.h +58 -0
- package/deps/librdkafka/src-cpp/CMakeLists.txt +90 -0
- package/deps/librdkafka/src-cpp/ConfImpl.cpp +84 -0
- package/deps/librdkafka/src-cpp/ConsumerImpl.cpp +244 -0
- package/deps/librdkafka/src-cpp/HandleImpl.cpp +436 -0
- package/deps/librdkafka/src-cpp/HeadersImpl.cpp +48 -0
- package/deps/librdkafka/src-cpp/KafkaConsumerImpl.cpp +296 -0
- package/deps/librdkafka/src-cpp/Makefile +55 -0
- package/deps/librdkafka/src-cpp/MessageImpl.cpp +38 -0
- package/deps/librdkafka/src-cpp/MetadataImpl.cpp +170 -0
- package/deps/librdkafka/src-cpp/ProducerImpl.cpp +197 -0
- package/deps/librdkafka/src-cpp/QueueImpl.cpp +70 -0
- package/deps/librdkafka/src-cpp/README.md +16 -0
- package/deps/librdkafka/src-cpp/RdKafka.cpp +59 -0
- package/deps/librdkafka/src-cpp/TopicImpl.cpp +124 -0
- package/deps/librdkafka/src-cpp/TopicPartitionImpl.cpp +57 -0
- package/deps/librdkafka/src-cpp/rdkafkacpp.h +3797 -0
- package/deps/librdkafka/src-cpp/rdkafkacpp_int.h +1641 -0
- package/deps/librdkafka/tests/0000-unittests.c +72 -0
- package/deps/librdkafka/tests/0001-multiobj.c +102 -0
- package/deps/librdkafka/tests/0002-unkpart.c +244 -0
- package/deps/librdkafka/tests/0003-msgmaxsize.c +173 -0
- package/deps/librdkafka/tests/0004-conf.c +934 -0
- package/deps/librdkafka/tests/0005-order.c +133 -0
- package/deps/librdkafka/tests/0006-symbols.c +163 -0
- package/deps/librdkafka/tests/0007-autotopic.c +136 -0
- package/deps/librdkafka/tests/0008-reqacks.c +179 -0
- package/deps/librdkafka/tests/0009-mock_cluster.c +272 -0
- package/deps/librdkafka/tests/0011-produce_batch.c +753 -0
- package/deps/librdkafka/tests/0012-produce_consume.c +537 -0
- package/deps/librdkafka/tests/0013-null-msgs.c +473 -0
- package/deps/librdkafka/tests/0014-reconsume-191.c +512 -0
- package/deps/librdkafka/tests/0015-offset_seeks.c +172 -0
- package/deps/librdkafka/tests/0016-client_swname.c +181 -0
- package/deps/librdkafka/tests/0017-compression.c +140 -0
- package/deps/librdkafka/tests/0018-cgrp_term.c +444 -0
- package/deps/librdkafka/tests/0019-list_groups.c +289 -0
- package/deps/librdkafka/tests/0020-destroy_hang.c +162 -0
- package/deps/librdkafka/tests/0021-rkt_destroy.c +72 -0
- package/deps/librdkafka/tests/0022-consume_batch.c +331 -0
- package/deps/librdkafka/tests/0025-timers.c +147 -0
- package/deps/librdkafka/tests/0026-consume_pause.c +547 -0
- package/deps/librdkafka/tests/0028-long_topicnames.c +79 -0
- package/deps/librdkafka/tests/0029-assign_offset.c +202 -0
- package/deps/librdkafka/tests/0030-offset_commit.c +589 -0
- package/deps/librdkafka/tests/0031-get_offsets.c +235 -0
- package/deps/librdkafka/tests/0033-regex_subscribe.c +536 -0
- package/deps/librdkafka/tests/0034-offset_reset.c +398 -0
- package/deps/librdkafka/tests/0035-api_version.c +73 -0
- package/deps/librdkafka/tests/0036-partial_fetch.c +87 -0
- package/deps/librdkafka/tests/0037-destroy_hang_local.c +85 -0
- package/deps/librdkafka/tests/0038-performance.c +121 -0
- package/deps/librdkafka/tests/0039-event.c +284 -0
- package/deps/librdkafka/tests/0040-io_event.c +257 -0
- package/deps/librdkafka/tests/0041-fetch_max_bytes.c +97 -0
- package/deps/librdkafka/tests/0042-many_topics.c +252 -0
- package/deps/librdkafka/tests/0043-no_connection.c +77 -0
- package/deps/librdkafka/tests/0044-partition_cnt.c +94 -0
- package/deps/librdkafka/tests/0045-subscribe_update.c +1010 -0
- package/deps/librdkafka/tests/0046-rkt_cache.c +65 -0
- package/deps/librdkafka/tests/0047-partial_buf_tmout.c +98 -0
- package/deps/librdkafka/tests/0048-partitioner.c +283 -0
- package/deps/librdkafka/tests/0049-consume_conn_close.c +162 -0
- package/deps/librdkafka/tests/0050-subscribe_adds.c +145 -0
- package/deps/librdkafka/tests/0051-assign_adds.c +126 -0
- package/deps/librdkafka/tests/0052-msg_timestamps.c +238 -0
- package/deps/librdkafka/tests/0053-stats_cb.cpp +527 -0
- package/deps/librdkafka/tests/0054-offset_time.cpp +236 -0
- package/deps/librdkafka/tests/0055-producer_latency.c +539 -0
- package/deps/librdkafka/tests/0056-balanced_group_mt.c +315 -0
- package/deps/librdkafka/tests/0057-invalid_topic.cpp +112 -0
- package/deps/librdkafka/tests/0058-log.cpp +123 -0
- package/deps/librdkafka/tests/0059-bsearch.cpp +241 -0
- package/deps/librdkafka/tests/0060-op_prio.cpp +163 -0
- package/deps/librdkafka/tests/0061-consumer_lag.cpp +295 -0
- package/deps/librdkafka/tests/0062-stats_event.c +126 -0
- package/deps/librdkafka/tests/0063-clusterid.cpp +180 -0
- package/deps/librdkafka/tests/0064-interceptors.c +529 -0
- package/deps/librdkafka/tests/0065-yield.cpp +140 -0
- package/deps/librdkafka/tests/0066-plugins.cpp +143 -0
- package/deps/librdkafka/tests/0067-empty_topic.cpp +151 -0
- package/deps/librdkafka/tests/0068-produce_timeout.c +136 -0
- package/deps/librdkafka/tests/0069-consumer_add_parts.c +119 -0
- package/deps/librdkafka/tests/0070-null_empty.cpp +197 -0
- package/deps/librdkafka/tests/0072-headers_ut.c +448 -0
- package/deps/librdkafka/tests/0073-headers.c +381 -0
- package/deps/librdkafka/tests/0074-producev.c +87 -0
- package/deps/librdkafka/tests/0075-retry.c +290 -0
- package/deps/librdkafka/tests/0076-produce_retry.c +452 -0
- package/deps/librdkafka/tests/0077-compaction.c +366 -0
- package/deps/librdkafka/tests/0078-c_from_cpp.cpp +96 -0
- package/deps/librdkafka/tests/0079-fork.c +93 -0
- package/deps/librdkafka/tests/0080-admin_ut.c +3095 -0
- package/deps/librdkafka/tests/0081-admin.c +5633 -0
- package/deps/librdkafka/tests/0082-fetch_max_bytes.cpp +137 -0
- package/deps/librdkafka/tests/0083-cb_event.c +233 -0
- package/deps/librdkafka/tests/0084-destroy_flags.c +208 -0
- package/deps/librdkafka/tests/0085-headers.cpp +392 -0
- package/deps/librdkafka/tests/0086-purge.c +368 -0
- package/deps/librdkafka/tests/0088-produce_metadata_timeout.c +162 -0
- package/deps/librdkafka/tests/0089-max_poll_interval.c +511 -0
- package/deps/librdkafka/tests/0090-idempotence.c +171 -0
- package/deps/librdkafka/tests/0091-max_poll_interval_timeout.c +295 -0
- package/deps/librdkafka/tests/0092-mixed_msgver.c +103 -0
- package/deps/librdkafka/tests/0093-holb.c +200 -0
- package/deps/librdkafka/tests/0094-idempotence_msg_timeout.c +231 -0
- package/deps/librdkafka/tests/0095-all_brokers_down.cpp +122 -0
- package/deps/librdkafka/tests/0097-ssl_verify.cpp +674 -0
- package/deps/librdkafka/tests/0098-consumer-txn.cpp +1218 -0
- package/deps/librdkafka/tests/0099-commit_metadata.c +194 -0
- package/deps/librdkafka/tests/0100-thread_interceptors.cpp +195 -0
- package/deps/librdkafka/tests/0101-fetch-from-follower.cpp +446 -0
- package/deps/librdkafka/tests/0102-static_group_rebalance.c +836 -0
- package/deps/librdkafka/tests/0103-transactions.c +1383 -0
- package/deps/librdkafka/tests/0104-fetch_from_follower_mock.c +625 -0
- package/deps/librdkafka/tests/0105-transactions_mock.c +3930 -0
- package/deps/librdkafka/tests/0106-cgrp_sess_timeout.c +318 -0
- package/deps/librdkafka/tests/0107-topic_recreate.c +259 -0
- package/deps/librdkafka/tests/0109-auto_create_topics.cpp +278 -0
- package/deps/librdkafka/tests/0110-batch_size.cpp +182 -0
- package/deps/librdkafka/tests/0111-delay_create_topics.cpp +127 -0
- package/deps/librdkafka/tests/0112-assign_unknown_part.c +87 -0
- package/deps/librdkafka/tests/0113-cooperative_rebalance.cpp +3473 -0
- package/deps/librdkafka/tests/0114-sticky_partitioning.cpp +176 -0
- package/deps/librdkafka/tests/0115-producer_auth.cpp +182 -0
- package/deps/librdkafka/tests/0116-kafkaconsumer_close.cpp +216 -0
- package/deps/librdkafka/tests/0117-mock_errors.c +331 -0
- package/deps/librdkafka/tests/0118-commit_rebalance.c +154 -0
- package/deps/librdkafka/tests/0119-consumer_auth.cpp +167 -0
- package/deps/librdkafka/tests/0120-asymmetric_subscription.c +185 -0
- package/deps/librdkafka/tests/0121-clusterid.c +115 -0
- package/deps/librdkafka/tests/0122-buffer_cleaning_after_rebalance.c +227 -0
- package/deps/librdkafka/tests/0123-connections_max_idle.c +98 -0
- package/deps/librdkafka/tests/0124-openssl_invalid_engine.c +69 -0
- package/deps/librdkafka/tests/0125-immediate_flush.c +144 -0
- package/deps/librdkafka/tests/0126-oauthbearer_oidc.c +1164 -0
- package/deps/librdkafka/tests/0127-fetch_queue_backoff.cpp +165 -0
- package/deps/librdkafka/tests/0128-sasl_callback_queue.cpp +217 -0
- package/deps/librdkafka/tests/0129-fetch_aborted_msgs.c +79 -0
- package/deps/librdkafka/tests/0130-store_offsets.c +178 -0
- package/deps/librdkafka/tests/0131-connect_timeout.c +81 -0
- package/deps/librdkafka/tests/0132-strategy_ordering.c +179 -0
- package/deps/librdkafka/tests/0133-ssl_keys.c +150 -0
- package/deps/librdkafka/tests/0134-ssl_provider.c +92 -0
- package/deps/librdkafka/tests/0135-sasl_credentials.cpp +279 -0
- package/deps/librdkafka/tests/0136-resolve_cb.c +181 -0
- package/deps/librdkafka/tests/0137-barrier_batch_consume.c +752 -0
- package/deps/librdkafka/tests/0138-admin_mock.c +281 -0
- package/deps/librdkafka/tests/0139-offset_validation_mock.c +950 -0
- package/deps/librdkafka/tests/0140-commit_metadata.cpp +108 -0
- package/deps/librdkafka/tests/0142-reauthentication.c +931 -0
- package/deps/librdkafka/tests/0143-exponential_backoff_mock.c +552 -0
- package/deps/librdkafka/tests/0144-idempotence_mock.c +373 -0
- package/deps/librdkafka/tests/0145-pause_resume_mock.c +119 -0
- package/deps/librdkafka/tests/0146-metadata_mock.c +505 -0
- package/deps/librdkafka/tests/0147-consumer_group_consumer_mock.c +1210 -0
- package/deps/librdkafka/tests/0148-offset_fetch_commit_error_mock.c +569 -0
- package/deps/librdkafka/tests/0149-broker-same-host-port.c +140 -0
- package/deps/librdkafka/tests/0150-telemetry_mock.c +651 -0
- package/deps/librdkafka/tests/0151-purge-brokers.c +566 -0
- package/deps/librdkafka/tests/0152-rebootstrap.c +59 -0
- package/deps/librdkafka/tests/0153-memberid.c +128 -0
- package/deps/librdkafka/tests/0155-share_group_heartbeat_mock.c +2695 -0
- package/deps/librdkafka/tests/0156-share_consumer_fetch_mock.c +1551 -0
- package/deps/librdkafka/tests/0157-share_consumer_ack_mock.c +1788 -0
- package/deps/librdkafka/tests/0158-share_consumer_transactions_mock.c +467 -0
- package/deps/librdkafka/tests/0170-share_consumer_subscription.c +2192 -0
- package/deps/librdkafka/tests/0171-share_consumer_consume.c +2150 -0
- package/deps/librdkafka/tests/0172-share_consumer_acknowledge.c +1919 -0
- package/deps/librdkafka/tests/0173-share_consumer_commit_async.c +2735 -0
- package/deps/librdkafka/tests/0174-share_consumer_concurrency.c +1434 -0
- package/deps/librdkafka/tests/0175-share_consumer_groups.c +574 -0
- package/deps/librdkafka/tests/0176-share_consumer_commit_sync.c +2986 -0
- package/deps/librdkafka/tests/0177-share_consumer_transactions.c +1815 -0
- package/deps/librdkafka/tests/0178-share_consumer_close.c +2102 -0
- package/deps/librdkafka/tests/0179-share_consumer_destroy.c +2482 -0
- package/deps/librdkafka/tests/0180-share_consumer_config.c +1677 -0
- package/deps/librdkafka/tests/0181-share_consumer_topic_delete.c +354 -0
- package/deps/librdkafka/tests/0182-share_consumer_error_handling_mock.c +3358 -0
- package/deps/librdkafka/tests/0183-share_consumer_leader_change_mock.c +1234 -0
- package/deps/librdkafka/tests/0184-share_consumer_topic_recreate.c +1356 -0
- package/deps/librdkafka/tests/0185-share_consumer_max_poll_interval.c +557 -0
- package/deps/librdkafka/tests/0186-share_consumer_fatal_error.c +360 -0
- package/deps/librdkafka/tests/0190-share_consumer_telemetry.c +387 -0
- package/deps/librdkafka/tests/1000-unktopic.c +164 -0
- package/deps/librdkafka/tests/8000-idle.cpp +60 -0
- package/deps/librdkafka/tests/8001-fetch_from_follower_mock_manual.c +113 -0
- package/deps/librdkafka/tests/CMakeLists.txt +193 -0
- package/deps/librdkafka/tests/LibrdkafkaTestApp.py +291 -0
- package/deps/librdkafka/tests/Makefile +182 -0
- package/deps/librdkafka/tests/README.md +514 -0
- package/deps/librdkafka/tests/autotest.sh +33 -0
- package/deps/librdkafka/tests/backtrace.gdb +30 -0
- package/deps/librdkafka/tests/broker_version_tests.py +315 -0
- package/deps/librdkafka/tests/buildbox.sh +17 -0
- package/deps/librdkafka/tests/cleanup-checker-tests.sh +20 -0
- package/deps/librdkafka/tests/cluster_testing.py +203 -0
- package/deps/librdkafka/tests/delete-test-topics.sh +56 -0
- package/deps/librdkafka/tests/fixtures/oauthbearer/jwt_assertion_template.json +10 -0
- package/deps/librdkafka/tests/fixtures/ssl/Makefile +8 -0
- package/deps/librdkafka/tests/fixtures/ssl/README.md +13 -0
- package/deps/librdkafka/tests/fixtures/ssl/client.keystore.intermediate.p12 +0 -0
- package/deps/librdkafka/tests/fixtures/ssl/client.keystore.p12 +0 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.certificate.intermediate.pem +72 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.certificate.pem +50 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.intermediate.key +46 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.key +46 -0
- package/deps/librdkafka/tests/fixtures/ssl/create_keys.sh +168 -0
- package/deps/librdkafka/tests/fuzzers/Makefile +12 -0
- package/deps/librdkafka/tests/fuzzers/README.md +31 -0
- package/deps/librdkafka/tests/fuzzers/fuzz_regex.c +74 -0
- package/deps/librdkafka/tests/fuzzers/helpers.h +90 -0
- package/deps/librdkafka/tests/gen-ssl-certs.sh +165 -0
- package/deps/librdkafka/tests/interactive_broker_version.py +170 -0
- package/deps/librdkafka/tests/interceptor_test/CMakeLists.txt +16 -0
- package/deps/librdkafka/tests/interceptor_test/Makefile +22 -0
- package/deps/librdkafka/tests/interceptor_test/interceptor_test.c +314 -0
- package/deps/librdkafka/tests/interceptor_test/interceptor_test.h +54 -0
- package/deps/librdkafka/tests/java/IncrementalRebalanceCli.java +97 -0
- package/deps/librdkafka/tests/java/Makefile +13 -0
- package/deps/librdkafka/tests/java/Murmur2Cli.java +46 -0
- package/deps/librdkafka/tests/java/README.md +14 -0
- package/deps/librdkafka/tests/java/TransactionProducerCli.java +162 -0
- package/deps/librdkafka/tests/java/run-class.sh +11 -0
- package/deps/librdkafka/tests/librdkafka.suppressions +483 -0
- package/deps/librdkafka/tests/lz4_manual_test.sh +59 -0
- package/deps/librdkafka/tests/multi-broker-version-test.sh +50 -0
- package/deps/librdkafka/tests/parse-refcnt.sh +43 -0
- package/deps/librdkafka/tests/performance_plot.py +115 -0
- package/deps/librdkafka/tests/plugin_test/Makefile +19 -0
- package/deps/librdkafka/tests/plugin_test/plugin_test.c +58 -0
- package/deps/librdkafka/tests/requirements.txt +2 -0
- package/deps/librdkafka/tests/run-all-tests.sh +81 -0
- package/deps/librdkafka/tests/run-consumer-tests.sh +16 -0
- package/deps/librdkafka/tests/run-producer-tests.sh +16 -0
- package/deps/librdkafka/tests/run-test-batches.py +157 -0
- package/deps/librdkafka/tests/run-test.sh +141 -0
- package/deps/librdkafka/tests/rusage.c +249 -0
- package/deps/librdkafka/tests/sasl_test.py +289 -0
- package/deps/librdkafka/tests/scenarios/README.md +6 -0
- package/deps/librdkafka/tests/scenarios/ak23.json +6 -0
- package/deps/librdkafka/tests/scenarios/default.json +5 -0
- package/deps/librdkafka/tests/scenarios/noautocreate.json +5 -0
- package/deps/librdkafka/tests/share_telemetry_verify/share-otel-collector-config.yaml +28 -0
- package/deps/librdkafka/tests/sockem.c +801 -0
- package/deps/librdkafka/tests/sockem.h +85 -0
- package/deps/librdkafka/tests/sockem_ctrl.c +145 -0
- package/deps/librdkafka/tests/sockem_ctrl.h +61 -0
- package/deps/librdkafka/tests/test.c +8376 -0
- package/deps/librdkafka/tests/test.conf.example +27 -0
- package/deps/librdkafka/tests/test.h +1150 -0
- package/deps/librdkafka/tests/testcpp.cpp +131 -0
- package/deps/librdkafka/tests/testcpp.h +388 -0
- package/deps/librdkafka/tests/testshared.h +416 -0
- package/deps/librdkafka/tests/tools/README.md +4 -0
- package/deps/librdkafka/tests/tools/stats/README.md +21 -0
- package/deps/librdkafka/tests/tools/stats/filter.jq +42 -0
- package/deps/librdkafka/tests/tools/stats/graph.py +150 -0
- package/deps/librdkafka/tests/tools/stats/requirements.txt +3 -0
- package/deps/librdkafka/tests/tools/stats/to_csv.py +124 -0
- package/deps/librdkafka/tests/trivup/trivup-0.15.0.tar.gz +0 -0
- package/deps/librdkafka/tests/until-fail.sh +87 -0
- package/deps/librdkafka/tests/xxxx-assign_partition.c +122 -0
- package/deps/librdkafka/tests/xxxx-metadata.cpp +159 -0
- package/deps/librdkafka/vcpkg.json +23 -0
- package/deps/librdkafka/win32/README.md +5 -0
- package/deps/librdkafka/win32/build-package.bat +3 -0
- package/deps/librdkafka/win32/build.bat +19 -0
- package/deps/librdkafka/win32/common.vcxproj +84 -0
- package/deps/librdkafka/win32/interceptor_test/interceptor_test.vcxproj +87 -0
- package/deps/librdkafka/win32/librdkafka.autopkg.template +54 -0
- package/deps/librdkafka/win32/librdkafka.master.testing.targets +13 -0
- package/deps/librdkafka/win32/librdkafka.sln +226 -0
- package/deps/librdkafka/win32/librdkafka.vcxproj +282 -0
- package/deps/librdkafka/win32/librdkafkacpp/librdkafkacpp.vcxproj +104 -0
- package/deps/librdkafka/win32/msbuild.ps1 +58 -0
- package/deps/librdkafka/win32/openssl_engine_example/openssl_engine_example.vcxproj +132 -0
- package/deps/librdkafka/win32/package-zip.ps1 +46 -0
- package/deps/librdkafka/win32/packages/repositories.config +4 -0
- package/deps/librdkafka/win32/push-package.bat +4 -0
- package/deps/librdkafka/win32/rdkafka_complex_consumer_example_cpp/rdkafka_complex_consumer_example_cpp.vcxproj +67 -0
- package/deps/librdkafka/win32/rdkafka_example/rdkafka_example.vcxproj +97 -0
- package/deps/librdkafka/win32/rdkafka_performance/rdkafka_performance.vcxproj +97 -0
- package/deps/librdkafka/win32/setup-msys2.ps1 +47 -0
- package/deps/librdkafka/win32/setup-vcpkg.ps1 +34 -0
- package/deps/librdkafka/win32/tests/test.conf.example +25 -0
- package/deps/librdkafka/win32/tests/tests.vcxproj +278 -0
- package/deps/librdkafka/win32/win_ssl_cert_store/win_ssl_cert_store.vcxproj +132 -0
- package/deps/librdkafka/win32/wingetopt.c +564 -0
- package/deps/librdkafka/win32/wingetopt.h +101 -0
- package/deps/librdkafka/win32/wintime.h +33 -0
- package/deps/librdkafka.gyp +83 -0
- package/deps/windows-install.py +80 -0
- package/index.d.ts +3 -0
- package/lib/admin.js +732 -0
- package/lib/client.js +679 -0
- package/lib/error.js +501 -0
- package/lib/index.js +8 -0
- package/lib/kafka-consumer-stream.js +399 -0
- package/lib/kafka-consumer.js +798 -0
- package/lib/kafkajs/_admin.js +1052 -0
- package/lib/kafkajs/_common.js +887 -0
- package/lib/kafkajs/_consumer.js +2156 -0
- package/lib/kafkajs/_consumer_cache.js +310 -0
- package/lib/kafkajs/_error.js +298 -0
- package/lib/kafkajs/_kafka.js +125 -0
- package/lib/kafkajs/_linked-list.js +220 -0
- package/lib/kafkajs/_producer.js +863 -0
- package/lib/kafkajs/index.js +5 -0
- package/lib/producer/high-level-producer.js +400 -0
- package/lib/producer-stream.js +337 -0
- package/lib/producer.js +405 -0
- package/lib/rdkafka.js +44 -0
- package/lib/tools/ref-counter.js +53 -0
- package/lib/topic-partition.js +91 -0
- package/lib/topic.js +43 -0
- package/lib/util.js +55 -0
- package/librdkafka.js +12 -0
- package/package.json +85 -0
- package/shim/source_location +54 -0
- package/src/admin.cc +1597 -0
- package/src/admin.h +112 -0
- package/src/binding.cc +76 -0
- package/src/binding.h +25 -0
- package/src/callbacks.cc +676 -0
- package/src/callbacks.h +302 -0
- package/src/common.cc +1628 -0
- package/src/common.h +192 -0
- package/src/config.cc +219 -0
- package/src/config.h +61 -0
- package/src/connection.cc +703 -0
- package/src/connection.h +117 -0
- package/src/errors.cc +119 -0
- package/src/errors.h +60 -0
- package/src/kafka-consumer.cc +1539 -0
- package/src/kafka-consumer.h +147 -0
- package/src/producer.cc +917 -0
- package/src/producer.h +124 -0
- package/src/topic.cc +173 -0
- package/src/topic.h +54 -0
- package/src/workers.cc +1714 -0
- package/src/workers.h +690 -0
- package/types/config.d.ts +1269 -0
- package/types/errors.d.ts +373 -0
- package/types/kafkajs.d.ts +568 -0
- package/types/rdkafka.d.ts +571 -0
- package/util/configure.js +32 -0
- package/util/generate-docs.sh +18 -0
- package/util/get-env.js +6 -0
- package/util/test-compile.js +11 -0
- package/util/test-producer-delivery.js +100 -0
|
@@ -0,0 +1,1356 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* librdkafka - Apache Kafka C library
|
|
3
|
+
*
|
|
4
|
+
* Copyright (c) 2026, Confluent Inc.
|
|
5
|
+
* All rights reserved.
|
|
6
|
+
*
|
|
7
|
+
* Redistribution and use in source and binary forms, with or without
|
|
8
|
+
* modification, are permitted provided that the following conditions are met:
|
|
9
|
+
*
|
|
10
|
+
* 1. Redistributions of source code must retain the above copyright notice,
|
|
11
|
+
* this list of conditions and the following disclaimer.
|
|
12
|
+
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
13
|
+
* this list of conditions and the following disclaimer in the documentation
|
|
14
|
+
* and/or other materials provided with the distribution.
|
|
15
|
+
*
|
|
16
|
+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
17
|
+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
18
|
+
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
19
|
+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
|
20
|
+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
21
|
+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
22
|
+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
23
|
+
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
24
|
+
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
25
|
+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
26
|
+
* POSSIBILITY OF SUCH DAMAGE.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
#include "test.h"
|
|
30
|
+
#include "testshared.h"
|
|
31
|
+
#include "rdkafka.h"
|
|
32
|
+
#include "../src/rdkafka_proto.h"
|
|
33
|
+
|
|
34
|
+
#define PARTITION_CNT 3
|
|
35
|
+
#define MSGS_PER_PARTITION 10
|
|
36
|
+
#define MSGS_PER_PHASE (PARTITION_CNT * MSGS_PER_PARTITION)
|
|
37
|
+
#define BATCH_SIZE 1024
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
static rd_kafka_t *common_producer;
|
|
41
|
+
static rd_kafka_t *common_admin;
|
|
42
|
+
|
|
43
|
+
/****************************************************************************
|
|
44
|
+
* Helpers
|
|
45
|
+
****************************************************************************/
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* @brief Produce MSGS_PER_PARTITION framework-encoded messages (NULL
|
|
49
|
+
* payload => librdkafka test helper auto-encodes
|
|
50
|
+
* testid+partition+msgid into payload and key) to each of the
|
|
51
|
+
* first \p partition_cnt partitions of \p topic.
|
|
52
|
+
*
|
|
53
|
+
* Per-partition msgids run [0, MSGS_PER_PARTITION), so msgver_verify_part
|
|
54
|
+
* can later assert exact-range coverage independently per partition.
|
|
55
|
+
*/
|
|
56
|
+
static void
|
|
57
|
+
produce_phase(const char *topic, uint64_t testid, int32_t partition_cnt) {
|
|
58
|
+
int32_t p;
|
|
59
|
+
for (p = 0; p < partition_cnt; p++) {
|
|
60
|
+
test_produce_msgs2(common_producer, topic, testid, p,
|
|
61
|
+
0 /*msg_base*/, MSGS_PER_PARTITION,
|
|
62
|
+
NULL /*payload*/, 0);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* @brief Fetch the broker-assigned topic_id for \p topic via the
|
|
68
|
+
* DescribeTopics admin API.
|
|
69
|
+
*
|
|
70
|
+
* @returns A newly-allocated Uuid that the caller must destroy with
|
|
71
|
+
* rd_kafka_Uuid_destroy().
|
|
72
|
+
*
|
|
73
|
+
* On any failure (topic missing, admin timeout, etc.) the test is failed.
|
|
74
|
+
*/
|
|
75
|
+
static rd_kafka_Uuid_t *fetch_topic_id(const char *topic) {
|
|
76
|
+
rd_kafka_queue_t *q;
|
|
77
|
+
rd_kafka_TopicCollection_t *tc;
|
|
78
|
+
rd_kafka_event_t *rkev;
|
|
79
|
+
const rd_kafka_DescribeTopics_result_t *res;
|
|
80
|
+
const rd_kafka_TopicDescription_t **descs;
|
|
81
|
+
rd_kafka_Uuid_t *topic_id;
|
|
82
|
+
size_t desc_cnt = 0;
|
|
83
|
+
const char *topics_arr[1];
|
|
84
|
+
|
|
85
|
+
topics_arr[0] = topic;
|
|
86
|
+
|
|
87
|
+
q = rd_kafka_queue_new(common_admin);
|
|
88
|
+
tc = rd_kafka_TopicCollection_of_topic_names(topics_arr, 1);
|
|
89
|
+
|
|
90
|
+
rd_kafka_DescribeTopics(common_admin, tc, NULL /*options*/, q);
|
|
91
|
+
|
|
92
|
+
rkev = rd_kafka_queue_poll(q, 30 * 1000);
|
|
93
|
+
TEST_ASSERT(rkev != NULL, "DescribeTopics(%s) timed out", topic);
|
|
94
|
+
TEST_ASSERT(rd_kafka_event_error(rkev) == RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
95
|
+
"DescribeTopics(%s) failed: %s", topic,
|
|
96
|
+
rd_kafka_event_error_string(rkev));
|
|
97
|
+
|
|
98
|
+
res = rd_kafka_event_DescribeTopics_result(rkev);
|
|
99
|
+
descs = rd_kafka_DescribeTopics_result_topics(res, &desc_cnt);
|
|
100
|
+
TEST_ASSERT(desc_cnt == 1, "Expected 1 topic description, got %zu",
|
|
101
|
+
desc_cnt);
|
|
102
|
+
TEST_ASSERT(
|
|
103
|
+
rd_kafka_error_code(rd_kafka_TopicDescription_error(descs[0])) ==
|
|
104
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
105
|
+
"Topic %s description error: %s", topic,
|
|
106
|
+
rd_kafka_error_string(rd_kafka_TopicDescription_error(descs[0])));
|
|
107
|
+
|
|
108
|
+
topic_id =
|
|
109
|
+
rd_kafka_Uuid_copy(rd_kafka_TopicDescription_topic_id(descs[0]));
|
|
110
|
+
|
|
111
|
+
rd_kafka_event_destroy(rkev);
|
|
112
|
+
rd_kafka_TopicCollection_destroy(tc);
|
|
113
|
+
rd_kafka_queue_destroy(q);
|
|
114
|
+
|
|
115
|
+
return topic_id;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* @brief Assert that \p id_before and \p id_after represent two different
|
|
120
|
+
* topic instances; fail the test otherwise.
|
|
121
|
+
*/
|
|
122
|
+
static void assert_topic_id_changed(const rd_kafka_Uuid_t *id_before,
|
|
123
|
+
const rd_kafka_Uuid_t *id_after) {
|
|
124
|
+
TEST_ASSERT(strcmp(rd_kafka_Uuid_base64str(id_before),
|
|
125
|
+
rd_kafka_Uuid_base64str(id_after)) != 0,
|
|
126
|
+
"Recreated topic_id (%s) matches the original; "
|
|
127
|
+
"broker did not actually recreate the topic",
|
|
128
|
+
rd_kafka_Uuid_base64str(id_before));
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* @brief Consume from a share consumer into \p mv until \p exp_cnt messages
|
|
133
|
+
* matching mv->testid have been collected, or \p timeout_ms elapses.
|
|
134
|
+
*
|
|
135
|
+
* Every consumed message is ACCEPT-ack'd. On every message we assert that
|
|
136
|
+
* delivery_count == 1 (no broker-side redelivery occurred for this run).
|
|
137
|
+
*
|
|
138
|
+
* @returns the count of matching messages collected into \p mv.
|
|
139
|
+
*/
|
|
140
|
+
static int consume_into_msgver(rd_kafka_share_t *rkshare,
|
|
141
|
+
test_msgver_t *mv,
|
|
142
|
+
int exp_cnt,
|
|
143
|
+
int timeout_ms) {
|
|
144
|
+
rd_kafka_messages_t *batch = NULL;
|
|
145
|
+
rd_ts_t deadline = test_clock() + (rd_ts_t)timeout_ms * 1000;
|
|
146
|
+
rd_kafka_t *rk = test_share_consumer_get_rk(rkshare);
|
|
147
|
+
|
|
148
|
+
while (mv->msgcnt < exp_cnt && test_clock() < deadline) {
|
|
149
|
+
rd_kafka_error_t *err;
|
|
150
|
+
size_t rcvd = 0;
|
|
151
|
+
size_t i;
|
|
152
|
+
|
|
153
|
+
err = rd_kafka_share_poll(rkshare, 500, &batch);
|
|
154
|
+
if (err) {
|
|
155
|
+
TEST_SAY("share_poll error: %s\n",
|
|
156
|
+
rd_kafka_error_string(err));
|
|
157
|
+
rd_kafka_error_destroy(err);
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
rcvd = rd_kafka_messages_count(batch);
|
|
162
|
+
|
|
163
|
+
for (i = 0; i < rcvd; i++) {
|
|
164
|
+
rd_kafka_message_t *m = rd_kafka_messages_get(batch, i);
|
|
165
|
+
|
|
166
|
+
if (m->err) {
|
|
167
|
+
TEST_SAY(
|
|
168
|
+
"Consumer event: %s "
|
|
169
|
+
"(topic=%s partition=%" PRId32 ")\n",
|
|
170
|
+
rd_kafka_err2str(m->err),
|
|
171
|
+
m->rkt ? rd_kafka_topic_name(m->rkt)
|
|
172
|
+
: "(none)",
|
|
173
|
+
m->partition);
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
TEST_ASSERT(
|
|
178
|
+
rd_kafka_message_delivery_count(m) == 1,
|
|
179
|
+
"Unexpected redelivery: delivery_count=%d on "
|
|
180
|
+
"topic=%s partition=%" PRId32 " offset=%" PRId64,
|
|
181
|
+
rd_kafka_message_delivery_count(m),
|
|
182
|
+
rd_kafka_topic_name(m->rkt), m->partition,
|
|
183
|
+
m->offset);
|
|
184
|
+
|
|
185
|
+
/* msgver auto-filters by mv->testid; messages from
|
|
186
|
+
* the other phase (or untagged) are silently
|
|
187
|
+
* dropped. */
|
|
188
|
+
test_msgver_add_msg(rk, mv, m);
|
|
189
|
+
rd_kafka_share_acknowledge(rkshare, m);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
rd_kafka_messages_destroy(batch);
|
|
193
|
+
batch = NULL;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
return mv->msgcnt;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* @brief Delete a topic, wait for the deletion to settle, then recreate
|
|
201
|
+
* with the same name and \p partition_cnt partitions. The new
|
|
202
|
+
* instance will have a fresh topic_id on the broker.
|
|
203
|
+
*/
|
|
204
|
+
static void recreate_topic(const char *topic, int partition_cnt) {
|
|
205
|
+
TEST_SAY("Deleting topic %s\n", topic);
|
|
206
|
+
test_delete_topic(common_admin, topic);
|
|
207
|
+
|
|
208
|
+
/* DeleteTopics is async on the broker; the controller may still
|
|
209
|
+
* be removing log segments when we issue CreateTopics. Without
|
|
210
|
+
* this pause, the broker frequently lets the CreateTopics request
|
|
211
|
+
* sit until DeleteTopics finishes, which can blow past the
|
|
212
|
+
* admin-op timeout. The same 5s pause pattern is used by
|
|
213
|
+
* 0107-topic_recreate. */
|
|
214
|
+
rd_sleep(5);
|
|
215
|
+
|
|
216
|
+
TEST_SAY("Recreating topic %s with %d partition(s)\n", topic,
|
|
217
|
+
partition_cnt);
|
|
218
|
+
test_create_topic_wait_exists(common_admin, topic, partition_cnt, -1,
|
|
219
|
+
60 * 1000);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* @brief Build a share consumer such that the periodic metadata refresh
|
|
224
|
+
* is the dominant channel for discovering topic_id changes:
|
|
225
|
+
* - topic.metadata.refresh.interval.ms is set to 500ms, well
|
|
226
|
+
* below the broker-dictated ShareGroupHeartbeat interval
|
|
227
|
+
* (typically 5s), so the refresh timer is virtually
|
|
228
|
+
* guaranteed to fire between heartbeats during the
|
|
229
|
+
* delete/recreate window;
|
|
230
|
+
*/
|
|
231
|
+
static rd_kafka_share_t *create_consumer_md_first(const char *group_id) {
|
|
232
|
+
rd_kafka_share_t *rkshare;
|
|
233
|
+
rd_kafka_conf_t *conf;
|
|
234
|
+
char errstr[512];
|
|
235
|
+
|
|
236
|
+
test_conf_init(&conf, NULL, 0);
|
|
237
|
+
|
|
238
|
+
test_conf_set(conf, "group.id", group_id);
|
|
239
|
+
test_conf_set(conf, "share.acknowledgement.mode", "explicit");
|
|
240
|
+
test_conf_set(conf, "topic.metadata.refresh.interval.ms", "500");
|
|
241
|
+
|
|
242
|
+
rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr));
|
|
243
|
+
TEST_ASSERT(rkshare, "Failed to create share consumer: %s", errstr);
|
|
244
|
+
|
|
245
|
+
return rkshare;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* @brief Build a share consumer such that the ShareGroupHeartbeat is
|
|
250
|
+
* the dominant channel for discovering topic_id changes; the
|
|
251
|
+
* default topic.metadata.refresh.interval.ms is 5min, so the
|
|
252
|
+
* periodic MD refresh effectively does not fire during a test.
|
|
253
|
+
*
|
|
254
|
+
* Same signature as create_consumer_md_first so both can be passed to
|
|
255
|
+
* do_test_recreate_two_phase as a builder function pointer.
|
|
256
|
+
*/
|
|
257
|
+
static rd_kafka_share_t *create_consumer_hb_first(const char *group_id) {
|
|
258
|
+
return test_create_share_consumer(group_id, "explicit");
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* @brief Background-thread payload that deletes and recreates a topic
|
|
263
|
+
* on a mock cluster after a short sleep.
|
|
264
|
+
*
|
|
265
|
+
* Used by recreate-during-close mock tests so the recreate lands in the
|
|
266
|
+
* middle of an artificially-delayed broker request on the main thread.
|
|
267
|
+
*/
|
|
268
|
+
struct recreate_thread_args {
|
|
269
|
+
rd_kafka_mock_cluster_t *mcluster;
|
|
270
|
+
const char *topic;
|
|
271
|
+
int32_t partition_cnt;
|
|
272
|
+
int sleep_ms;
|
|
273
|
+
};
|
|
274
|
+
|
|
275
|
+
static int recreate_thread_main(void *p) {
|
|
276
|
+
struct recreate_thread_args *args = p;
|
|
277
|
+
rd_usleep((int64_t)args->sleep_ms * 1000, NULL);
|
|
278
|
+
TEST_SAY("[recreate-thread] Deleting topic %s\n", args->topic);
|
|
279
|
+
TEST_ASSERT(rd_kafka_mock_topic_delete(args->mcluster, args->topic) ==
|
|
280
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
281
|
+
"[recreate-thread] mock_topic_delete failed");
|
|
282
|
+
TEST_SAY("[recreate-thread] Recreating topic %s with %" PRId32
|
|
283
|
+
" partition(s)\n",
|
|
284
|
+
args->topic, args->partition_cnt);
|
|
285
|
+
TEST_ASSERT(rd_kafka_mock_topic_create(args->mcluster, args->topic,
|
|
286
|
+
args->partition_cnt,
|
|
287
|
+
1) == RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
288
|
+
"[recreate-thread] mock_topic_create failed");
|
|
289
|
+
return 0;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* @brief Background producer thread: produces one record every
|
|
294
|
+
* producer_period_ms with payload "<phase>:<seq>", where
|
|
295
|
+
* phase is whatever the controlling thread has most recently
|
|
296
|
+
* written into args->phase. Tolerates produce errors (the
|
|
297
|
+
* topic may not exist mid-test).
|
|
298
|
+
*
|
|
299
|
+
*
|
|
300
|
+
* Used by the survives-delete-and-recreate test to keep produce
|
|
301
|
+
* activity steady through the delete window.
|
|
302
|
+
*/
|
|
303
|
+
struct producer_thread_args {
|
|
304
|
+
rd_kafka_t *producer;
|
|
305
|
+
const char *topic;
|
|
306
|
+
int period_ms;
|
|
307
|
+
char phase[16]; /* "before" / "during" / "after" */
|
|
308
|
+
rd_bool_t stop;
|
|
309
|
+
};
|
|
310
|
+
|
|
311
|
+
static int producer_thread_main(void *p) {
|
|
312
|
+
struct producer_thread_args *args = p;
|
|
313
|
+
int seq = 0;
|
|
314
|
+
while (!args->stop) {
|
|
315
|
+
char payload[64];
|
|
316
|
+
rd_snprintf(payload, sizeof(payload), "%s:%d", args->phase,
|
|
317
|
+
seq++);
|
|
318
|
+
/* Errors during the delete window are expected; the
|
|
319
|
+
* test tolerates them silently. */
|
|
320
|
+
(void)rd_kafka_producev(
|
|
321
|
+
args->producer, RD_KAFKA_V_TOPIC(args->topic),
|
|
322
|
+
RD_KAFKA_V_VALUE(payload, strlen(payload)),
|
|
323
|
+
RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), RD_KAFKA_V_END);
|
|
324
|
+
rd_usleep((int64_t)args->period_ms * 1000, NULL);
|
|
325
|
+
}
|
|
326
|
+
return 0;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/****************************************************************************
|
|
330
|
+
* Test Cases
|
|
331
|
+
****************************************************************************/
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* @brief Generic two-phase recreate test: produce + consume + verify
|
|
335
|
+
* a "before" phase, recreate the topic (possibly with a different
|
|
336
|
+
* partition count), then produce + consume + verify an "after"
|
|
337
|
+
* phase against the new topic instance.
|
|
338
|
+
*
|
|
339
|
+
* 1. Subscribe; produce MSGS_PER_PHASE framework-encoded messages
|
|
340
|
+
* tagged with testid_before; consume them all via a "before" msgver
|
|
341
|
+
* and assert exact range coverage and zero redelivery.
|
|
342
|
+
* 2. Delete + recreate the topic with \p after_partition_cnt partitions.
|
|
343
|
+
* The broker assigns a fresh topic_id. Whichever of (HB, periodic
|
|
344
|
+
* metadata refresh) the \p ctor builder favors will carry the
|
|
345
|
+
* change to the client first.
|
|
346
|
+
* 3. Produce after_partition_cnt * MSGS_PER_PARTITION records tagged
|
|
347
|
+
* with testid_after; consume them all via an "after" msgver and
|
|
348
|
+
* assert exact range coverage, zero redelivery, and (implicitly
|
|
349
|
+
* via testid filtering) zero stale "before" messages.
|
|
350
|
+
*/
|
|
351
|
+
static void do_test_recreate_two_phase(const char *label,
|
|
352
|
+
const char *group_id,
|
|
353
|
+
rd_kafka_share_t *(*ctor)(const char *),
|
|
354
|
+
int32_t after_partition_cnt) {
|
|
355
|
+
const char *topic;
|
|
356
|
+
rd_kafka_share_t *rkshare;
|
|
357
|
+
test_msgver_t mv_before, mv_after;
|
|
358
|
+
uint64_t testid_before, testid_after;
|
|
359
|
+
rd_kafka_Uuid_t *id_before, *id_after;
|
|
360
|
+
int got;
|
|
361
|
+
int32_t p;
|
|
362
|
+
const int msgs_per_phase_after =
|
|
363
|
+
after_partition_cnt * MSGS_PER_PARTITION;
|
|
364
|
+
|
|
365
|
+
SUB_TEST_QUICK("%s", label);
|
|
366
|
+
|
|
367
|
+
testid_before = test_id_generate();
|
|
368
|
+
testid_after = test_id_generate();
|
|
369
|
+
|
|
370
|
+
topic = test_mk_topic_name(label, 1);
|
|
371
|
+
|
|
372
|
+
test_create_topic_wait_exists(common_admin, topic, PARTITION_CNT, -1,
|
|
373
|
+
60 * 1000);
|
|
374
|
+
|
|
375
|
+
id_before = fetch_topic_id(topic);
|
|
376
|
+
TEST_SAY("Initial topic_id: %s\n", rd_kafka_Uuid_base64str(id_before));
|
|
377
|
+
|
|
378
|
+
test_share_set_auto_offset_reset(group_id, "earliest");
|
|
379
|
+
|
|
380
|
+
rkshare = ctor(group_id);
|
|
381
|
+
test_share_consumer_subscribe_multi(rkshare, 1, topic);
|
|
382
|
+
|
|
383
|
+
/* ---- Phase 1: pre-recreate ---- */
|
|
384
|
+
TEST_SAY(
|
|
385
|
+
"Phase 1: producing %d msgs/partition x %d partitions "
|
|
386
|
+
"(testid=%" PRIu64 ")\n",
|
|
387
|
+
MSGS_PER_PARTITION, PARTITION_CNT, testid_before);
|
|
388
|
+
produce_phase(topic, testid_before, PARTITION_CNT);
|
|
389
|
+
|
|
390
|
+
test_msgver_init(&mv_before, testid_before);
|
|
391
|
+
got =
|
|
392
|
+
consume_into_msgver(rkshare, &mv_before, MSGS_PER_PHASE, 30 * 1000);
|
|
393
|
+
TEST_ASSERT(got == MSGS_PER_PHASE,
|
|
394
|
+
"Phase 1: expected %d msgs matching testid_before, "
|
|
395
|
+
"got %d",
|
|
396
|
+
MSGS_PER_PHASE, got);
|
|
397
|
+
for (p = 0; p < PARTITION_CNT; p++)
|
|
398
|
+
test_msgver_verify_part("phase1-before-part", &mv_before,
|
|
399
|
+
TEST_MSGVER_ALL_PART, topic, p,
|
|
400
|
+
0 /*msg_base*/, MSGS_PER_PARTITION);
|
|
401
|
+
test_msgver_clear(&mv_before);
|
|
402
|
+
|
|
403
|
+
/* ---- Recreate ---- */
|
|
404
|
+
recreate_topic(topic, after_partition_cnt);
|
|
405
|
+
|
|
406
|
+
id_after = fetch_topic_id(topic);
|
|
407
|
+
TEST_SAY("Recreated topic_id: %s\n", rd_kafka_Uuid_base64str(id_after));
|
|
408
|
+
assert_topic_id_changed(id_before, id_after);
|
|
409
|
+
|
|
410
|
+
/* ---- Phase 2: post-recreate ---- */
|
|
411
|
+
TEST_SAY("Phase 2: producing %d msgs/partition x %" PRId32
|
|
412
|
+
" partitions on recreated topic (testid=%" PRIu64 ")\n",
|
|
413
|
+
MSGS_PER_PARTITION, after_partition_cnt, testid_after);
|
|
414
|
+
produce_phase(topic, testid_after, after_partition_cnt);
|
|
415
|
+
|
|
416
|
+
test_msgver_init(&mv_after, testid_after);
|
|
417
|
+
got = consume_into_msgver(rkshare, &mv_after, msgs_per_phase_after,
|
|
418
|
+
60 * 1000);
|
|
419
|
+
TEST_ASSERT(got == msgs_per_phase_after,
|
|
420
|
+
"Phase 2: expected %d msgs matching testid_after, "
|
|
421
|
+
"got %d (post-recreate)",
|
|
422
|
+
msgs_per_phase_after, got);
|
|
423
|
+
for (p = 0; p < after_partition_cnt; p++)
|
|
424
|
+
test_msgver_verify_part("phase2-after-part", &mv_after,
|
|
425
|
+
TEST_MSGVER_ALL_PART, topic, p,
|
|
426
|
+
0 /*msg_base*/, MSGS_PER_PARTITION);
|
|
427
|
+
test_msgver_clear(&mv_after);
|
|
428
|
+
|
|
429
|
+
test_share_consumer_close(rkshare);
|
|
430
|
+
test_share_destroy(rkshare);
|
|
431
|
+
rd_kafka_Uuid_destroy(id_before);
|
|
432
|
+
rd_kafka_Uuid_destroy(id_after);
|
|
433
|
+
|
|
434
|
+
SUB_TEST_PASS();
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/**
|
|
438
|
+
* @brief Stress test: many rapid topic recreations interleaved with
|
|
439
|
+
* random produce activity and varying partition counts.
|
|
440
|
+
*
|
|
441
|
+
* Each chaos cycle picks (uniformly at random):
|
|
442
|
+
* - A short pre-recreate sleep in [200, 1500] ms, to let the
|
|
443
|
+
* consumer's state advance unpredictably between cycles.
|
|
444
|
+
* - Whether to produce 0..5 records to each partition of the
|
|
445
|
+
* current topic instance before the recreate; if any are
|
|
446
|
+
* produced, they use a per-cycle testid that the test does not
|
|
447
|
+
* track (we don't require these records to be received).
|
|
448
|
+
* - A new partition count from {2, 3, 4} for the recreated topic,
|
|
449
|
+
* exercising shrink, no-change, and grow paths from one cycle to
|
|
450
|
+
* the next.
|
|
451
|
+
*
|
|
452
|
+
* After chaos_cycles iterations the test produces a final batch of
|
|
453
|
+
* MSGS_PER_PARTITION records to every partition of whatever topic
|
|
454
|
+
* instance is currently live (partition count is whatever chaos left
|
|
455
|
+
* it at) and asserts:
|
|
456
|
+
* - The final batch is fully consumed (per-partition exact-range).
|
|
457
|
+
* - All final-batch records have delivery_count == 1.
|
|
458
|
+
* - The broker-assigned topic_id changed end-to-end.
|
|
459
|
+
*/
|
|
460
|
+
static void do_test_recreate_chaos(void) {
|
|
461
|
+
const int chaos_cycles = 10;
|
|
462
|
+
const int chaos_sleep_min_ms = 200;
|
|
463
|
+
const int chaos_sleep_max_ms = 1500;
|
|
464
|
+
const int chaos_max_records_per_part = 5;
|
|
465
|
+
const int32_t chaos_partition_choices[] = {2, 3, 4};
|
|
466
|
+
const int chaos_partition_choices_cnt =
|
|
467
|
+
sizeof(chaos_partition_choices) /
|
|
468
|
+
sizeof(chaos_partition_choices[0]);
|
|
469
|
+
const char *topic;
|
|
470
|
+
const char *group_id = "0184-share-recreate-chaos";
|
|
471
|
+
rd_kafka_share_t *rkshare;
|
|
472
|
+
test_msgver_t mv_warmup, mv_final;
|
|
473
|
+
uint64_t testid_warmup, testid_final, seed_testid;
|
|
474
|
+
rd_kafka_Uuid_t *id_initial, *id_final;
|
|
475
|
+
int32_t current_partition_cnt = PARTITION_CNT;
|
|
476
|
+
int cycle;
|
|
477
|
+
int got;
|
|
478
|
+
int32_t p;
|
|
479
|
+
int final_msgs;
|
|
480
|
+
|
|
481
|
+
SUB_TEST_QUICK();
|
|
482
|
+
|
|
483
|
+
seed_testid = test_id_generate();
|
|
484
|
+
srand((unsigned)seed_testid);
|
|
485
|
+
TEST_SAY("Chaos seed (from testid): %" PRIu64 "\n", seed_testid);
|
|
486
|
+
|
|
487
|
+
testid_warmup = test_id_generate();
|
|
488
|
+
testid_final = test_id_generate();
|
|
489
|
+
|
|
490
|
+
topic = test_mk_topic_name("0184-recreate-chaos", 1);
|
|
491
|
+
|
|
492
|
+
test_create_topic_wait_exists(common_admin, topic, PARTITION_CNT, -1,
|
|
493
|
+
60 * 1000);
|
|
494
|
+
id_initial = fetch_topic_id(topic);
|
|
495
|
+
TEST_SAY("Initial topic_id: %s\n", rd_kafka_Uuid_base64str(id_initial));
|
|
496
|
+
|
|
497
|
+
test_share_set_auto_offset_reset(group_id, "earliest");
|
|
498
|
+
|
|
499
|
+
/* HB-first regime: long MD refresh so HB drives discovery
|
|
500
|
+
* across recreate cycles. */
|
|
501
|
+
rkshare = test_create_share_consumer(group_id, "explicit");
|
|
502
|
+
|
|
503
|
+
test_share_consumer_subscribe_multi(rkshare, 1, topic);
|
|
504
|
+
|
|
505
|
+
/* Warmup: drive the consumer to a fully-active state before
|
|
506
|
+
* chaos begins. Produces MSGS_PER_PARTITION to every
|
|
507
|
+
* partition; consumes them all. */
|
|
508
|
+
TEST_SAY(
|
|
509
|
+
"Warmup: producing %d msgs/partition x %d partitions "
|
|
510
|
+
"(testid=%" PRIu64 ")\n",
|
|
511
|
+
MSGS_PER_PARTITION, PARTITION_CNT, testid_warmup);
|
|
512
|
+
produce_phase(topic, testid_warmup, PARTITION_CNT);
|
|
513
|
+
|
|
514
|
+
test_msgver_init(&mv_warmup, testid_warmup);
|
|
515
|
+
got =
|
|
516
|
+
consume_into_msgver(rkshare, &mv_warmup, MSGS_PER_PHASE, 30 * 1000);
|
|
517
|
+
TEST_ASSERT(got == MSGS_PER_PHASE, "Warmup: expected %d msgs, got %d",
|
|
518
|
+
MSGS_PER_PHASE, got);
|
|
519
|
+
test_msgver_clear(&mv_warmup);
|
|
520
|
+
|
|
521
|
+
/* Chaos loop. */
|
|
522
|
+
for (cycle = 0; cycle < chaos_cycles; cycle++) {
|
|
523
|
+
int sleep_ms;
|
|
524
|
+
int32_t next_partition_cnt;
|
|
525
|
+
int do_produce;
|
|
526
|
+
int n_per_part;
|
|
527
|
+
|
|
528
|
+
sleep_ms =
|
|
529
|
+
chaos_sleep_min_ms +
|
|
530
|
+
(rand() % (chaos_sleep_max_ms - chaos_sleep_min_ms + 1));
|
|
531
|
+
next_partition_cnt =
|
|
532
|
+
chaos_partition_choices[rand() %
|
|
533
|
+
chaos_partition_choices_cnt];
|
|
534
|
+
do_produce = rand() % 2;
|
|
535
|
+
n_per_part = do_produce
|
|
536
|
+
? (rand() % (chaos_max_records_per_part + 1))
|
|
537
|
+
: 0;
|
|
538
|
+
|
|
539
|
+
TEST_SAY(
|
|
540
|
+
"[chaos cycle %d/%d] sleep=%dms, produce=%d "
|
|
541
|
+
"msg(s)/partition (cur_partitions=%" PRId32
|
|
542
|
+
"), next_partition_cnt=%" PRId32 "\n",
|
|
543
|
+
cycle + 1, chaos_cycles, sleep_ms, n_per_part,
|
|
544
|
+
current_partition_cnt, next_partition_cnt);
|
|
545
|
+
|
|
546
|
+
rd_usleep((int64_t)sleep_ms * 1000, NULL);
|
|
547
|
+
|
|
548
|
+
/* Optional mid-cycle produce. Uses a throwaway testid
|
|
549
|
+
* so these records are silently ignored on consume —
|
|
550
|
+
* they may or may not arrive depending on what state
|
|
551
|
+
* the consumer is in. */
|
|
552
|
+
if (n_per_part > 0) {
|
|
553
|
+
uint64_t cycle_testid = test_id_generate();
|
|
554
|
+
int32_t cp;
|
|
555
|
+
for (cp = 0; cp < current_partition_cnt; cp++) {
|
|
556
|
+
test_produce_msgs2(common_producer, topic,
|
|
557
|
+
cycle_testid, cp,
|
|
558
|
+
0 /*msg_base*/, n_per_part,
|
|
559
|
+
NULL /*payload*/, 0);
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
recreate_topic(topic, next_partition_cnt);
|
|
564
|
+
current_partition_cnt = next_partition_cnt;
|
|
565
|
+
|
|
566
|
+
/* Force a producer-side metadata refresh
|
|
567
|
+
* so its view matches the broker before we try to
|
|
568
|
+
* produce again. */
|
|
569
|
+
test_wait_topic_exists(common_producer, topic, 60 * 1000);
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
/* Settle: produce a final batch we will require to be fully
|
|
573
|
+
* consumed, against whatever topic instance chaos left us
|
|
574
|
+
* with. */
|
|
575
|
+
TEST_SAY("Settle: producing %d msgs/partition x %" PRId32
|
|
576
|
+
" partitions (testid=%" PRIu64 ")\n",
|
|
577
|
+
MSGS_PER_PARTITION, current_partition_cnt, testid_final);
|
|
578
|
+
produce_phase(topic, testid_final, current_partition_cnt);
|
|
579
|
+
|
|
580
|
+
final_msgs = current_partition_cnt * MSGS_PER_PARTITION;
|
|
581
|
+
|
|
582
|
+
test_msgver_init(&mv_final, testid_final);
|
|
583
|
+
got = consume_into_msgver(rkshare, &mv_final, final_msgs, 60 * 1000);
|
|
584
|
+
TEST_ASSERT(got == final_msgs,
|
|
585
|
+
"Settle: expected %d msgs matching testid_final, "
|
|
586
|
+
"got %d after %d chaos cycles",
|
|
587
|
+
final_msgs, got, chaos_cycles);
|
|
588
|
+
for (p = 0; p < current_partition_cnt; p++)
|
|
589
|
+
test_msgver_verify_part("chaos-final-part", &mv_final,
|
|
590
|
+
TEST_MSGVER_ALL_PART, topic, p,
|
|
591
|
+
0 /*msg_base*/, MSGS_PER_PARTITION);
|
|
592
|
+
test_msgver_clear(&mv_final);
|
|
593
|
+
|
|
594
|
+
/* Verify chaos actually moved the topic_id. */
|
|
595
|
+
id_final = fetch_topic_id(topic);
|
|
596
|
+
TEST_SAY("Final topic_id: %s\n", rd_kafka_Uuid_base64str(id_final));
|
|
597
|
+
assert_topic_id_changed(id_initial, id_final);
|
|
598
|
+
|
|
599
|
+
test_share_consumer_close(rkshare);
|
|
600
|
+
test_share_destroy(rkshare);
|
|
601
|
+
rd_kafka_Uuid_destroy(id_initial);
|
|
602
|
+
rd_kafka_Uuid_destroy(id_final);
|
|
603
|
+
|
|
604
|
+
SUB_TEST_PASS();
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
/**
|
|
608
|
+
* @brief Subscribed consumer survives topic delete + recreate-with-
|
|
609
|
+
* different-partition-count while a producer is actively
|
|
610
|
+
* producing across the whole window.
|
|
611
|
+
*
|
|
612
|
+
* Differs from the other recreate tests in that this one keeps a
|
|
613
|
+
* concurrent producer thread running through the entire delete window.
|
|
614
|
+
*
|
|
615
|
+
* Assertions:
|
|
616
|
+
* - At least one "before" record consumed (consumer was healthy).
|
|
617
|
+
* - At least one "after" record consumed (consumer recovered).
|
|
618
|
+
* - "During" records may or may not be consumed; no assertion.
|
|
619
|
+
* - Final topic_id differs from the initial topic_id (broker
|
|
620
|
+
* actually recreated the topic).
|
|
621
|
+
*/
|
|
622
|
+
static void do_test_recreate_survives_concurrent_producer(void) {
|
|
623
|
+
const char *topic;
|
|
624
|
+
const char *group_id = "0184-share-recreate-concurrent-producer";
|
|
625
|
+
const int32_t partition_cnt_a = 3;
|
|
626
|
+
const int32_t partition_cnt_b = 5;
|
|
627
|
+
const int producer_period_ms = 100; /* ~10 msgs/sec */
|
|
628
|
+
const int phase_duration_ms = 10000;
|
|
629
|
+
rd_kafka_share_t *rkshare;
|
|
630
|
+
rd_kafka_messages_t *batch = NULL;
|
|
631
|
+
struct producer_thread_args producer_args;
|
|
632
|
+
thrd_t producer_thrd;
|
|
633
|
+
rd_kafka_Uuid_t *id_initial, *id_final;
|
|
634
|
+
int before_cnt = 0, during_cnt = 0, after_cnt = 0;
|
|
635
|
+
rd_ts_t consume_deadline;
|
|
636
|
+
|
|
637
|
+
SUB_TEST_QUICK();
|
|
638
|
+
|
|
639
|
+
topic = test_mk_topic_name("0184-recreate-concurrent-producer", 1);
|
|
640
|
+
|
|
641
|
+
test_create_topic_wait_exists(common_admin, topic, partition_cnt_a, -1,
|
|
642
|
+
60 * 1000);
|
|
643
|
+
id_initial = fetch_topic_id(topic);
|
|
644
|
+
TEST_SAY("Initial topic_id: %s (partition_cnt=%" PRId32 ")\n",
|
|
645
|
+
rd_kafka_Uuid_base64str(id_initial), partition_cnt_a);
|
|
646
|
+
|
|
647
|
+
test_share_set_auto_offset_reset(group_id, "earliest");
|
|
648
|
+
|
|
649
|
+
rkshare = test_create_share_consumer(group_id, "implicit");
|
|
650
|
+
|
|
651
|
+
test_share_consumer_subscribe_multi(rkshare, 1, topic);
|
|
652
|
+
|
|
653
|
+
/* Start background producer in "before" phase. */
|
|
654
|
+
producer_args.producer = common_producer;
|
|
655
|
+
producer_args.topic = topic;
|
|
656
|
+
producer_args.period_ms = producer_period_ms;
|
|
657
|
+
producer_args.stop = rd_false;
|
|
658
|
+
rd_snprintf(producer_args.phase, sizeof(producer_args.phase), "before");
|
|
659
|
+
|
|
660
|
+
TEST_ASSERT(thrd_create(&producer_thrd, producer_thread_main,
|
|
661
|
+
&producer_args) == thrd_success,
|
|
662
|
+
"thrd_create failed");
|
|
663
|
+
|
|
664
|
+
/* --- Phase: BEFORE.
|
|
665
|
+
* Consume for a few seconds while producer thread feeds the
|
|
666
|
+
* topic. Count "before:" records to confirm the pipeline is
|
|
667
|
+
* live. */
|
|
668
|
+
consume_deadline = test_clock() + (rd_ts_t)phase_duration_ms * 1000;
|
|
669
|
+
while (test_clock() < consume_deadline) {
|
|
670
|
+
rd_kafka_error_t *err;
|
|
671
|
+
size_t rcvd = 0;
|
|
672
|
+
size_t i;
|
|
673
|
+
err = rd_kafka_share_poll(rkshare, 200, &batch);
|
|
674
|
+
if (err) {
|
|
675
|
+
TEST_SAY(
|
|
676
|
+
"Phase BEFORE: share_poll error: "
|
|
677
|
+
"%s\n",
|
|
678
|
+
rd_kafka_error_string(err));
|
|
679
|
+
rd_kafka_error_destroy(err);
|
|
680
|
+
continue;
|
|
681
|
+
}
|
|
682
|
+
rcvd = rd_kafka_messages_count(batch);
|
|
683
|
+
for (i = 0; i < rcvd; i++) {
|
|
684
|
+
rd_kafka_message_t *m = rd_kafka_messages_get(batch, i);
|
|
685
|
+
if (!m->err && m->payload && m->len >= 7 &&
|
|
686
|
+
!strncmp((const char *)m->payload, "before:", 7))
|
|
687
|
+
before_cnt++;
|
|
688
|
+
}
|
|
689
|
+
rd_kafka_messages_destroy(batch);
|
|
690
|
+
batch = NULL;
|
|
691
|
+
}
|
|
692
|
+
TEST_SAY("Phase BEFORE: consumed %d \"before:\" record(s)\n",
|
|
693
|
+
before_cnt);
|
|
694
|
+
|
|
695
|
+
/* --- Transition: enter DURING. Delete the topic but keep the
|
|
696
|
+
* producer running. Most produces in this window will fail with
|
|
697
|
+
* Unknown topic-type errors; that's expected. */
|
|
698
|
+
TEST_SAY("Phase transition: entering DURING (delete topic)\n");
|
|
699
|
+
rd_snprintf(producer_args.phase, sizeof(producer_args.phase), "during");
|
|
700
|
+
test_delete_topic(common_admin, topic);
|
|
701
|
+
|
|
702
|
+
/* Drain the consumer briefly during the delete window. Most
|
|
703
|
+
* fetches will return errors. We don't track "during:" counts
|
|
704
|
+
* for an assertion; just log. */
|
|
705
|
+
consume_deadline = test_clock() + (rd_ts_t)phase_duration_ms * 1000;
|
|
706
|
+
while (test_clock() < consume_deadline) {
|
|
707
|
+
rd_kafka_error_t *err;
|
|
708
|
+
size_t rcvd = 0;
|
|
709
|
+
size_t i;
|
|
710
|
+
err = rd_kafka_share_poll(rkshare, 200, &batch);
|
|
711
|
+
if (err) {
|
|
712
|
+
TEST_SAY(
|
|
713
|
+
"Phase DURING: share_poll error: "
|
|
714
|
+
"%s\n",
|
|
715
|
+
rd_kafka_error_string(err));
|
|
716
|
+
rd_kafka_error_destroy(err);
|
|
717
|
+
continue;
|
|
718
|
+
}
|
|
719
|
+
rcvd = rd_kafka_messages_count(batch);
|
|
720
|
+
for (i = 0; i < rcvd; i++) {
|
|
721
|
+
rd_kafka_message_t *m = rd_kafka_messages_get(batch, i);
|
|
722
|
+
if (!m->err && m->payload && m->len >= 7 &&
|
|
723
|
+
!strncmp((const char *)m->payload, "during:", 7))
|
|
724
|
+
during_cnt++;
|
|
725
|
+
}
|
|
726
|
+
rd_kafka_messages_destroy(batch);
|
|
727
|
+
batch = NULL;
|
|
728
|
+
}
|
|
729
|
+
TEST_SAY(
|
|
730
|
+
"Phase DURING: consumed %d \"during:\" record(s) "
|
|
731
|
+
"(no assertion on this count)\n",
|
|
732
|
+
during_cnt);
|
|
733
|
+
|
|
734
|
+
/* --- Transition: recreate with a different partition count
|
|
735
|
+
* and switch producer to "after". */
|
|
736
|
+
TEST_SAY("Recreating topic with partition_cnt=%" PRId32 " (was %" PRId32
|
|
737
|
+
")\n",
|
|
738
|
+
partition_cnt_b, partition_cnt_a);
|
|
739
|
+
rd_sleep(5); /* let delete settle on the broker */
|
|
740
|
+
test_create_topic_wait_exists(common_admin, topic, partition_cnt_b, -1,
|
|
741
|
+
60 * 1000);
|
|
742
|
+
|
|
743
|
+
id_final = fetch_topic_id(topic);
|
|
744
|
+
TEST_SAY("Recreated topic_id: %s (partition_cnt=%" PRId32 ")\n",
|
|
745
|
+
rd_kafka_Uuid_base64str(id_final), partition_cnt_b);
|
|
746
|
+
|
|
747
|
+
/* Force the producer to refresh its metadata for the recreated
|
|
748
|
+
* topic.
|
|
749
|
+
*/
|
|
750
|
+
test_wait_topic_exists(common_producer, topic, 60 * 1000);
|
|
751
|
+
|
|
752
|
+
rd_snprintf(producer_args.phase, sizeof(producer_args.phase), "after");
|
|
753
|
+
|
|
754
|
+
/* --- Phase: AFTER.
|
|
755
|
+
* Consume for a longer window to give the consumer time to
|
|
756
|
+
* reconcile the new topic_id and start delivering "after:"
|
|
757
|
+
* records. */
|
|
758
|
+
consume_deadline = test_clock() + 60 * 1000 * 1000;
|
|
759
|
+
while (after_cnt == 0 && test_clock() < consume_deadline) {
|
|
760
|
+
rd_kafka_error_t *err;
|
|
761
|
+
size_t rcvd = 0;
|
|
762
|
+
size_t i;
|
|
763
|
+
err = rd_kafka_share_poll(rkshare, 500, &batch);
|
|
764
|
+
if (err) {
|
|
765
|
+
TEST_SAY(
|
|
766
|
+
"Phase AFTER: share_poll error: "
|
|
767
|
+
"%s\n",
|
|
768
|
+
rd_kafka_error_string(err));
|
|
769
|
+
rd_kafka_error_destroy(err);
|
|
770
|
+
continue;
|
|
771
|
+
}
|
|
772
|
+
rcvd = rd_kafka_messages_count(batch);
|
|
773
|
+
for (i = 0; i < rcvd; i++) {
|
|
774
|
+
rd_kafka_message_t *m = rd_kafka_messages_get(batch, i);
|
|
775
|
+
if (!m->err && m->payload && m->len >= 6 &&
|
|
776
|
+
!strncmp((const char *)m->payload, "after:", 6))
|
|
777
|
+
after_cnt++;
|
|
778
|
+
}
|
|
779
|
+
rd_kafka_messages_destroy(batch);
|
|
780
|
+
batch = NULL;
|
|
781
|
+
}
|
|
782
|
+
TEST_SAY("Phase AFTER: consumed %d \"after:\" record(s)\n", after_cnt);
|
|
783
|
+
|
|
784
|
+
/* Stop and join the producer thread before any assertion
|
|
785
|
+
* runs.
|
|
786
|
+
*/
|
|
787
|
+
producer_args.stop = rd_true;
|
|
788
|
+
thrd_join(producer_thrd, NULL);
|
|
789
|
+
|
|
790
|
+
test_share_consumer_close(rkshare);
|
|
791
|
+
test_share_destroy(rkshare);
|
|
792
|
+
|
|
793
|
+
TEST_ASSERT(before_cnt > 0,
|
|
794
|
+
"Expected at least one \"before:\" record to be "
|
|
795
|
+
"consumed");
|
|
796
|
+
assert_topic_id_changed(id_initial, id_final);
|
|
797
|
+
TEST_ASSERT(after_cnt > 0,
|
|
798
|
+
"Consumer did not recover after recreate: zero "
|
|
799
|
+
"\"after:\" records seen within 60s");
|
|
800
|
+
|
|
801
|
+
rd_kafka_Uuid_destroy(id_initial);
|
|
802
|
+
rd_kafka_Uuid_destroy(id_final);
|
|
803
|
+
|
|
804
|
+
SUB_TEST_PASS();
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
/**
|
|
808
|
+
* @brief Topic ID changes server-side while the consumer is in
|
|
809
|
+
* the middle of close(); verify close completes cleanly.
|
|
810
|
+
*
|
|
811
|
+
* Assertions:
|
|
812
|
+
* - close() completes within a reasonable time
|
|
813
|
+
*/
|
|
814
|
+
static void test_recreate_during_close(void) {
|
|
815
|
+
rd_kafka_mock_cluster_t *mcluster;
|
|
816
|
+
const char *bootstraps;
|
|
817
|
+
rd_kafka_share_t *rkshare;
|
|
818
|
+
rd_kafka_error_t *error;
|
|
819
|
+
const char *topic = "0184-recreate-during-close";
|
|
820
|
+
const char *group = "0184-share-recreate-during-close";
|
|
821
|
+
const int n_msgs = 10;
|
|
822
|
+
const int delay_ms = 5000;
|
|
823
|
+
rd_kafka_message_t *rkmessages[64];
|
|
824
|
+
rd_kafka_messages_t *batches[64];
|
|
825
|
+
size_t batches_cnt = 0;
|
|
826
|
+
struct recreate_thread_args thread_args;
|
|
827
|
+
thrd_t recreate_thrd;
|
|
828
|
+
rd_kafka_conf_t *conf;
|
|
829
|
+
size_t rcvd = 0;
|
|
830
|
+
int attempts = 0;
|
|
831
|
+
int max_attempt = 30;
|
|
832
|
+
int i;
|
|
833
|
+
rd_ts_t t_start, t_elapsed_ms;
|
|
834
|
+
const int64_t close_upper_bound_ms = delay_ms + 2000;
|
|
835
|
+
|
|
836
|
+
SUB_TEST_QUICK();
|
|
837
|
+
|
|
838
|
+
mcluster = test_mock_cluster_new(1, &bootstraps);
|
|
839
|
+
rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1);
|
|
840
|
+
|
|
841
|
+
TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 1, 1) ==
|
|
842
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
843
|
+
"Failed to create mock topic");
|
|
844
|
+
|
|
845
|
+
TEST_SAY("Producing %d messages to topic %s\n", n_msgs, topic);
|
|
846
|
+
test_produce_msgs_easy_v(topic, 0, 0, 0, n_msgs, 16,
|
|
847
|
+
"bootstrap.servers", bootstraps, NULL);
|
|
848
|
+
|
|
849
|
+
TEST_SAY("Creating share consumer with explicit ack mode\n");
|
|
850
|
+
test_conf_init(&conf, NULL, 0);
|
|
851
|
+
test_conf_set(conf, "bootstrap.servers", bootstraps);
|
|
852
|
+
test_conf_set(conf, "group.id", group);
|
|
853
|
+
test_conf_set(conf, "share.acknowledgement.mode", "explicit");
|
|
854
|
+
test_conf_set(conf, "topic.metadata.refresh.interval.ms", "500");
|
|
855
|
+
rkshare = rd_kafka_share_consumer_new(conf, NULL, 0);
|
|
856
|
+
TEST_ASSERT(rkshare != NULL, "Failed to create share consumer");
|
|
857
|
+
|
|
858
|
+
test_share_consumer_subscribe_multi(rkshare, 1, topic);
|
|
859
|
+
|
|
860
|
+
TEST_SAY("Consuming up to %d messages\n", n_msgs);
|
|
861
|
+
while (rcvd < (size_t)n_msgs && attempts < max_attempt) {
|
|
862
|
+
rd_kafka_messages_t *batch = NULL;
|
|
863
|
+
size_t batch_rcvd;
|
|
864
|
+
size_t k;
|
|
865
|
+
error = rd_kafka_share_poll(rkshare, 3000, &batch);
|
|
866
|
+
if (error) {
|
|
867
|
+
TEST_SAY("Consume attempt %d: %s\n", attempts,
|
|
868
|
+
rd_kafka_error_string(error));
|
|
869
|
+
rd_kafka_error_destroy(error);
|
|
870
|
+
rd_kafka_messages_destroy(batch);
|
|
871
|
+
attempts++;
|
|
872
|
+
continue;
|
|
873
|
+
}
|
|
874
|
+
batch_rcvd = rd_kafka_messages_count(batch);
|
|
875
|
+
if (batch_rcvd > 0 &&
|
|
876
|
+
batches_cnt < sizeof(batches) / sizeof(batches[0]) &&
|
|
877
|
+
rcvd + batch_rcvd <=
|
|
878
|
+
sizeof(rkmessages) / sizeof(rkmessages[0])) {
|
|
879
|
+
for (k = 0; k < batch_rcvd; k++)
|
|
880
|
+
rkmessages[rcvd + k] =
|
|
881
|
+
rd_kafka_messages_get(batch, k);
|
|
882
|
+
rcvd += batch_rcvd;
|
|
883
|
+
batches[batches_cnt++] = batch;
|
|
884
|
+
} else {
|
|
885
|
+
rd_kafka_messages_destroy(batch);
|
|
886
|
+
}
|
|
887
|
+
attempts++;
|
|
888
|
+
}
|
|
889
|
+
TEST_ASSERT(rcvd == (size_t)n_msgs,
|
|
890
|
+
"Expected to consume %d messages, got %zu", n_msgs, rcvd);
|
|
891
|
+
|
|
892
|
+
TEST_SAY("Staging acks for all %zu messages\n", rcvd);
|
|
893
|
+
for (i = 0; i < (int)rcvd; i++) {
|
|
894
|
+
rd_kafka_resp_err_t ack_err =
|
|
895
|
+
rd_kafka_share_acknowledge(rkshare, rkmessages[i]);
|
|
896
|
+
TEST_ASSERT(ack_err == RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
897
|
+
"ack %d failed: %s", i, rd_kafka_err2str(ack_err));
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
TEST_SAY("Injecting %dms delay on the next ShareAcknowledge response\n",
|
|
901
|
+
delay_ms);
|
|
902
|
+
TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts(
|
|
903
|
+
mcluster, 1, RD_KAFKAP_ShareAcknowledge, 1,
|
|
904
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
905
|
+
delay_ms) == RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
906
|
+
"Failed to inject ShareAcknowledge delay");
|
|
907
|
+
|
|
908
|
+
thread_args.mcluster = mcluster;
|
|
909
|
+
thread_args.topic = topic;
|
|
910
|
+
thread_args.partition_cnt = 1;
|
|
911
|
+
thread_args.sleep_ms = 1000;
|
|
912
|
+
TEST_ASSERT(thrd_create(&recreate_thrd, recreate_thread_main,
|
|
913
|
+
&thread_args) == thrd_success,
|
|
914
|
+
"thrd_create failed");
|
|
915
|
+
|
|
916
|
+
TEST_SAY(
|
|
917
|
+
"Calling close(); broker will hold ack for %dms while "
|
|
918
|
+
"background thread recreates the topic\n",
|
|
919
|
+
delay_ms);
|
|
920
|
+
t_start = test_clock();
|
|
921
|
+
error = rd_kafka_share_consumer_close(rkshare);
|
|
922
|
+
t_elapsed_ms = (test_clock() - t_start) / 1000;
|
|
923
|
+
TEST_SAY("close() returned in %" PRId64 " ms\n", t_elapsed_ms);
|
|
924
|
+
|
|
925
|
+
if (error) {
|
|
926
|
+
TEST_SAY("close() returned error: %s\n",
|
|
927
|
+
rd_kafka_error_string(error));
|
|
928
|
+
rd_kafka_error_destroy(error);
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
TEST_ASSERT(t_elapsed_ms <= close_upper_bound_ms,
|
|
932
|
+
"close() took %" PRId64 " ms, expected <= %" PRId64
|
|
933
|
+
" ms (broker delay + overhead)",
|
|
934
|
+
t_elapsed_ms, close_upper_bound_ms);
|
|
935
|
+
|
|
936
|
+
thrd_join(recreate_thrd, NULL);
|
|
937
|
+
|
|
938
|
+
for (i = 0; i < (int)batches_cnt; i++)
|
|
939
|
+
rd_kafka_messages_destroy(batches[i]);
|
|
940
|
+
|
|
941
|
+
test_share_destroy(rkshare);
|
|
942
|
+
test_mock_cluster_destroy(mcluster);
|
|
943
|
+
SUB_TEST_PASS();
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
/* ===================================================================
|
|
947
|
+
* Log callback: counts the "no partitions are assigned" fetch-stall
|
|
948
|
+
* log emitted while the share consumer's assignment is empty.
|
|
949
|
+
* =================================================================== */
|
|
950
|
+
static void recreate_stall_log_cb(const rd_kafka_t *rk,
|
|
951
|
+
int level,
|
|
952
|
+
const char *fac,
|
|
953
|
+
const char *buf) {
|
|
954
|
+
rd_atomic32_t *cnt = rd_kafka_opaque(rk);
|
|
955
|
+
if (cnt && !strcmp(fac, "FETCHMORE") &&
|
|
956
|
+
strstr(buf, "no partitions are assigned"))
|
|
957
|
+
rd_atomic32_add(cnt, 1);
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
/**
|
|
961
|
+
* @brief Delete the consumer's only topic so the broker revokes its
|
|
962
|
+
* assignment to empty, verify the consumer reports the
|
|
963
|
+
* no-partitions fetch stall, then recreate the topic and verify
|
|
964
|
+
* records flow again.
|
|
965
|
+
*
|
|
966
|
+
* Reproduces the observed delete/recreate latency: with no partitions
|
|
967
|
+
* assigned the consumer keeps wanting to fetch and emits the
|
|
968
|
+
* rate-limited "Fetch stalled: ... no partitions are assigned" log
|
|
969
|
+
* until the recreated topic (fresh topic_id) is reassigned.
|
|
970
|
+
*/
|
|
971
|
+
static void do_test_recreate_stall_then_recover(void) {
|
|
972
|
+
const char *topic;
|
|
973
|
+
const char *group = "0184-share-recreate-stall";
|
|
974
|
+
rd_kafka_share_t *rkshare;
|
|
975
|
+
rd_kafka_conf_t *conf;
|
|
976
|
+
rd_atomic32_t stall_cnt;
|
|
977
|
+
test_msgver_t mv_before, mv_after;
|
|
978
|
+
uint64_t testid_before, testid_after;
|
|
979
|
+
rd_kafka_Uuid_t *id_before, *id_after;
|
|
980
|
+
rd_kafka_messages_t *batch = NULL;
|
|
981
|
+
rd_ts_t deadline;
|
|
982
|
+
int got, stalls;
|
|
983
|
+
char errstr[512];
|
|
984
|
+
|
|
985
|
+
SUB_TEST_QUICK();
|
|
986
|
+
|
|
987
|
+
rd_atomic32_init(&stall_cnt, 0);
|
|
988
|
+
testid_before = test_id_generate();
|
|
989
|
+
testid_after = test_id_generate();
|
|
990
|
+
|
|
991
|
+
topic = test_mk_topic_name("0184-recreate-stall", 1);
|
|
992
|
+
test_create_topic_wait_exists(common_admin, topic, PARTITION_CNT, -1,
|
|
993
|
+
60 * 1000);
|
|
994
|
+
|
|
995
|
+
id_before = fetch_topic_id(topic);
|
|
996
|
+
test_share_set_auto_offset_reset(group, "earliest");
|
|
997
|
+
|
|
998
|
+
/* Fast metadata refresh + debug=consumer so the no-partitions
|
|
999
|
+
* stall log is emitted and observable via the log callback. */
|
|
1000
|
+
test_conf_init(&conf, NULL, 0);
|
|
1001
|
+
test_conf_set(conf, "group.id", group);
|
|
1002
|
+
test_conf_set(conf, "share.acknowledgement.mode", "explicit");
|
|
1003
|
+
test_conf_set(conf, "topic.metadata.refresh.interval.ms", "500");
|
|
1004
|
+
test_conf_set(conf, "debug", "consumer");
|
|
1005
|
+
rd_kafka_conf_set_log_cb(conf, recreate_stall_log_cb);
|
|
1006
|
+
rd_kafka_conf_set_opaque(conf, &stall_cnt);
|
|
1007
|
+
rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr));
|
|
1008
|
+
TEST_ASSERT(rkshare, "Failed to create share consumer: %s", errstr);
|
|
1009
|
+
|
|
1010
|
+
test_share_consumer_subscribe_multi(rkshare, 1, topic);
|
|
1011
|
+
|
|
1012
|
+
/* ---- Phase 1: consume the original generation ---- */
|
|
1013
|
+
produce_phase(topic, testid_before, PARTITION_CNT);
|
|
1014
|
+
test_msgver_init(&mv_before, testid_before);
|
|
1015
|
+
got =
|
|
1016
|
+
consume_into_msgver(rkshare, &mv_before, MSGS_PER_PHASE, 30 * 1000);
|
|
1017
|
+
TEST_ASSERT(got == MSGS_PER_PHASE, "Phase 1: expected %d msgs, got %d",
|
|
1018
|
+
MSGS_PER_PHASE, got);
|
|
1019
|
+
test_msgver_clear(&mv_before);
|
|
1020
|
+
|
|
1021
|
+
/* ---- Delete: the broker revokes the assignment to empty ---- */
|
|
1022
|
+
TEST_SAY("Deleting topic %s; expecting a no-partitions fetch stall\n",
|
|
1023
|
+
topic);
|
|
1024
|
+
test_delete_topic(common_admin, topic);
|
|
1025
|
+
|
|
1026
|
+
/* Poll until the no-partitions stall is logged. The log is
|
|
1027
|
+
* rate-limited, so a single occurrence confirms the path. */
|
|
1028
|
+
deadline = test_clock() + (rd_ts_t)60 * 1000 * 1000;
|
|
1029
|
+
while (rd_atomic32_get(&stall_cnt) < 1 && test_clock() < deadline) {
|
|
1030
|
+
rd_kafka_error_t *error =
|
|
1031
|
+
rd_kafka_share_poll(rkshare, 500, &batch);
|
|
1032
|
+
if (error) {
|
|
1033
|
+
rd_kafka_error_destroy(error);
|
|
1034
|
+
} else {
|
|
1035
|
+
/* Ack any straggler so the next poll can proceed
|
|
1036
|
+
* in explicit mode. */
|
|
1037
|
+
size_t i, n = rd_kafka_messages_count(batch);
|
|
1038
|
+
for (i = 0; i < n; i++)
|
|
1039
|
+
rd_kafka_share_acknowledge(
|
|
1040
|
+
rkshare, rd_kafka_messages_get(batch, i));
|
|
1041
|
+
}
|
|
1042
|
+
rd_kafka_messages_destroy(batch);
|
|
1043
|
+
batch = NULL;
|
|
1044
|
+
}
|
|
1045
|
+
stalls = rd_atomic32_get(&stall_cnt);
|
|
1046
|
+
TEST_SAY("\"no partitions assigned\" stall log count: %d\n", stalls);
|
|
1047
|
+
TEST_ASSERT(stalls >= 1,
|
|
1048
|
+
"expected the no-partitions stall to be logged, got %d",
|
|
1049
|
+
stalls);
|
|
1050
|
+
|
|
1051
|
+
/* ---- Recreate (fresh topic_id): the broker reassigns ---- */
|
|
1052
|
+
rd_sleep(5); /* let DeleteTopics settle, as in recreate_topic() */
|
|
1053
|
+
test_create_topic_wait_exists(common_admin, topic, PARTITION_CNT, -1,
|
|
1054
|
+
60 * 1000);
|
|
1055
|
+
id_after = fetch_topic_id(topic);
|
|
1056
|
+
assert_topic_id_changed(id_before, id_after);
|
|
1057
|
+
|
|
1058
|
+
/* ---- Phase 2: records flow again ---- */
|
|
1059
|
+
produce_phase(topic, testid_after, PARTITION_CNT);
|
|
1060
|
+
test_msgver_init(&mv_after, testid_after);
|
|
1061
|
+
got =
|
|
1062
|
+
consume_into_msgver(rkshare, &mv_after, MSGS_PER_PHASE, 60 * 1000);
|
|
1063
|
+
TEST_ASSERT(got == MSGS_PER_PHASE,
|
|
1064
|
+
"Phase 2: expected %d msgs after recreate, got %d",
|
|
1065
|
+
MSGS_PER_PHASE, got);
|
|
1066
|
+
test_msgver_clear(&mv_after);
|
|
1067
|
+
|
|
1068
|
+
test_share_consumer_close(rkshare);
|
|
1069
|
+
test_share_destroy(rkshare);
|
|
1070
|
+
rd_kafka_Uuid_destroy(id_before);
|
|
1071
|
+
rd_kafka_Uuid_destroy(id_after);
|
|
1072
|
+
|
|
1073
|
+
SUB_TEST_PASS();
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1076
|
+
int main_0184_share_consumer_topic_recreate(int argc, char **argv) {
|
|
1077
|
+
/* Topic deletion is not supported against Windows brokers. */
|
|
1078
|
+
if (!strcmp(test_getenv("TEST_BROKER_OS", ""), "windows")) {
|
|
1079
|
+
TEST_SKIP("Topic deletion not supported on Windows brokers\n");
|
|
1080
|
+
return 0;
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
test_timeout_set(300);
|
|
1084
|
+
|
|
1085
|
+
common_producer = test_create_producer();
|
|
1086
|
+
common_admin = test_create_producer();
|
|
1087
|
+
|
|
1088
|
+
do_test_recreate_two_phase("0184-recreate-hb-first",
|
|
1089
|
+
"0184-share-recreate-hb-first",
|
|
1090
|
+
create_consumer_hb_first, PARTITION_CNT);
|
|
1091
|
+
do_test_recreate_two_phase("0184-recreate-md-first",
|
|
1092
|
+
"0184-share-recreate-md-first",
|
|
1093
|
+
create_consumer_md_first, PARTITION_CNT);
|
|
1094
|
+
do_test_recreate_two_phase("0184-recreate-shrink-hb-first",
|
|
1095
|
+
"0184-share-recreate-shrink-hb-first",
|
|
1096
|
+
create_consumer_hb_first, 2);
|
|
1097
|
+
do_test_recreate_two_phase("0184-recreate-shrink-md-first",
|
|
1098
|
+
"0184-share-recreate-shrink-md-first",
|
|
1099
|
+
create_consumer_md_first, 2);
|
|
1100
|
+
do_test_recreate_chaos();
|
|
1101
|
+
do_test_recreate_survives_concurrent_producer();
|
|
1102
|
+
do_test_recreate_stall_then_recover();
|
|
1103
|
+
|
|
1104
|
+
rd_kafka_destroy(common_admin);
|
|
1105
|
+
rd_kafka_destroy(common_producer);
|
|
1106
|
+
|
|
1107
|
+
return 0;
|
|
1108
|
+
}
|
|
1109
|
+
|
|
1110
|
+
/**
|
|
1111
|
+
* @brief Same-name topic recreate where a single consumer owns the same
|
|
1112
|
+
* (name, partition) tuples in both generations.
|
|
1113
|
+
*
|
|
1114
|
+
* The cooperative-assignment reconciler in rd_kafka_cgrp.c builds set
|
|
1115
|
+
* subtraction maps keyed by (topic_name, partition), so OLD-id and
|
|
1116
|
+
* NEW-id entries for the same (name, partition) collide in the diff.
|
|
1117
|
+
* When a single consumer owns every partition of a same-name recreated
|
|
1118
|
+
* topic, no partition lands in newly_added, no PARTITION_JOIN ->
|
|
1119
|
+
* share_session_toppar_add fires for the NEW rktp, and ShareFetch
|
|
1120
|
+
* never asks for the NEW generation.
|
|
1121
|
+
*
|
|
1122
|
+
* To deterministically expose the bug we need the consumer's heartbeat
|
|
1123
|
+
* to deliver a direct OLD-id -> NEW-id target transition with no
|
|
1124
|
+
* intermediate empty assignment. The mock cluster's heartbeat handler
|
|
1125
|
+
* normally only recalculates assignments on member events (join,
|
|
1126
|
+
* leave, subscription change) -- not on topic delete/recreate -- so a
|
|
1127
|
+
* naive delete + recreate would leave the mock emitting the stale
|
|
1128
|
+
* OLD-id assignment forever, never reaching the reconciler bug
|
|
1129
|
+
* condition. Instead, the test uses
|
|
1130
|
+
* rd_kafka_mock_sharegroup_target_assignment to manually inject the
|
|
1131
|
+
* NEW-id assignment after the recreate. The mock resolves the
|
|
1132
|
+
* partition list's topic name to the NEW topic_id at assignment-set
|
|
1133
|
+
* time, so the consumer's next heartbeat carries
|
|
1134
|
+
* (NEW-id, partition) tuples while its current assignment still holds
|
|
1135
|
+
* the (OLD-id, partition) tuples from PHASE 1 -- exactly the input
|
|
1136
|
+
* that triggers the (topic_name, partition) hash collision.
|
|
1137
|
+
*
|
|
1138
|
+
* Pre-fix: PHASE 2 times out with zero NEW records consumed.
|
|
1139
|
+
* Post-fix: PHASE 2 consumes all NEW records.
|
|
1140
|
+
*/
|
|
1141
|
+
static void do_test_recreate_same_name_partition_collision(void) {
|
|
1142
|
+
rd_kafka_mock_cluster_t *mcluster;
|
|
1143
|
+
const char *bootstraps;
|
|
1144
|
+
rd_kafka_share_t *rkshare;
|
|
1145
|
+
rd_kafka_conf_t *conf;
|
|
1146
|
+
rd_kafka_conf_t *producer_conf;
|
|
1147
|
+
rd_kafka_t *producer;
|
|
1148
|
+
const char *topic = "0184-recreate-collision";
|
|
1149
|
+
const char *group = "0184-share-recreate-collision";
|
|
1150
|
+
const int n_partitions = 3;
|
|
1151
|
+
const int n_msgs_per_part = 10;
|
|
1152
|
+
const int total_per_phase = n_partitions * n_msgs_per_part;
|
|
1153
|
+
rd_kafka_messages_t *batch = NULL;
|
|
1154
|
+
int old_got = 0;
|
|
1155
|
+
int new_got = 0;
|
|
1156
|
+
rd_ts_t deadline;
|
|
1157
|
+
int32_t p;
|
|
1158
|
+
char **member_ids;
|
|
1159
|
+
size_t member_cnt;
|
|
1160
|
+
size_t mi;
|
|
1161
|
+
rd_kafka_resp_err_t err;
|
|
1162
|
+
rd_kafka_topic_partition_list_t *new_assignment;
|
|
1163
|
+
rd_kafka_topic_partition_list_t *member_assignments[1];
|
|
1164
|
+
const char *member_id_arr[1];
|
|
1165
|
+
|
|
1166
|
+
SUB_TEST_QUICK();
|
|
1167
|
+
|
|
1168
|
+
mcluster = test_mock_cluster_new(1, &bootstraps);
|
|
1169
|
+
rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1);
|
|
1170
|
+
|
|
1171
|
+
/* Short heartbeat keeps the post-injection latency tight; the
|
|
1172
|
+
* consumer's next heartbeat after the manual assignment-set
|
|
1173
|
+
* delivers the NEW-id target within ~1s. */
|
|
1174
|
+
rd_kafka_mock_sharegroup_set_heartbeat_interval(mcluster, 1000);
|
|
1175
|
+
|
|
1176
|
+
/* Persistent producer for the whole test. The mock cluster
|
|
1177
|
+
* destroys every share-group session on the broker when any
|
|
1178
|
+
* client connection closes (see
|
|
1179
|
+
* rd_kafka_mock_sharegrps_node_connection_closed in
|
|
1180
|
+
* rdkafka_mock_sharegrp.c), so spinning up a fresh producer
|
|
1181
|
+
* instance per produce call would invalidate the consumer's
|
|
1182
|
+
* session as collateral damage and break PHASE 2's fetch.
|
|
1183
|
+
* Keeping one producer alive end-to-end avoids that. */
|
|
1184
|
+
test_conf_init(&producer_conf, NULL, 0);
|
|
1185
|
+
test_conf_set(producer_conf, "bootstrap.servers", bootstraps);
|
|
1186
|
+
rd_kafka_conf_set_dr_msg_cb(producer_conf, test_dr_msg_cb);
|
|
1187
|
+
producer = test_create_handle(RD_KAFKA_PRODUCER, producer_conf);
|
|
1188
|
+
TEST_ASSERT(producer != NULL, "producer create failed");
|
|
1189
|
+
|
|
1190
|
+
/* Initial topic generation. */
|
|
1191
|
+
TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, n_partitions,
|
|
1192
|
+
1) == RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1193
|
+
"mock_topic_create (initial) failed");
|
|
1194
|
+
|
|
1195
|
+
/* Produce n_msgs_per_part records to every partition of the
|
|
1196
|
+
* initial generation so PHASE 1 has something to drain. */
|
|
1197
|
+
for (p = 0; p < n_partitions; p++)
|
|
1198
|
+
test_produce_msgs2(producer, topic, 0 /*testid*/, p,
|
|
1199
|
+
0 /*msg_base*/, n_msgs_per_part,
|
|
1200
|
+
NULL /*payload*/, 0);
|
|
1201
|
+
|
|
1202
|
+
/* Long topic.metadata.refresh.interval.ms forces the heartbeat
|
|
1203
|
+
* to be the channel that carries topic_id changes, which is
|
|
1204
|
+
* what the reconciler bug rides on. */
|
|
1205
|
+
test_conf_init(&conf, NULL, 0);
|
|
1206
|
+
test_conf_set(conf, "bootstrap.servers", bootstraps);
|
|
1207
|
+
test_conf_set(conf, "group.id", group);
|
|
1208
|
+
test_conf_set(conf, "share.acknowledgement.mode", "explicit");
|
|
1209
|
+
test_conf_set(conf, "topic.metadata.refresh.interval.ms", "300000");
|
|
1210
|
+
rkshare = rd_kafka_share_consumer_new(conf, NULL, 0);
|
|
1211
|
+
TEST_ASSERT(rkshare != NULL, "share_consumer_new failed");
|
|
1212
|
+
|
|
1213
|
+
test_share_consumer_subscribe_multi(rkshare, 1, topic);
|
|
1214
|
+
|
|
1215
|
+
/* ---- PHASE 1: drain the OLD generation ----
|
|
1216
|
+
* Single consumer in the share group => it owns every
|
|
1217
|
+
* partition in this generation. Explicit-ack each record so
|
|
1218
|
+
* the next consume_batch can proceed. */
|
|
1219
|
+
TEST_SAY("PHASE 1: draining %d OLD-generation record(s)\n",
|
|
1220
|
+
total_per_phase);
|
|
1221
|
+
deadline = test_clock() + (rd_ts_t)30 * 1000 * 1000;
|
|
1222
|
+
while (old_got < total_per_phase && test_clock() < deadline) {
|
|
1223
|
+
rd_kafka_error_t *error;
|
|
1224
|
+
size_t cnt;
|
|
1225
|
+
size_t i;
|
|
1226
|
+
|
|
1227
|
+
error = rd_kafka_share_poll(rkshare, 500, &batch);
|
|
1228
|
+
if (error) {
|
|
1229
|
+
rd_kafka_error_destroy(error);
|
|
1230
|
+
rd_kafka_messages_destroy(batch);
|
|
1231
|
+
batch = NULL;
|
|
1232
|
+
continue;
|
|
1233
|
+
}
|
|
1234
|
+
cnt = rd_kafka_messages_count(batch);
|
|
1235
|
+
for (i = 0; i < cnt; i++) {
|
|
1236
|
+
rd_kafka_message_t *m = rd_kafka_messages_get(batch, i);
|
|
1237
|
+
if (!m->err) {
|
|
1238
|
+
old_got++;
|
|
1239
|
+
rd_kafka_share_acknowledge(rkshare, m);
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
rd_kafka_messages_destroy(batch);
|
|
1243
|
+
batch = NULL;
|
|
1244
|
+
}
|
|
1245
|
+
TEST_ASSERT(old_got == total_per_phase,
|
|
1246
|
+
"PHASE 1: expected %d OLD records, got %d", total_per_phase,
|
|
1247
|
+
old_got);
|
|
1248
|
+
|
|
1249
|
+
/* Capture the consumer's member id so we can pin a target
|
|
1250
|
+
* assignment for it after the recreate. */
|
|
1251
|
+
err = rd_kafka_mock_sharegroup_get_member_ids(mcluster, group,
|
|
1252
|
+
&member_ids, &member_cnt);
|
|
1253
|
+
TEST_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1254
|
+
"mock_sharegroup_get_member_ids failed: %s",
|
|
1255
|
+
rd_kafka_err2str(err));
|
|
1256
|
+
TEST_ASSERT(member_cnt == 1,
|
|
1257
|
+
"expected exactly 1 member in share group, got %zu",
|
|
1258
|
+
member_cnt);
|
|
1259
|
+
|
|
1260
|
+
/* ---- Delete + recreate ----
|
|
1261
|
+
* Mock generates a fresh topic_id for the recreated topic. The
|
|
1262
|
+
* existing share-group member's stored assignment still
|
|
1263
|
+
* references the destroyed OLD topic; we override it below. */
|
|
1264
|
+
TEST_SAY("Recreating topic same name, same partition count\n");
|
|
1265
|
+
TEST_ASSERT(rd_kafka_mock_topic_delete(mcluster, topic) ==
|
|
1266
|
+
RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1267
|
+
"mock_topic_delete failed");
|
|
1268
|
+
TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, n_partitions,
|
|
1269
|
+
1) == RD_KAFKA_RESP_ERR_NO_ERROR,
|
|
1270
|
+
"mock_topic_create (recreate) failed");
|
|
1271
|
+
|
|
1272
|
+
/* Pin the member's target assignment to the recreated topic.
|
|
1273
|
+
* mock_sharegroup_target_assignment resolves the topic name
|
|
1274
|
+
* to the current mtopic's id at set-time, so the partition
|
|
1275
|
+
* list ends up carrying the NEW topic_id. The mock bumps the
|
|
1276
|
+
* member epoch on set so the consumer's next heartbeat
|
|
1277
|
+
* receives this assignment. From the consumer's point of
|
|
1278
|
+
* view: current rkcg_group_assignment still has the OLD-id
|
|
1279
|
+
* partitions from PHASE 1; the next HB target carries the
|
|
1280
|
+
* NEW-id partitions. That is the precise input that triggers
|
|
1281
|
+
* the reconciler's (topic_name, partition) hash collision. */
|
|
1282
|
+
new_assignment = rd_kafka_topic_partition_list_new(n_partitions);
|
|
1283
|
+
for (p = 0; p < n_partitions; p++)
|
|
1284
|
+
rd_kafka_topic_partition_list_add(new_assignment, topic, p);
|
|
1285
|
+
member_id_arr[0] = member_ids[0];
|
|
1286
|
+
member_assignments[0] = new_assignment;
|
|
1287
|
+
rd_kafka_mock_sharegroup_target_assignment(
|
|
1288
|
+
mcluster, group, member_id_arr, member_assignments, 1);
|
|
1289
|
+
rd_kafka_topic_partition_list_destroy(new_assignment);
|
|
1290
|
+
|
|
1291
|
+
/* Produce records to every partition of the recreated topic.
|
|
1292
|
+
* Pre-fix these never reach the consumer. */
|
|
1293
|
+
for (p = 0; p < n_partitions; p++)
|
|
1294
|
+
test_produce_msgs2(producer, topic, 0 /*testid*/, p,
|
|
1295
|
+
0 /*msg_base*/, n_msgs_per_part,
|
|
1296
|
+
NULL /*payload*/, 0);
|
|
1297
|
+
|
|
1298
|
+
/* ---- PHASE 2: drain the NEW generation ----
|
|
1299
|
+
* Generous deadline so the heartbeat has time to deliver the
|
|
1300
|
+
* NEW target, metadata to resolve for the new topic_id, and
|
|
1301
|
+
* the share session to start fetching when the fix is in
|
|
1302
|
+
* place. Pre-fix the reconciler diff collapses
|
|
1303
|
+
* (name, partition) so the NEW rktps are never added to the
|
|
1304
|
+
* share session; this loop hits the deadline at zero. */
|
|
1305
|
+
TEST_SAY("PHASE 2: draining %d NEW-generation record(s)\n",
|
|
1306
|
+
total_per_phase);
|
|
1307
|
+
deadline = test_clock() + (rd_ts_t)10 * 1000 * 1000;
|
|
1308
|
+
while (new_got < total_per_phase && test_clock() < deadline) {
|
|
1309
|
+
rd_kafka_error_t *error;
|
|
1310
|
+
size_t cnt;
|
|
1311
|
+
size_t i;
|
|
1312
|
+
|
|
1313
|
+
error = rd_kafka_share_poll(rkshare, 500, &batch);
|
|
1314
|
+
if (error) {
|
|
1315
|
+
rd_kafka_error_destroy(error);
|
|
1316
|
+
rd_kafka_messages_destroy(batch);
|
|
1317
|
+
batch = NULL;
|
|
1318
|
+
continue;
|
|
1319
|
+
}
|
|
1320
|
+
cnt = rd_kafka_messages_count(batch);
|
|
1321
|
+
for (i = 0; i < cnt; i++) {
|
|
1322
|
+
rd_kafka_message_t *m = rd_kafka_messages_get(batch, i);
|
|
1323
|
+
if (!m->err) {
|
|
1324
|
+
new_got++;
|
|
1325
|
+
rd_kafka_share_acknowledge(rkshare, m);
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
rd_kafka_messages_destroy(batch);
|
|
1329
|
+
batch = NULL;
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
TEST_ASSERT(new_got == total_per_phase,
|
|
1333
|
+
"PHASE 2: expected %d NEW records after same-name "
|
|
1334
|
+
"recreate, got %d. Cooperative reconciler likely "
|
|
1335
|
+
"suppressed the NEW rktp adds because (name, partition) "
|
|
1336
|
+
"collided with OLD-id entries in rkcg_group_assignment.",
|
|
1337
|
+
total_per_phase, new_got);
|
|
1338
|
+
|
|
1339
|
+
for (mi = 0; mi < member_cnt; mi++)
|
|
1340
|
+
rd_free(member_ids[mi]);
|
|
1341
|
+
rd_free(member_ids);
|
|
1342
|
+
|
|
1343
|
+
test_share_consumer_close(rkshare);
|
|
1344
|
+
test_share_destroy(rkshare);
|
|
1345
|
+
rd_kafka_destroy(producer);
|
|
1346
|
+
test_mock_cluster_destroy(mcluster);
|
|
1347
|
+
SUB_TEST_PASS();
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
int main_0184_share_consumer_topic_recreate_local(int argc, char **argv) {
|
|
1351
|
+
TEST_SKIP_MOCK_CLUSTER(0);
|
|
1352
|
+
test_timeout_set(180);
|
|
1353
|
+
test_recreate_during_close();
|
|
1354
|
+
do_test_recreate_same_name_partition_collision();
|
|
1355
|
+
return 0;
|
|
1356
|
+
}
|