@ohos-ports/confluentinc-kafka-javascript 1.10.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +457 -0
- package/LICENSE.kafkajs +31 -0
- package/LICENSE.node-rdkafka +25 -0
- package/LICENSE.txt +20 -0
- package/README.md +133 -0
- package/binding.gyp +184 -0
- package/build/Release/confluent-kafka-javascript.node +0 -0
- package/build/deps/librdkafka++.so.1 +0 -0
- package/build/deps/librdkafka.Makefile +6 -0
- package/build/deps/librdkafka.so.1 +0 -0
- package/build/deps/librdkafka.target.mk +55 -0
- package/deps/librdkafka/.clang-format +136 -0
- package/deps/librdkafka/.clang-format-cpp +103 -0
- package/deps/librdkafka/.dir-locals.el +10 -0
- package/deps/librdkafka/.formatignore +35 -0
- package/deps/librdkafka/.gdbmacros +19 -0
- package/deps/librdkafka/CHANGELOG.md +2477 -0
- package/deps/librdkafka/CMakeLists.txt +291 -0
- package/deps/librdkafka/CODE_OF_CONDUCT.md +46 -0
- package/deps/librdkafka/CONFIGURATION.md +212 -0
- package/deps/librdkafka/CONTRIBUTING.md +431 -0
- package/deps/librdkafka/Doxyfile +2375 -0
- package/deps/librdkafka/INTRODUCTION.md +2752 -0
- package/deps/librdkafka/LICENSE +26 -0
- package/deps/librdkafka/LICENSE.cjson +22 -0
- package/deps/librdkafka/LICENSE.crc32c +28 -0
- package/deps/librdkafka/LICENSE.fnv1a +18 -0
- package/deps/librdkafka/LICENSE.hdrhistogram +27 -0
- package/deps/librdkafka/LICENSE.lz4 +26 -0
- package/deps/librdkafka/LICENSE.murmur2 +25 -0
- package/deps/librdkafka/LICENSE.nanopb +22 -0
- package/deps/librdkafka/LICENSE.opentelemetry +203 -0
- package/deps/librdkafka/LICENSE.pycrc +23 -0
- package/deps/librdkafka/LICENSE.queue +31 -0
- package/deps/librdkafka/LICENSE.regexp +5 -0
- package/deps/librdkafka/LICENSE.snappy +36 -0
- package/deps/librdkafka/LICENSE.tinycthread +26 -0
- package/deps/librdkafka/LICENSE.wingetopt +49 -0
- package/deps/librdkafka/LICENSES.txt +625 -0
- package/deps/librdkafka/Makefile +125 -0
- package/deps/librdkafka/README.md +201 -0
- package/deps/librdkafka/README.win32 +26 -0
- package/deps/librdkafka/STATISTICS.md +624 -0
- package/deps/librdkafka/configure +214 -0
- package/deps/librdkafka/configure.self +341 -0
- package/deps/librdkafka/dev-conf.sh +123 -0
- package/deps/librdkafka/lds-gen.py +73 -0
- package/deps/librdkafka/mainpage.doxy +40 -0
- package/deps/librdkafka/mklove/Makefile.base +329 -0
- package/deps/librdkafka/mklove/modules/configure.atomics +144 -0
- package/deps/librdkafka/mklove/modules/configure.base +2484 -0
- package/deps/librdkafka/mklove/modules/configure.builtin +70 -0
- package/deps/librdkafka/mklove/modules/configure.cc +186 -0
- package/deps/librdkafka/mklove/modules/configure.cxx +8 -0
- package/deps/librdkafka/mklove/modules/configure.fileversion +65 -0
- package/deps/librdkafka/mklove/modules/configure.gitversion +29 -0
- package/deps/librdkafka/mklove/modules/configure.good_cflags +18 -0
- package/deps/librdkafka/mklove/modules/configure.host +132 -0
- package/deps/librdkafka/mklove/modules/configure.lib +49 -0
- package/deps/librdkafka/mklove/modules/configure.libcurl +99 -0
- package/deps/librdkafka/mklove/modules/configure.libsasl2 +36 -0
- package/deps/librdkafka/mklove/modules/configure.libssl +147 -0
- package/deps/librdkafka/mklove/modules/configure.libzstd +58 -0
- package/deps/librdkafka/mklove/modules/configure.parseversion +95 -0
- package/deps/librdkafka/mklove/modules/configure.pic +16 -0
- package/deps/librdkafka/mklove/modules/configure.socket +20 -0
- package/deps/librdkafka/mklove/modules/configure.zlib +61 -0
- package/deps/librdkafka/mklove/modules/patches/README.md +8 -0
- package/deps/librdkafka/mklove/modules/patches/libcurl.0000-no-runtime-linking-check.patch +11 -0
- package/deps/librdkafka/mklove/modules/patches/libssl.0000-osx-rand-include-fix-OpenSSL-PR16409.patch +56 -0
- package/deps/librdkafka/service.yml +182 -0
- package/deps/librdkafka/src/CMakeLists.txt +379 -0
- package/deps/librdkafka/src/Makefile +105 -0
- package/deps/librdkafka/src/README.lz4.md +30 -0
- package/deps/librdkafka/src/cJSON.c +3191 -0
- package/deps/librdkafka/src/cJSON.h +306 -0
- package/deps/librdkafka/src/crc32c.c +430 -0
- package/deps/librdkafka/src/crc32c.h +38 -0
- package/deps/librdkafka/src/generate_proto.sh +66 -0
- package/deps/librdkafka/src/librdkafka_cgrp_synch.png +0 -0
- package/deps/librdkafka/src/lz4.c +2727 -0
- package/deps/librdkafka/src/lz4.h +842 -0
- package/deps/librdkafka/src/lz4frame.c +2078 -0
- package/deps/librdkafka/src/lz4frame.h +692 -0
- package/deps/librdkafka/src/lz4frame_static.h +47 -0
- package/deps/librdkafka/src/lz4hc.c +1631 -0
- package/deps/librdkafka/src/lz4hc.h +413 -0
- package/deps/librdkafka/src/nanopb/pb.h +917 -0
- package/deps/librdkafka/src/nanopb/pb_common.c +388 -0
- package/deps/librdkafka/src/nanopb/pb_common.h +49 -0
- package/deps/librdkafka/src/nanopb/pb_decode.c +1727 -0
- package/deps/librdkafka/src/nanopb/pb_decode.h +193 -0
- package/deps/librdkafka/src/nanopb/pb_encode.c +1000 -0
- package/deps/librdkafka/src/nanopb/pb_encode.h +185 -0
- package/deps/librdkafka/src/opentelemetry/common.pb.c +32 -0
- package/deps/librdkafka/src/opentelemetry/common.pb.h +170 -0
- package/deps/librdkafka/src/opentelemetry/metrics.options +2 -0
- package/deps/librdkafka/src/opentelemetry/metrics.pb.c +67 -0
- package/deps/librdkafka/src/opentelemetry/metrics.pb.h +966 -0
- package/deps/librdkafka/src/opentelemetry/resource.pb.c +12 -0
- package/deps/librdkafka/src/opentelemetry/resource.pb.h +58 -0
- package/deps/librdkafka/src/queue.h +850 -0
- package/deps/librdkafka/src/rd.h +592 -0
- package/deps/librdkafka/src/rdaddr.c +255 -0
- package/deps/librdkafka/src/rdaddr.h +202 -0
- package/deps/librdkafka/src/rdatomic.h +262 -0
- package/deps/librdkafka/src/rdavg.h +260 -0
- package/deps/librdkafka/src/rdavl.c +210 -0
- package/deps/librdkafka/src/rdavl.h +250 -0
- package/deps/librdkafka/src/rdbase64.c +200 -0
- package/deps/librdkafka/src/rdbase64.h +43 -0
- package/deps/librdkafka/src/rdbuf.c +1884 -0
- package/deps/librdkafka/src/rdbuf.h +375 -0
- package/deps/librdkafka/src/rdcrc32.c +114 -0
- package/deps/librdkafka/src/rdcrc32.h +170 -0
- package/deps/librdkafka/src/rddl.c +179 -0
- package/deps/librdkafka/src/rddl.h +43 -0
- package/deps/librdkafka/src/rdendian.h +175 -0
- package/deps/librdkafka/src/rdfloat.h +67 -0
- package/deps/librdkafka/src/rdfnv1a.c +113 -0
- package/deps/librdkafka/src/rdfnv1a.h +35 -0
- package/deps/librdkafka/src/rdgz.c +120 -0
- package/deps/librdkafka/src/rdgz.h +46 -0
- package/deps/librdkafka/src/rdhdrhistogram.c +721 -0
- package/deps/librdkafka/src/rdhdrhistogram.h +87 -0
- package/deps/librdkafka/src/rdhttp.c +831 -0
- package/deps/librdkafka/src/rdhttp.h +101 -0
- package/deps/librdkafka/src/rdinterval.h +177 -0
- package/deps/librdkafka/src/rdkafka.c +7287 -0
- package/deps/librdkafka/src/rdkafka.h +11795 -0
- package/deps/librdkafka/src/rdkafka_admin.c +10165 -0
- package/deps/librdkafka/src/rdkafka_admin.h +658 -0
- package/deps/librdkafka/src/rdkafka_assignment.c +1282 -0
- package/deps/librdkafka/src/rdkafka_assignment.h +73 -0
- package/deps/librdkafka/src/rdkafka_assignor.c +1800 -0
- package/deps/librdkafka/src/rdkafka_assignor.h +402 -0
- package/deps/librdkafka/src/rdkafka_aux.c +409 -0
- package/deps/librdkafka/src/rdkafka_aux.h +174 -0
- package/deps/librdkafka/src/rdkafka_background.c +222 -0
- package/deps/librdkafka/src/rdkafka_broker.c +6736 -0
- package/deps/librdkafka/src/rdkafka_broker.h +851 -0
- package/deps/librdkafka/src/rdkafka_buf.c +543 -0
- package/deps/librdkafka/src/rdkafka_buf.h +1525 -0
- package/deps/librdkafka/src/rdkafka_cert.c +576 -0
- package/deps/librdkafka/src/rdkafka_cert.h +62 -0
- package/deps/librdkafka/src/rdkafka_cgrp.c +8310 -0
- package/deps/librdkafka/src/rdkafka_cgrp.h +589 -0
- package/deps/librdkafka/src/rdkafka_conf.c +5177 -0
- package/deps/librdkafka/src/rdkafka_conf.h +746 -0
- package/deps/librdkafka/src/rdkafka_confval.h +97 -0
- package/deps/librdkafka/src/rdkafka_coord.c +623 -0
- package/deps/librdkafka/src/rdkafka_coord.h +132 -0
- package/deps/librdkafka/src/rdkafka_error.c +228 -0
- package/deps/librdkafka/src/rdkafka_error.h +80 -0
- package/deps/librdkafka/src/rdkafka_event.c +502 -0
- package/deps/librdkafka/src/rdkafka_event.h +126 -0
- package/deps/librdkafka/src/rdkafka_feature.c +898 -0
- package/deps/librdkafka/src/rdkafka_feature.h +104 -0
- package/deps/librdkafka/src/rdkafka_fetcher.c +3820 -0
- package/deps/librdkafka/src/rdkafka_fetcher.h +62 -0
- package/deps/librdkafka/src/rdkafka_header.c +220 -0
- package/deps/librdkafka/src/rdkafka_header.h +76 -0
- package/deps/librdkafka/src/rdkafka_idempotence.c +807 -0
- package/deps/librdkafka/src/rdkafka_idempotence.h +144 -0
- package/deps/librdkafka/src/rdkafka_int.h +1505 -0
- package/deps/librdkafka/src/rdkafka_interceptor.c +842 -0
- package/deps/librdkafka/src/rdkafka_interceptor.h +104 -0
- package/deps/librdkafka/src/rdkafka_lz4.c +450 -0
- package/deps/librdkafka/src/rdkafka_lz4.h +49 -0
- package/deps/librdkafka/src/rdkafka_metadata.c +2347 -0
- package/deps/librdkafka/src/rdkafka_metadata.h +350 -0
- package/deps/librdkafka/src/rdkafka_metadata_cache.c +1183 -0
- package/deps/librdkafka/src/rdkafka_mock.c +4514 -0
- package/deps/librdkafka/src/rdkafka_mock.h +867 -0
- package/deps/librdkafka/src/rdkafka_mock_cgrp.c +1877 -0
- package/deps/librdkafka/src/rdkafka_mock_handlers.c +5255 -0
- package/deps/librdkafka/src/rdkafka_mock_int.h +1005 -0
- package/deps/librdkafka/src/rdkafka_mock_sharegrp.c +1207 -0
- package/deps/librdkafka/src/rdkafka_msg.c +2619 -0
- package/deps/librdkafka/src/rdkafka_msg.h +629 -0
- package/deps/librdkafka/src/rdkafka_msgbatch.h +62 -0
- package/deps/librdkafka/src/rdkafka_msgset.h +105 -0
- package/deps/librdkafka/src/rdkafka_msgset_reader.c +2099 -0
- package/deps/librdkafka/src/rdkafka_msgset_writer.c +1474 -0
- package/deps/librdkafka/src/rdkafka_offset.c +1565 -0
- package/deps/librdkafka/src/rdkafka_offset.h +150 -0
- package/deps/librdkafka/src/rdkafka_op.c +1219 -0
- package/deps/librdkafka/src/rdkafka_op.h +1065 -0
- package/deps/librdkafka/src/rdkafka_partition.c +5109 -0
- package/deps/librdkafka/src/rdkafka_partition.h +1215 -0
- package/deps/librdkafka/src/rdkafka_pattern.c +228 -0
- package/deps/librdkafka/src/rdkafka_pattern.h +70 -0
- package/deps/librdkafka/src/rdkafka_plugin.c +213 -0
- package/deps/librdkafka/src/rdkafka_plugin.h +41 -0
- package/deps/librdkafka/src/rdkafka_proto.h +739 -0
- package/deps/librdkafka/src/rdkafka_protocol.h +131 -0
- package/deps/librdkafka/src/rdkafka_queue.c +1351 -0
- package/deps/librdkafka/src/rdkafka_queue.h +1235 -0
- package/deps/librdkafka/src/rdkafka_range_assignor.c +1748 -0
- package/deps/librdkafka/src/rdkafka_request.c +7298 -0
- package/deps/librdkafka/src/rdkafka_request.h +761 -0
- package/deps/librdkafka/src/rdkafka_roundrobin_assignor.c +123 -0
- package/deps/librdkafka/src/rdkafka_sasl.c +589 -0
- package/deps/librdkafka/src/rdkafka_sasl.h +63 -0
- package/deps/librdkafka/src/rdkafka_sasl_cyrus.c +722 -0
- package/deps/librdkafka/src/rdkafka_sasl_int.h +89 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer.c +1833 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer.h +52 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer_oidc.c +1827 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer_oidc.h +52 -0
- package/deps/librdkafka/src/rdkafka_sasl_plain.c +142 -0
- package/deps/librdkafka/src/rdkafka_sasl_scram.c +858 -0
- package/deps/librdkafka/src/rdkafka_sasl_win32.c +550 -0
- package/deps/librdkafka/src/rdkafka_share_acknowledgement.c +1445 -0
- package/deps/librdkafka/src/rdkafka_share_acknowledgement.h +373 -0
- package/deps/librdkafka/src/rdkafka_ssl.c +2235 -0
- package/deps/librdkafka/src/rdkafka_ssl.h +86 -0
- package/deps/librdkafka/src/rdkafka_sticky_assignor.c +4785 -0
- package/deps/librdkafka/src/rdkafka_subscription.c +411 -0
- package/deps/librdkafka/src/rdkafka_telemetry.c +761 -0
- package/deps/librdkafka/src/rdkafka_telemetry.h +52 -0
- package/deps/librdkafka/src/rdkafka_telemetry_decode.c +1307 -0
- package/deps/librdkafka/src/rdkafka_telemetry_decode.h +59 -0
- package/deps/librdkafka/src/rdkafka_telemetry_encode.c +1309 -0
- package/deps/librdkafka/src/rdkafka_telemetry_encode.h +459 -0
- package/deps/librdkafka/src/rdkafka_timer.c +407 -0
- package/deps/librdkafka/src/rdkafka_timer.h +117 -0
- package/deps/librdkafka/src/rdkafka_topic.c +2382 -0
- package/deps/librdkafka/src/rdkafka_topic.h +369 -0
- package/deps/librdkafka/src/rdkafka_transport.c +1309 -0
- package/deps/librdkafka/src/rdkafka_transport.h +99 -0
- package/deps/librdkafka/src/rdkafka_transport_int.h +100 -0
- package/deps/librdkafka/src/rdkafka_txnmgr.c +3256 -0
- package/deps/librdkafka/src/rdkafka_txnmgr.h +171 -0
- package/deps/librdkafka/src/rdkafka_zstd.c +226 -0
- package/deps/librdkafka/src/rdkafka_zstd.h +57 -0
- package/deps/librdkafka/src/rdlist.c +593 -0
- package/deps/librdkafka/src/rdlist.h +444 -0
- package/deps/librdkafka/src/rdlog.c +89 -0
- package/deps/librdkafka/src/rdlog.h +41 -0
- package/deps/librdkafka/src/rdmap.c +508 -0
- package/deps/librdkafka/src/rdmap.h +492 -0
- package/deps/librdkafka/src/rdmurmur2.c +167 -0
- package/deps/librdkafka/src/rdmurmur2.h +35 -0
- package/deps/librdkafka/src/rdports.c +61 -0
- package/deps/librdkafka/src/rdports.h +38 -0
- package/deps/librdkafka/src/rdposix.h +250 -0
- package/deps/librdkafka/src/rdrand.c +151 -0
- package/deps/librdkafka/src/rdrand.h +72 -0
- package/deps/librdkafka/src/rdregex.c +156 -0
- package/deps/librdkafka/src/rdregex.h +43 -0
- package/deps/librdkafka/src/rdsignal.h +57 -0
- package/deps/librdkafka/src/rdstring.c +645 -0
- package/deps/librdkafka/src/rdstring.h +98 -0
- package/deps/librdkafka/src/rdsysqueue.h +404 -0
- package/deps/librdkafka/src/rdtime.h +356 -0
- package/deps/librdkafka/src/rdtypes.h +86 -0
- package/deps/librdkafka/src/rdunittest.c +564 -0
- package/deps/librdkafka/src/rdunittest.h +232 -0
- package/deps/librdkafka/src/rdunittest_acknowledge.c +1177 -0
- package/deps/librdkafka/src/rdunittest_fetcher.c +1211 -0
- package/deps/librdkafka/src/rdunittest_msgset_errors.c +1521 -0
- package/deps/librdkafka/src/rdvarint.c +134 -0
- package/deps/librdkafka/src/rdvarint.h +165 -0
- package/deps/librdkafka/src/rdwin32.h +382 -0
- package/deps/librdkafka/src/rdxxhash.c +1030 -0
- package/deps/librdkafka/src/rdxxhash.h +328 -0
- package/deps/librdkafka/src/regexp.c +1352 -0
- package/deps/librdkafka/src/regexp.h +41 -0
- package/deps/librdkafka/src/snappy.c +1866 -0
- package/deps/librdkafka/src/snappy.h +62 -0
- package/deps/librdkafka/src/snappy_compat.h +138 -0
- package/deps/librdkafka/src/statistics_schema.json +444 -0
- package/deps/librdkafka/src/tinycthread.c +932 -0
- package/deps/librdkafka/src/tinycthread.h +503 -0
- package/deps/librdkafka/src/tinycthread_extra.c +210 -0
- package/deps/librdkafka/src/tinycthread_extra.h +218 -0
- package/deps/librdkafka/src/win32_config.h +58 -0
- package/deps/librdkafka/src-cpp/CMakeLists.txt +90 -0
- package/deps/librdkafka/src-cpp/ConfImpl.cpp +84 -0
- package/deps/librdkafka/src-cpp/ConsumerImpl.cpp +244 -0
- package/deps/librdkafka/src-cpp/HandleImpl.cpp +436 -0
- package/deps/librdkafka/src-cpp/HeadersImpl.cpp +48 -0
- package/deps/librdkafka/src-cpp/KafkaConsumerImpl.cpp +296 -0
- package/deps/librdkafka/src-cpp/Makefile +55 -0
- package/deps/librdkafka/src-cpp/MessageImpl.cpp +38 -0
- package/deps/librdkafka/src-cpp/MetadataImpl.cpp +170 -0
- package/deps/librdkafka/src-cpp/ProducerImpl.cpp +197 -0
- package/deps/librdkafka/src-cpp/QueueImpl.cpp +70 -0
- package/deps/librdkafka/src-cpp/README.md +16 -0
- package/deps/librdkafka/src-cpp/RdKafka.cpp +59 -0
- package/deps/librdkafka/src-cpp/TopicImpl.cpp +124 -0
- package/deps/librdkafka/src-cpp/TopicPartitionImpl.cpp +57 -0
- package/deps/librdkafka/src-cpp/rdkafkacpp.h +3797 -0
- package/deps/librdkafka/src-cpp/rdkafkacpp_int.h +1641 -0
- package/deps/librdkafka/tests/0000-unittests.c +72 -0
- package/deps/librdkafka/tests/0001-multiobj.c +102 -0
- package/deps/librdkafka/tests/0002-unkpart.c +244 -0
- package/deps/librdkafka/tests/0003-msgmaxsize.c +173 -0
- package/deps/librdkafka/tests/0004-conf.c +934 -0
- package/deps/librdkafka/tests/0005-order.c +133 -0
- package/deps/librdkafka/tests/0006-symbols.c +163 -0
- package/deps/librdkafka/tests/0007-autotopic.c +136 -0
- package/deps/librdkafka/tests/0008-reqacks.c +179 -0
- package/deps/librdkafka/tests/0009-mock_cluster.c +272 -0
- package/deps/librdkafka/tests/0011-produce_batch.c +753 -0
- package/deps/librdkafka/tests/0012-produce_consume.c +537 -0
- package/deps/librdkafka/tests/0013-null-msgs.c +473 -0
- package/deps/librdkafka/tests/0014-reconsume-191.c +512 -0
- package/deps/librdkafka/tests/0015-offset_seeks.c +172 -0
- package/deps/librdkafka/tests/0016-client_swname.c +181 -0
- package/deps/librdkafka/tests/0017-compression.c +140 -0
- package/deps/librdkafka/tests/0018-cgrp_term.c +444 -0
- package/deps/librdkafka/tests/0019-list_groups.c +289 -0
- package/deps/librdkafka/tests/0020-destroy_hang.c +162 -0
- package/deps/librdkafka/tests/0021-rkt_destroy.c +72 -0
- package/deps/librdkafka/tests/0022-consume_batch.c +331 -0
- package/deps/librdkafka/tests/0025-timers.c +147 -0
- package/deps/librdkafka/tests/0026-consume_pause.c +547 -0
- package/deps/librdkafka/tests/0028-long_topicnames.c +79 -0
- package/deps/librdkafka/tests/0029-assign_offset.c +202 -0
- package/deps/librdkafka/tests/0030-offset_commit.c +589 -0
- package/deps/librdkafka/tests/0031-get_offsets.c +235 -0
- package/deps/librdkafka/tests/0033-regex_subscribe.c +536 -0
- package/deps/librdkafka/tests/0034-offset_reset.c +398 -0
- package/deps/librdkafka/tests/0035-api_version.c +73 -0
- package/deps/librdkafka/tests/0036-partial_fetch.c +87 -0
- package/deps/librdkafka/tests/0037-destroy_hang_local.c +85 -0
- package/deps/librdkafka/tests/0038-performance.c +121 -0
- package/deps/librdkafka/tests/0039-event.c +284 -0
- package/deps/librdkafka/tests/0040-io_event.c +257 -0
- package/deps/librdkafka/tests/0041-fetch_max_bytes.c +97 -0
- package/deps/librdkafka/tests/0042-many_topics.c +252 -0
- package/deps/librdkafka/tests/0043-no_connection.c +77 -0
- package/deps/librdkafka/tests/0044-partition_cnt.c +94 -0
- package/deps/librdkafka/tests/0045-subscribe_update.c +1010 -0
- package/deps/librdkafka/tests/0046-rkt_cache.c +65 -0
- package/deps/librdkafka/tests/0047-partial_buf_tmout.c +98 -0
- package/deps/librdkafka/tests/0048-partitioner.c +283 -0
- package/deps/librdkafka/tests/0049-consume_conn_close.c +162 -0
- package/deps/librdkafka/tests/0050-subscribe_adds.c +145 -0
- package/deps/librdkafka/tests/0051-assign_adds.c +126 -0
- package/deps/librdkafka/tests/0052-msg_timestamps.c +238 -0
- package/deps/librdkafka/tests/0053-stats_cb.cpp +527 -0
- package/deps/librdkafka/tests/0054-offset_time.cpp +236 -0
- package/deps/librdkafka/tests/0055-producer_latency.c +539 -0
- package/deps/librdkafka/tests/0056-balanced_group_mt.c +315 -0
- package/deps/librdkafka/tests/0057-invalid_topic.cpp +112 -0
- package/deps/librdkafka/tests/0058-log.cpp +123 -0
- package/deps/librdkafka/tests/0059-bsearch.cpp +241 -0
- package/deps/librdkafka/tests/0060-op_prio.cpp +163 -0
- package/deps/librdkafka/tests/0061-consumer_lag.cpp +295 -0
- package/deps/librdkafka/tests/0062-stats_event.c +126 -0
- package/deps/librdkafka/tests/0063-clusterid.cpp +180 -0
- package/deps/librdkafka/tests/0064-interceptors.c +529 -0
- package/deps/librdkafka/tests/0065-yield.cpp +140 -0
- package/deps/librdkafka/tests/0066-plugins.cpp +143 -0
- package/deps/librdkafka/tests/0067-empty_topic.cpp +151 -0
- package/deps/librdkafka/tests/0068-produce_timeout.c +136 -0
- package/deps/librdkafka/tests/0069-consumer_add_parts.c +119 -0
- package/deps/librdkafka/tests/0070-null_empty.cpp +197 -0
- package/deps/librdkafka/tests/0072-headers_ut.c +448 -0
- package/deps/librdkafka/tests/0073-headers.c +381 -0
- package/deps/librdkafka/tests/0074-producev.c +87 -0
- package/deps/librdkafka/tests/0075-retry.c +290 -0
- package/deps/librdkafka/tests/0076-produce_retry.c +452 -0
- package/deps/librdkafka/tests/0077-compaction.c +366 -0
- package/deps/librdkafka/tests/0078-c_from_cpp.cpp +96 -0
- package/deps/librdkafka/tests/0079-fork.c +93 -0
- package/deps/librdkafka/tests/0080-admin_ut.c +3095 -0
- package/deps/librdkafka/tests/0081-admin.c +5633 -0
- package/deps/librdkafka/tests/0082-fetch_max_bytes.cpp +137 -0
- package/deps/librdkafka/tests/0083-cb_event.c +233 -0
- package/deps/librdkafka/tests/0084-destroy_flags.c +208 -0
- package/deps/librdkafka/tests/0085-headers.cpp +392 -0
- package/deps/librdkafka/tests/0086-purge.c +368 -0
- package/deps/librdkafka/tests/0088-produce_metadata_timeout.c +162 -0
- package/deps/librdkafka/tests/0089-max_poll_interval.c +511 -0
- package/deps/librdkafka/tests/0090-idempotence.c +171 -0
- package/deps/librdkafka/tests/0091-max_poll_interval_timeout.c +295 -0
- package/deps/librdkafka/tests/0092-mixed_msgver.c +103 -0
- package/deps/librdkafka/tests/0093-holb.c +200 -0
- package/deps/librdkafka/tests/0094-idempotence_msg_timeout.c +231 -0
- package/deps/librdkafka/tests/0095-all_brokers_down.cpp +122 -0
- package/deps/librdkafka/tests/0097-ssl_verify.cpp +674 -0
- package/deps/librdkafka/tests/0098-consumer-txn.cpp +1218 -0
- package/deps/librdkafka/tests/0099-commit_metadata.c +194 -0
- package/deps/librdkafka/tests/0100-thread_interceptors.cpp +195 -0
- package/deps/librdkafka/tests/0101-fetch-from-follower.cpp +446 -0
- package/deps/librdkafka/tests/0102-static_group_rebalance.c +836 -0
- package/deps/librdkafka/tests/0103-transactions.c +1383 -0
- package/deps/librdkafka/tests/0104-fetch_from_follower_mock.c +625 -0
- package/deps/librdkafka/tests/0105-transactions_mock.c +3930 -0
- package/deps/librdkafka/tests/0106-cgrp_sess_timeout.c +318 -0
- package/deps/librdkafka/tests/0107-topic_recreate.c +259 -0
- package/deps/librdkafka/tests/0109-auto_create_topics.cpp +278 -0
- package/deps/librdkafka/tests/0110-batch_size.cpp +182 -0
- package/deps/librdkafka/tests/0111-delay_create_topics.cpp +127 -0
- package/deps/librdkafka/tests/0112-assign_unknown_part.c +87 -0
- package/deps/librdkafka/tests/0113-cooperative_rebalance.cpp +3473 -0
- package/deps/librdkafka/tests/0114-sticky_partitioning.cpp +176 -0
- package/deps/librdkafka/tests/0115-producer_auth.cpp +182 -0
- package/deps/librdkafka/tests/0116-kafkaconsumer_close.cpp +216 -0
- package/deps/librdkafka/tests/0117-mock_errors.c +331 -0
- package/deps/librdkafka/tests/0118-commit_rebalance.c +154 -0
- package/deps/librdkafka/tests/0119-consumer_auth.cpp +167 -0
- package/deps/librdkafka/tests/0120-asymmetric_subscription.c +185 -0
- package/deps/librdkafka/tests/0121-clusterid.c +115 -0
- package/deps/librdkafka/tests/0122-buffer_cleaning_after_rebalance.c +227 -0
- package/deps/librdkafka/tests/0123-connections_max_idle.c +98 -0
- package/deps/librdkafka/tests/0124-openssl_invalid_engine.c +69 -0
- package/deps/librdkafka/tests/0125-immediate_flush.c +144 -0
- package/deps/librdkafka/tests/0126-oauthbearer_oidc.c +1164 -0
- package/deps/librdkafka/tests/0127-fetch_queue_backoff.cpp +165 -0
- package/deps/librdkafka/tests/0128-sasl_callback_queue.cpp +217 -0
- package/deps/librdkafka/tests/0129-fetch_aborted_msgs.c +79 -0
- package/deps/librdkafka/tests/0130-store_offsets.c +178 -0
- package/deps/librdkafka/tests/0131-connect_timeout.c +81 -0
- package/deps/librdkafka/tests/0132-strategy_ordering.c +179 -0
- package/deps/librdkafka/tests/0133-ssl_keys.c +150 -0
- package/deps/librdkafka/tests/0134-ssl_provider.c +92 -0
- package/deps/librdkafka/tests/0135-sasl_credentials.cpp +279 -0
- package/deps/librdkafka/tests/0136-resolve_cb.c +181 -0
- package/deps/librdkafka/tests/0137-barrier_batch_consume.c +752 -0
- package/deps/librdkafka/tests/0138-admin_mock.c +281 -0
- package/deps/librdkafka/tests/0139-offset_validation_mock.c +950 -0
- package/deps/librdkafka/tests/0140-commit_metadata.cpp +108 -0
- package/deps/librdkafka/tests/0142-reauthentication.c +931 -0
- package/deps/librdkafka/tests/0143-exponential_backoff_mock.c +552 -0
- package/deps/librdkafka/tests/0144-idempotence_mock.c +373 -0
- package/deps/librdkafka/tests/0145-pause_resume_mock.c +119 -0
- package/deps/librdkafka/tests/0146-metadata_mock.c +505 -0
- package/deps/librdkafka/tests/0147-consumer_group_consumer_mock.c +1210 -0
- package/deps/librdkafka/tests/0148-offset_fetch_commit_error_mock.c +569 -0
- package/deps/librdkafka/tests/0149-broker-same-host-port.c +140 -0
- package/deps/librdkafka/tests/0150-telemetry_mock.c +651 -0
- package/deps/librdkafka/tests/0151-purge-brokers.c +566 -0
- package/deps/librdkafka/tests/0152-rebootstrap.c +59 -0
- package/deps/librdkafka/tests/0153-memberid.c +128 -0
- package/deps/librdkafka/tests/0155-share_group_heartbeat_mock.c +2695 -0
- package/deps/librdkafka/tests/0156-share_consumer_fetch_mock.c +1551 -0
- package/deps/librdkafka/tests/0157-share_consumer_ack_mock.c +1788 -0
- package/deps/librdkafka/tests/0158-share_consumer_transactions_mock.c +467 -0
- package/deps/librdkafka/tests/0170-share_consumer_subscription.c +2192 -0
- package/deps/librdkafka/tests/0171-share_consumer_consume.c +2150 -0
- package/deps/librdkafka/tests/0172-share_consumer_acknowledge.c +1919 -0
- package/deps/librdkafka/tests/0173-share_consumer_commit_async.c +2735 -0
- package/deps/librdkafka/tests/0174-share_consumer_concurrency.c +1434 -0
- package/deps/librdkafka/tests/0175-share_consumer_groups.c +574 -0
- package/deps/librdkafka/tests/0176-share_consumer_commit_sync.c +2986 -0
- package/deps/librdkafka/tests/0177-share_consumer_transactions.c +1815 -0
- package/deps/librdkafka/tests/0178-share_consumer_close.c +2102 -0
- package/deps/librdkafka/tests/0179-share_consumer_destroy.c +2482 -0
- package/deps/librdkafka/tests/0180-share_consumer_config.c +1677 -0
- package/deps/librdkafka/tests/0181-share_consumer_topic_delete.c +354 -0
- package/deps/librdkafka/tests/0182-share_consumer_error_handling_mock.c +3358 -0
- package/deps/librdkafka/tests/0183-share_consumer_leader_change_mock.c +1234 -0
- package/deps/librdkafka/tests/0184-share_consumer_topic_recreate.c +1356 -0
- package/deps/librdkafka/tests/0185-share_consumer_max_poll_interval.c +557 -0
- package/deps/librdkafka/tests/0186-share_consumer_fatal_error.c +360 -0
- package/deps/librdkafka/tests/0190-share_consumer_telemetry.c +387 -0
- package/deps/librdkafka/tests/1000-unktopic.c +164 -0
- package/deps/librdkafka/tests/8000-idle.cpp +60 -0
- package/deps/librdkafka/tests/8001-fetch_from_follower_mock_manual.c +113 -0
- package/deps/librdkafka/tests/CMakeLists.txt +193 -0
- package/deps/librdkafka/tests/LibrdkafkaTestApp.py +291 -0
- package/deps/librdkafka/tests/Makefile +182 -0
- package/deps/librdkafka/tests/README.md +514 -0
- package/deps/librdkafka/tests/autotest.sh +33 -0
- package/deps/librdkafka/tests/backtrace.gdb +30 -0
- package/deps/librdkafka/tests/broker_version_tests.py +315 -0
- package/deps/librdkafka/tests/buildbox.sh +17 -0
- package/deps/librdkafka/tests/cleanup-checker-tests.sh +20 -0
- package/deps/librdkafka/tests/cluster_testing.py +203 -0
- package/deps/librdkafka/tests/delete-test-topics.sh +56 -0
- package/deps/librdkafka/tests/fixtures/oauthbearer/jwt_assertion_template.json +10 -0
- package/deps/librdkafka/tests/fixtures/ssl/Makefile +8 -0
- package/deps/librdkafka/tests/fixtures/ssl/README.md +13 -0
- package/deps/librdkafka/tests/fixtures/ssl/client.keystore.intermediate.p12 +0 -0
- package/deps/librdkafka/tests/fixtures/ssl/client.keystore.p12 +0 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.certificate.intermediate.pem +72 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.certificate.pem +50 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.intermediate.key +46 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.key +46 -0
- package/deps/librdkafka/tests/fixtures/ssl/create_keys.sh +168 -0
- package/deps/librdkafka/tests/fuzzers/Makefile +12 -0
- package/deps/librdkafka/tests/fuzzers/README.md +31 -0
- package/deps/librdkafka/tests/fuzzers/fuzz_regex.c +74 -0
- package/deps/librdkafka/tests/fuzzers/helpers.h +90 -0
- package/deps/librdkafka/tests/gen-ssl-certs.sh +165 -0
- package/deps/librdkafka/tests/interactive_broker_version.py +170 -0
- package/deps/librdkafka/tests/interceptor_test/CMakeLists.txt +16 -0
- package/deps/librdkafka/tests/interceptor_test/Makefile +22 -0
- package/deps/librdkafka/tests/interceptor_test/interceptor_test.c +314 -0
- package/deps/librdkafka/tests/interceptor_test/interceptor_test.h +54 -0
- package/deps/librdkafka/tests/java/IncrementalRebalanceCli.java +97 -0
- package/deps/librdkafka/tests/java/Makefile +13 -0
- package/deps/librdkafka/tests/java/Murmur2Cli.java +46 -0
- package/deps/librdkafka/tests/java/README.md +14 -0
- package/deps/librdkafka/tests/java/TransactionProducerCli.java +162 -0
- package/deps/librdkafka/tests/java/run-class.sh +11 -0
- package/deps/librdkafka/tests/librdkafka.suppressions +483 -0
- package/deps/librdkafka/tests/lz4_manual_test.sh +59 -0
- package/deps/librdkafka/tests/multi-broker-version-test.sh +50 -0
- package/deps/librdkafka/tests/parse-refcnt.sh +43 -0
- package/deps/librdkafka/tests/performance_plot.py +115 -0
- package/deps/librdkafka/tests/plugin_test/Makefile +19 -0
- package/deps/librdkafka/tests/plugin_test/plugin_test.c +58 -0
- package/deps/librdkafka/tests/requirements.txt +2 -0
- package/deps/librdkafka/tests/run-all-tests.sh +81 -0
- package/deps/librdkafka/tests/run-consumer-tests.sh +16 -0
- package/deps/librdkafka/tests/run-producer-tests.sh +16 -0
- package/deps/librdkafka/tests/run-test-batches.py +157 -0
- package/deps/librdkafka/tests/run-test.sh +141 -0
- package/deps/librdkafka/tests/rusage.c +249 -0
- package/deps/librdkafka/tests/sasl_test.py +289 -0
- package/deps/librdkafka/tests/scenarios/README.md +6 -0
- package/deps/librdkafka/tests/scenarios/ak23.json +6 -0
- package/deps/librdkafka/tests/scenarios/default.json +5 -0
- package/deps/librdkafka/tests/scenarios/noautocreate.json +5 -0
- package/deps/librdkafka/tests/share_telemetry_verify/share-otel-collector-config.yaml +28 -0
- package/deps/librdkafka/tests/sockem.c +801 -0
- package/deps/librdkafka/tests/sockem.h +85 -0
- package/deps/librdkafka/tests/sockem_ctrl.c +145 -0
- package/deps/librdkafka/tests/sockem_ctrl.h +61 -0
- package/deps/librdkafka/tests/test.c +8376 -0
- package/deps/librdkafka/tests/test.conf.example +27 -0
- package/deps/librdkafka/tests/test.h +1150 -0
- package/deps/librdkafka/tests/testcpp.cpp +131 -0
- package/deps/librdkafka/tests/testcpp.h +388 -0
- package/deps/librdkafka/tests/testshared.h +416 -0
- package/deps/librdkafka/tests/tools/README.md +4 -0
- package/deps/librdkafka/tests/tools/stats/README.md +21 -0
- package/deps/librdkafka/tests/tools/stats/filter.jq +42 -0
- package/deps/librdkafka/tests/tools/stats/graph.py +150 -0
- package/deps/librdkafka/tests/tools/stats/requirements.txt +3 -0
- package/deps/librdkafka/tests/tools/stats/to_csv.py +124 -0
- package/deps/librdkafka/tests/trivup/trivup-0.15.0.tar.gz +0 -0
- package/deps/librdkafka/tests/until-fail.sh +87 -0
- package/deps/librdkafka/tests/xxxx-assign_partition.c +122 -0
- package/deps/librdkafka/tests/xxxx-metadata.cpp +159 -0
- package/deps/librdkafka/vcpkg.json +23 -0
- package/deps/librdkafka/win32/README.md +5 -0
- package/deps/librdkafka/win32/build-package.bat +3 -0
- package/deps/librdkafka/win32/build.bat +19 -0
- package/deps/librdkafka/win32/common.vcxproj +84 -0
- package/deps/librdkafka/win32/interceptor_test/interceptor_test.vcxproj +87 -0
- package/deps/librdkafka/win32/librdkafka.autopkg.template +54 -0
- package/deps/librdkafka/win32/librdkafka.master.testing.targets +13 -0
- package/deps/librdkafka/win32/librdkafka.sln +226 -0
- package/deps/librdkafka/win32/librdkafka.vcxproj +282 -0
- package/deps/librdkafka/win32/librdkafkacpp/librdkafkacpp.vcxproj +104 -0
- package/deps/librdkafka/win32/msbuild.ps1 +58 -0
- package/deps/librdkafka/win32/openssl_engine_example/openssl_engine_example.vcxproj +132 -0
- package/deps/librdkafka/win32/package-zip.ps1 +46 -0
- package/deps/librdkafka/win32/packages/repositories.config +4 -0
- package/deps/librdkafka/win32/push-package.bat +4 -0
- package/deps/librdkafka/win32/rdkafka_complex_consumer_example_cpp/rdkafka_complex_consumer_example_cpp.vcxproj +67 -0
- package/deps/librdkafka/win32/rdkafka_example/rdkafka_example.vcxproj +97 -0
- package/deps/librdkafka/win32/rdkafka_performance/rdkafka_performance.vcxproj +97 -0
- package/deps/librdkafka/win32/setup-msys2.ps1 +47 -0
- package/deps/librdkafka/win32/setup-vcpkg.ps1 +34 -0
- package/deps/librdkafka/win32/tests/test.conf.example +25 -0
- package/deps/librdkafka/win32/tests/tests.vcxproj +278 -0
- package/deps/librdkafka/win32/win_ssl_cert_store/win_ssl_cert_store.vcxproj +132 -0
- package/deps/librdkafka/win32/wingetopt.c +564 -0
- package/deps/librdkafka/win32/wingetopt.h +101 -0
- package/deps/librdkafka/win32/wintime.h +33 -0
- package/deps/librdkafka.gyp +83 -0
- package/deps/windows-install.py +80 -0
- package/index.d.ts +3 -0
- package/lib/admin.js +732 -0
- package/lib/client.js +679 -0
- package/lib/error.js +501 -0
- package/lib/index.js +8 -0
- package/lib/kafka-consumer-stream.js +399 -0
- package/lib/kafka-consumer.js +798 -0
- package/lib/kafkajs/_admin.js +1052 -0
- package/lib/kafkajs/_common.js +887 -0
- package/lib/kafkajs/_consumer.js +2156 -0
- package/lib/kafkajs/_consumer_cache.js +310 -0
- package/lib/kafkajs/_error.js +298 -0
- package/lib/kafkajs/_kafka.js +125 -0
- package/lib/kafkajs/_linked-list.js +220 -0
- package/lib/kafkajs/_producer.js +863 -0
- package/lib/kafkajs/index.js +5 -0
- package/lib/producer/high-level-producer.js +400 -0
- package/lib/producer-stream.js +337 -0
- package/lib/producer.js +405 -0
- package/lib/rdkafka.js +44 -0
- package/lib/tools/ref-counter.js +53 -0
- package/lib/topic-partition.js +91 -0
- package/lib/topic.js +43 -0
- package/lib/util.js +55 -0
- package/librdkafka.js +12 -0
- package/package.json +85 -0
- package/shim/source_location +54 -0
- package/src/admin.cc +1597 -0
- package/src/admin.h +112 -0
- package/src/binding.cc +76 -0
- package/src/binding.h +25 -0
- package/src/callbacks.cc +676 -0
- package/src/callbacks.h +302 -0
- package/src/common.cc +1628 -0
- package/src/common.h +192 -0
- package/src/config.cc +219 -0
- package/src/config.h +61 -0
- package/src/connection.cc +703 -0
- package/src/connection.h +117 -0
- package/src/errors.cc +119 -0
- package/src/errors.h +60 -0
- package/src/kafka-consumer.cc +1539 -0
- package/src/kafka-consumer.h +147 -0
- package/src/producer.cc +917 -0
- package/src/producer.h +124 -0
- package/src/topic.cc +173 -0
- package/src/topic.h +54 -0
- package/src/workers.cc +1714 -0
- package/src/workers.h +690 -0
- package/types/config.d.ts +1269 -0
- package/types/errors.d.ts +373 -0
- package/types/kafkajs.d.ts +568 -0
- package/types/rdkafka.d.ts +571 -0
- package/util/configure.js +32 -0
- package/util/generate-docs.sh +18 -0
- package/util/get-env.js +6 -0
- package/util/test-compile.js +11 -0
- package/util/test-producer-delivery.js +100 -0
|
@@ -0,0 +1,3820 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* librdkafka - The Apache Kafka C/C++ library
|
|
3
|
+
*
|
|
4
|
+
* Copyright (c) 2022, Magnus Edenhill
|
|
5
|
+
* 2023, Confluent Inc.
|
|
6
|
+
* All rights reserved.
|
|
7
|
+
*
|
|
8
|
+
* Redistribution and use in source and binary forms, with or without
|
|
9
|
+
* modification, are permitted provided that the following conditions are met:
|
|
10
|
+
*
|
|
11
|
+
* 1. Redistributions of source code must retain the above copyright notice,
|
|
12
|
+
* this list of conditions and the following disclaimer.
|
|
13
|
+
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
14
|
+
* this list of conditions and the following disclaimer in the documentation
|
|
15
|
+
* and/or other materials provided with the distribution.
|
|
16
|
+
*
|
|
17
|
+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
18
|
+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
19
|
+
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
20
|
+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
|
21
|
+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
22
|
+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
23
|
+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
24
|
+
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
25
|
+
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
26
|
+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
27
|
+
* POSSIBILITY OF SUCH DAMAGE.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* @name Fetcher
|
|
33
|
+
*
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
#include "rdkafka_int.h"
|
|
37
|
+
#include "rdkafka_offset.h"
|
|
38
|
+
#include "rdkafka_msgset.h"
|
|
39
|
+
#include "rdkafka_fetcher.h"
|
|
40
|
+
#include "rdkafka_request.h"
|
|
41
|
+
#include "rdkafka_share_acknowledgement.h"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Backoff the next Fetch request (due to error).
|
|
46
|
+
*/
|
|
47
|
+
static void rd_kafka_broker_fetch_backoff(rd_kafka_broker_t *rkb,
|
|
48
|
+
rd_kafka_resp_err_t err) {
|
|
49
|
+
int backoff_ms = rkb->rkb_rk->rk_conf.fetch_error_backoff_ms;
|
|
50
|
+
rkb->rkb_ts_fetch_backoff = rd_clock() + (backoff_ms * 1000);
|
|
51
|
+
rd_rkb_dbg(rkb, FETCH, "BACKOFF", "Fetch backoff for %dms: %s",
|
|
52
|
+
backoff_ms, rd_kafka_err2str(err));
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* @brief Backoff the next Fetch for specific partition
|
|
57
|
+
*
|
|
58
|
+
* @returns the absolute backoff time (the current time for no backoff).
|
|
59
|
+
*/
|
|
60
|
+
static rd_ts_t rd_kafka_toppar_fetch_backoff(rd_kafka_broker_t *rkb,
|
|
61
|
+
rd_kafka_toppar_t *rktp,
|
|
62
|
+
rd_kafka_resp_err_t err) {
|
|
63
|
+
int backoff_ms;
|
|
64
|
+
|
|
65
|
+
/* Don't back off on reaching end of partition */
|
|
66
|
+
if (err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
|
|
67
|
+
rktp->rktp_ts_fetch_backoff = 0;
|
|
68
|
+
return rd_clock(); /* Immediate: No practical backoff */
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
if (err == RD_KAFKA_RESP_ERR__QUEUE_FULL)
|
|
72
|
+
backoff_ms = rkb->rkb_rk->rk_conf.fetch_queue_backoff_ms;
|
|
73
|
+
else
|
|
74
|
+
backoff_ms = rkb->rkb_rk->rk_conf.fetch_error_backoff_ms;
|
|
75
|
+
|
|
76
|
+
if (unlikely(!backoff_ms)) {
|
|
77
|
+
rktp->rktp_ts_fetch_backoff = 0;
|
|
78
|
+
return rd_clock(); /* Immediate: No practical backoff */
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/* Certain errors that may require manual intervention should have
|
|
82
|
+
* a longer backoff time. */
|
|
83
|
+
if (err == RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED)
|
|
84
|
+
backoff_ms = RD_MAX(1000, backoff_ms * 10);
|
|
85
|
+
|
|
86
|
+
rktp->rktp_ts_fetch_backoff = rd_clock() + (backoff_ms * 1000);
|
|
87
|
+
|
|
88
|
+
rd_rkb_dbg(rkb, FETCH, "BACKOFF",
|
|
89
|
+
"%s [%" PRId32 "]: Fetch backoff for %dms%s%s",
|
|
90
|
+
rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition,
|
|
91
|
+
backoff_ms, err ? ": " : "",
|
|
92
|
+
err ? rd_kafka_err2str(err) : "");
|
|
93
|
+
|
|
94
|
+
return rktp->rktp_ts_fetch_backoff;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* @brief Handle preferred replica in fetch response.
|
|
99
|
+
*
|
|
100
|
+
* @locks rd_kafka_toppar_lock(rktp) and
|
|
101
|
+
* rd_kafka_rdlock(rk) must NOT be held.
|
|
102
|
+
*
|
|
103
|
+
* @locality broker thread
|
|
104
|
+
*/
|
|
105
|
+
static void rd_kafka_fetch_preferred_replica_handle(rd_kafka_toppar_t *rktp,
|
|
106
|
+
rd_kafka_buf_t *rkbuf,
|
|
107
|
+
rd_kafka_broker_t *rkb,
|
|
108
|
+
int32_t preferred_id) {
|
|
109
|
+
const rd_ts_t one_minute = 60 * 1000 * 1000;
|
|
110
|
+
const rd_ts_t five_seconds = 5 * 1000 * 1000;
|
|
111
|
+
rd_kafka_broker_t *preferred_rkb;
|
|
112
|
+
rd_kafka_t *rk = rktp->rktp_rkt->rkt_rk;
|
|
113
|
+
rd_ts_t new_intvl =
|
|
114
|
+
rd_interval_immediate(&rktp->rktp_new_lease_intvl, one_minute, 0);
|
|
115
|
+
|
|
116
|
+
if (new_intvl < 0) {
|
|
117
|
+
/* In lieu of KIP-320, the toppar is delegated back to
|
|
118
|
+
* the leader in the event of an offset out-of-range
|
|
119
|
+
* error (KIP-392 error case #4) because this scenario
|
|
120
|
+
* implies the preferred replica is out-of-sync.
|
|
121
|
+
*
|
|
122
|
+
* If program execution reaches here, the leader has
|
|
123
|
+
* relatively quickly instructed the client back to
|
|
124
|
+
* a preferred replica, quite possibly the same one
|
|
125
|
+
* as before (possibly resulting from stale metadata),
|
|
126
|
+
* so we back off the toppar to slow down potential
|
|
127
|
+
* back-and-forth.
|
|
128
|
+
*/
|
|
129
|
+
|
|
130
|
+
if (rd_interval_immediate(&rktp->rktp_new_lease_log_intvl,
|
|
131
|
+
one_minute, 0) > 0)
|
|
132
|
+
rd_rkb_log(rkb, LOG_NOTICE, "FETCH",
|
|
133
|
+
"%.*s [%" PRId32
|
|
134
|
+
"]: preferred replica "
|
|
135
|
+
"(%" PRId32
|
|
136
|
+
") lease changing too quickly "
|
|
137
|
+
"(%" PRId64
|
|
138
|
+
"s < 60s): possibly due to "
|
|
139
|
+
"unavailable replica or stale cluster "
|
|
140
|
+
"state: backing off next fetch",
|
|
141
|
+
RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
|
|
142
|
+
rktp->rktp_partition, preferred_id,
|
|
143
|
+
(one_minute - -new_intvl) / (1000 * 1000));
|
|
144
|
+
|
|
145
|
+
rd_kafka_toppar_fetch_backoff(rkb, rktp,
|
|
146
|
+
RD_KAFKA_RESP_ERR_NO_ERROR);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
rd_kafka_rdlock(rk);
|
|
150
|
+
preferred_rkb = rd_kafka_broker_find_by_nodeid(rk, preferred_id);
|
|
151
|
+
rd_kafka_rdunlock(rk);
|
|
152
|
+
|
|
153
|
+
if (preferred_rkb) {
|
|
154
|
+
rd_interval_reset_to_now(&rktp->rktp_lease_intvl, 0);
|
|
155
|
+
rd_kafka_toppar_lock(rktp);
|
|
156
|
+
rd_kafka_toppar_broker_update(rktp, preferred_id, preferred_rkb,
|
|
157
|
+
"preferred replica updated");
|
|
158
|
+
rd_kafka_toppar_unlock(rktp);
|
|
159
|
+
rd_kafka_broker_destroy(preferred_rkb);
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
if (rd_interval_immediate(&rktp->rktp_metadata_intvl, five_seconds, 0) >
|
|
164
|
+
0) {
|
|
165
|
+
rd_rkb_log(rkb, LOG_NOTICE, "FETCH",
|
|
166
|
+
"%.*s [%" PRId32 "]: preferred replica (%" PRId32
|
|
167
|
+
") "
|
|
168
|
+
"is unknown: refreshing metadata",
|
|
169
|
+
RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
|
|
170
|
+
rktp->rktp_partition, preferred_id);
|
|
171
|
+
|
|
172
|
+
rd_kafka_metadata_refresh_brokers(
|
|
173
|
+
rktp->rktp_rkt->rkt_rk, NULL,
|
|
174
|
+
"preferred replica unavailable");
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
rd_kafka_toppar_fetch_backoff(rkb, rktp,
|
|
178
|
+
RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* @brief Handle partition-specific Fetch error.
|
|
184
|
+
*/
|
|
185
|
+
static void rd_kafka_fetch_reply_handle_partition_error(
|
|
186
|
+
rd_kafka_broker_t *rkb,
|
|
187
|
+
rd_kafka_toppar_t *rktp,
|
|
188
|
+
const struct rd_kafka_toppar_ver *tver,
|
|
189
|
+
rd_kafka_resp_err_t err,
|
|
190
|
+
int64_t HighwaterMarkOffset) {
|
|
191
|
+
|
|
192
|
+
rd_rkb_dbg(rkb, FETCH, "FETCHERR",
|
|
193
|
+
"%.*s [%" PRId32 "]: Fetch failed at %s: %s",
|
|
194
|
+
RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
|
|
195
|
+
rktp->rktp_partition,
|
|
196
|
+
rd_kafka_fetch_pos2str(rktp->rktp_offsets.fetch_pos),
|
|
197
|
+
rd_kafka_err2name(err));
|
|
198
|
+
|
|
199
|
+
/* Some errors should be passed to the
|
|
200
|
+
* application while some handled by rdkafka */
|
|
201
|
+
switch (err) {
|
|
202
|
+
/* Errors handled by rdkafka */
|
|
203
|
+
case RD_KAFKA_RESP_ERR_OFFSET_NOT_AVAILABLE:
|
|
204
|
+
case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART:
|
|
205
|
+
case RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE:
|
|
206
|
+
case RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER:
|
|
207
|
+
case RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE:
|
|
208
|
+
case RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE:
|
|
209
|
+
case RD_KAFKA_RESP_ERR_KAFKA_STORAGE_ERROR:
|
|
210
|
+
case RD_KAFKA_RESP_ERR_UNKNOWN_LEADER_EPOCH:
|
|
211
|
+
case RD_KAFKA_RESP_ERR_FENCED_LEADER_EPOCH:
|
|
212
|
+
case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID:
|
|
213
|
+
if (err == RD_KAFKA_RESP_ERR_OFFSET_NOT_AVAILABLE) {
|
|
214
|
+
/* Occurs when:
|
|
215
|
+
* - Msg exists on broker but
|
|
216
|
+
* offset > HWM, or:
|
|
217
|
+
* - HWM is >= offset, but msg not
|
|
218
|
+
* yet available at that offset
|
|
219
|
+
* (replica is out of sync).
|
|
220
|
+
* - partition leader is out of sync.
|
|
221
|
+
*
|
|
222
|
+
* Handle by requesting metadata update, changing back
|
|
223
|
+
* to the leader, and then retrying FETCH
|
|
224
|
+
* (with backoff).
|
|
225
|
+
*/
|
|
226
|
+
rd_rkb_dbg(rkb, MSG, "FETCH",
|
|
227
|
+
"Topic %s [%" PRId32
|
|
228
|
+
"]: %s not "
|
|
229
|
+
"available on broker %" PRId32
|
|
230
|
+
" (leader %" PRId32
|
|
231
|
+
"): updating metadata and retrying",
|
|
232
|
+
rktp->rktp_rkt->rkt_topic->str,
|
|
233
|
+
rktp->rktp_partition,
|
|
234
|
+
rd_kafka_fetch_pos2str(
|
|
235
|
+
rktp->rktp_offsets.fetch_pos),
|
|
236
|
+
rktp->rktp_broker_id, rktp->rktp_leader_id);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
if (err == RD_KAFKA_RESP_ERR_UNKNOWN_LEADER_EPOCH) {
|
|
240
|
+
rd_rkb_dbg(rkb, MSG | RD_KAFKA_DBG_CONSUMER, "FETCH",
|
|
241
|
+
"Topic %s [%" PRId32
|
|
242
|
+
"]: Fetch failed at %s: %s: broker %" PRId32
|
|
243
|
+
"has not yet caught up on latest metadata: "
|
|
244
|
+
"retrying",
|
|
245
|
+
rktp->rktp_rkt->rkt_topic->str,
|
|
246
|
+
rktp->rktp_partition,
|
|
247
|
+
rd_kafka_fetch_pos2str(
|
|
248
|
+
rktp->rktp_offsets.fetch_pos),
|
|
249
|
+
rd_kafka_err2str(err), rktp->rktp_broker_id);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
if (rktp->rktp_broker_id != rktp->rktp_leader_id) {
|
|
253
|
+
rd_kafka_toppar_delegate_to_leader(rktp);
|
|
254
|
+
}
|
|
255
|
+
/* Request metadata information update*/
|
|
256
|
+
rd_kafka_toppar_leader_unavailable(rktp, "fetch", err);
|
|
257
|
+
break;
|
|
258
|
+
|
|
259
|
+
case RD_KAFKA_RESP_ERR_OFFSET_OUT_OF_RANGE: {
|
|
260
|
+
rd_kafka_fetch_pos_t err_pos;
|
|
261
|
+
|
|
262
|
+
if (rktp->rktp_broker_id != rktp->rktp_leader_id &&
|
|
263
|
+
rktp->rktp_offsets.fetch_pos.offset > HighwaterMarkOffset) {
|
|
264
|
+
rd_kafka_log(rkb->rkb_rk, LOG_WARNING, "FETCH",
|
|
265
|
+
"Topic %s [%" PRId32
|
|
266
|
+
"]: %s "
|
|
267
|
+
" out of range (HighwaterMark %" PRId64
|
|
268
|
+
" fetching from "
|
|
269
|
+
"broker %" PRId32 " (leader %" PRId32
|
|
270
|
+
"): reverting to leader",
|
|
271
|
+
rktp->rktp_rkt->rkt_topic->str,
|
|
272
|
+
rktp->rktp_partition,
|
|
273
|
+
rd_kafka_fetch_pos2str(
|
|
274
|
+
rktp->rktp_offsets.fetch_pos),
|
|
275
|
+
HighwaterMarkOffset, rktp->rktp_broker_id,
|
|
276
|
+
rktp->rktp_leader_id);
|
|
277
|
+
|
|
278
|
+
/* Out of range error cannot be taken as definitive
|
|
279
|
+
* when fetching from follower.
|
|
280
|
+
* Revert back to the leader in lieu of KIP-320.
|
|
281
|
+
*/
|
|
282
|
+
rd_kafka_toppar_delegate_to_leader(rktp);
|
|
283
|
+
break;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/* Application error */
|
|
287
|
+
err_pos = rktp->rktp_offsets.fetch_pos;
|
|
288
|
+
rktp->rktp_offsets.fetch_pos.offset = RD_KAFKA_OFFSET_INVALID;
|
|
289
|
+
rktp->rktp_offsets.fetch_pos.leader_epoch = -1;
|
|
290
|
+
rd_kafka_offset_reset(rktp, rd_kafka_broker_id(rkb), err_pos,
|
|
291
|
+
err,
|
|
292
|
+
"fetch failed due to requested offset "
|
|
293
|
+
"not available on the broker");
|
|
294
|
+
} break;
|
|
295
|
+
|
|
296
|
+
case RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED:
|
|
297
|
+
/* If we're not authorized to access the
|
|
298
|
+
* topic mark it as errored to deny
|
|
299
|
+
* further Fetch requests. */
|
|
300
|
+
if (rktp->rktp_last_error != err) {
|
|
301
|
+
rd_kafka_consumer_err(
|
|
302
|
+
rktp->rktp_fetchq, rd_kafka_broker_id(rkb), err,
|
|
303
|
+
tver->version, NULL, rktp,
|
|
304
|
+
rktp->rktp_offsets.fetch_pos.offset,
|
|
305
|
+
"Fetch from broker %" PRId32 " failed: %s",
|
|
306
|
+
rd_kafka_broker_id(rkb), rd_kafka_err2str(err));
|
|
307
|
+
rktp->rktp_last_error = err;
|
|
308
|
+
}
|
|
309
|
+
break;
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
/* Application errors */
|
|
313
|
+
case RD_KAFKA_RESP_ERR__PARTITION_EOF:
|
|
314
|
+
if (rkb->rkb_rk->rk_conf.enable_partition_eof)
|
|
315
|
+
rd_kafka_consumer_err(
|
|
316
|
+
rktp->rktp_fetchq, rd_kafka_broker_id(rkb), err,
|
|
317
|
+
tver->version, NULL, rktp,
|
|
318
|
+
rktp->rktp_offsets.fetch_pos.offset,
|
|
319
|
+
"Fetch from broker %" PRId32
|
|
320
|
+
" reached end of "
|
|
321
|
+
"partition at offset %" PRId64
|
|
322
|
+
" (HighwaterMark %" PRId64 ")",
|
|
323
|
+
rd_kafka_broker_id(rkb),
|
|
324
|
+
rktp->rktp_offsets.fetch_pos.offset,
|
|
325
|
+
HighwaterMarkOffset);
|
|
326
|
+
break;
|
|
327
|
+
|
|
328
|
+
case RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE:
|
|
329
|
+
default: /* and all other errors */
|
|
330
|
+
rd_dassert(tver->version > 0);
|
|
331
|
+
rd_kafka_consumer_err(
|
|
332
|
+
rktp->rktp_fetchq, rd_kafka_broker_id(rkb), err,
|
|
333
|
+
tver->version, NULL, rktp,
|
|
334
|
+
rktp->rktp_offsets.fetch_pos.offset,
|
|
335
|
+
"Fetch from broker %" PRId32 " failed at %s: %s",
|
|
336
|
+
rd_kafka_broker_id(rkb),
|
|
337
|
+
rd_kafka_fetch_pos2str(rktp->rktp_offsets.fetch_pos),
|
|
338
|
+
rd_kafka_err2str(err));
|
|
339
|
+
break;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/* Back off the next fetch for this partition */
|
|
343
|
+
rd_kafka_toppar_fetch_backoff(rkb, rktp, err);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
static void rd_kafkap_Fetch_reply_tags_set_topic_cnt(
|
|
347
|
+
rd_kafkap_Fetch_reply_tags_t *reply_tags,
|
|
348
|
+
int32_t TopicCnt) {
|
|
349
|
+
reply_tags->TopicCnt = TopicCnt;
|
|
350
|
+
rd_dassert(!reply_tags->Topics);
|
|
351
|
+
reply_tags->Topics = rd_calloc(TopicCnt, sizeof(*reply_tags->Topics));
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
static void
|
|
355
|
+
rd_kafkap_Fetch_reply_tags_set_topic(rd_kafkap_Fetch_reply_tags_t *reply_tags,
|
|
356
|
+
int TopicIdx,
|
|
357
|
+
rd_kafka_Uuid_t TopicId,
|
|
358
|
+
int32_t PartitionCnt) {
|
|
359
|
+
reply_tags->Topics[TopicIdx].TopicId = TopicId;
|
|
360
|
+
reply_tags->Topics[TopicIdx].PartitionCnt = PartitionCnt;
|
|
361
|
+
rd_dassert(!reply_tags->Topics[TopicIdx].Partitions);
|
|
362
|
+
reply_tags->Topics[TopicIdx].Partitions = rd_calloc(
|
|
363
|
+
PartitionCnt, sizeof(*reply_tags->Topics[TopicIdx].Partitions));
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
static void
|
|
368
|
+
rd_kafkap_Fetch_reply_tags_destroy(rd_kafkap_Fetch_reply_tags_t *reply_tags) {
|
|
369
|
+
int i;
|
|
370
|
+
for (i = 0; i < reply_tags->TopicCnt; i++) {
|
|
371
|
+
RD_IF_FREE(reply_tags->Topics[i].Partitions, rd_free);
|
|
372
|
+
}
|
|
373
|
+
RD_IF_FREE(reply_tags->Topics, rd_free);
|
|
374
|
+
RD_IF_FREE(reply_tags->NodeEndpoints.NodeEndpoints, rd_free);
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
static int rd_kafkap_Fetch_reply_tags_partition_parse(
|
|
378
|
+
rd_kafka_buf_t *rkbuf,
|
|
379
|
+
uint64_t tagtype,
|
|
380
|
+
uint64_t taglen,
|
|
381
|
+
rd_kafkap_Fetch_reply_tags_Topic_t *TopicTags,
|
|
382
|
+
rd_kafkap_Fetch_reply_tags_Partition_t *PartitionTags) {
|
|
383
|
+
switch (tagtype) {
|
|
384
|
+
case 1: /* CurrentLeader */
|
|
385
|
+
if (rd_kafka_buf_read_CurrentLeader(
|
|
386
|
+
rkbuf, &PartitionTags->CurrentLeader) == -1)
|
|
387
|
+
goto err_parse;
|
|
388
|
+
TopicTags->partitions_with_leader_change_cnt++;
|
|
389
|
+
return 1;
|
|
390
|
+
default:
|
|
391
|
+
return 0;
|
|
392
|
+
}
|
|
393
|
+
err_parse:
|
|
394
|
+
return -1;
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
static int
|
|
398
|
+
rd_kafkap_Fetch_reply_tags_parse(rd_kafka_buf_t *rkbuf,
|
|
399
|
+
uint64_t tagtype,
|
|
400
|
+
uint64_t taglen,
|
|
401
|
+
rd_kafkap_Fetch_reply_tags_t *tags) {
|
|
402
|
+
switch (tagtype) {
|
|
403
|
+
case 0: /* NodeEndpoints */
|
|
404
|
+
if (rd_kafka_buf_read_NodeEndpoints(rkbuf,
|
|
405
|
+
&tags->NodeEndpoints) == -1)
|
|
406
|
+
goto err_parse;
|
|
407
|
+
return 1;
|
|
408
|
+
default:
|
|
409
|
+
return 0;
|
|
410
|
+
}
|
|
411
|
+
err_parse:
|
|
412
|
+
return -1;
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
static void
|
|
416
|
+
rd_kafka_handle_Fetch_metadata_update(rd_kafka_broker_t *rkb,
|
|
417
|
+
rd_kafkap_Fetch_reply_tags_t *FetchTags) {
|
|
418
|
+
if (FetchTags->topics_with_leader_change_cnt &&
|
|
419
|
+
FetchTags->NodeEndpoints.NodeEndpoints) {
|
|
420
|
+
rd_kafka_metadata_t *md = NULL;
|
|
421
|
+
rd_kafka_metadata_internal_t *mdi = NULL;
|
|
422
|
+
rd_tmpabuf_t tbuf;
|
|
423
|
+
int32_t nodeid;
|
|
424
|
+
rd_kafka_op_t *rko;
|
|
425
|
+
int i, changed_topic, changed_partition;
|
|
426
|
+
|
|
427
|
+
rd_kafka_broker_lock(rkb);
|
|
428
|
+
nodeid = rkb->rkb_nodeid;
|
|
429
|
+
rd_kafka_broker_unlock(rkb);
|
|
430
|
+
|
|
431
|
+
rd_tmpabuf_new(&tbuf, 0, rd_true /*assert on fail*/);
|
|
432
|
+
rd_tmpabuf_add_alloc(&tbuf, sizeof(*mdi));
|
|
433
|
+
rd_kafkap_leader_discovery_tmpabuf_add_alloc_brokers(
|
|
434
|
+
&tbuf, &FetchTags->NodeEndpoints);
|
|
435
|
+
rd_kafkap_leader_discovery_tmpabuf_add_alloc_topics(
|
|
436
|
+
&tbuf, FetchTags->topics_with_leader_change_cnt);
|
|
437
|
+
for (i = 0; i < FetchTags->TopicCnt; i++) {
|
|
438
|
+
if (!FetchTags->Topics[i]
|
|
439
|
+
.partitions_with_leader_change_cnt)
|
|
440
|
+
continue;
|
|
441
|
+
rd_kafkap_leader_discovery_tmpabuf_add_alloc_topic(
|
|
442
|
+
&tbuf, NULL,
|
|
443
|
+
FetchTags->Topics[i]
|
|
444
|
+
.partitions_with_leader_change_cnt);
|
|
445
|
+
}
|
|
446
|
+
rd_tmpabuf_finalize(&tbuf);
|
|
447
|
+
|
|
448
|
+
mdi = rd_tmpabuf_alloc(&tbuf, sizeof(*mdi));
|
|
449
|
+
md = &mdi->metadata;
|
|
450
|
+
|
|
451
|
+
rd_kafkap_leader_discovery_metadata_init(mdi, nodeid);
|
|
452
|
+
|
|
453
|
+
rd_kafkap_leader_discovery_set_brokers(
|
|
454
|
+
&tbuf, mdi, &FetchTags->NodeEndpoints);
|
|
455
|
+
|
|
456
|
+
rd_kafkap_leader_discovery_set_topic_cnt(
|
|
457
|
+
&tbuf, mdi, FetchTags->topics_with_leader_change_cnt);
|
|
458
|
+
|
|
459
|
+
changed_topic = 0;
|
|
460
|
+
for (i = 0; i < FetchTags->TopicCnt; i++) {
|
|
461
|
+
int j;
|
|
462
|
+
if (!FetchTags->Topics[i]
|
|
463
|
+
.partitions_with_leader_change_cnt)
|
|
464
|
+
continue;
|
|
465
|
+
|
|
466
|
+
rd_kafkap_leader_discovery_set_topic(
|
|
467
|
+
&tbuf, mdi, changed_topic,
|
|
468
|
+
FetchTags->Topics[i].TopicId, NULL,
|
|
469
|
+
FetchTags->Topics[i]
|
|
470
|
+
.partitions_with_leader_change_cnt);
|
|
471
|
+
|
|
472
|
+
changed_partition = 0;
|
|
473
|
+
for (j = 0; j < FetchTags->Topics[i].PartitionCnt;
|
|
474
|
+
j++) {
|
|
475
|
+
if (FetchTags->Topics[i]
|
|
476
|
+
.Partitions[j]
|
|
477
|
+
.CurrentLeader.LeaderId < 0)
|
|
478
|
+
continue;
|
|
479
|
+
|
|
480
|
+
rd_kafkap_Fetch_reply_tags_Partition_t
|
|
481
|
+
*Partition =
|
|
482
|
+
&FetchTags->Topics[i].Partitions[j];
|
|
483
|
+
rd_kafkap_leader_discovery_set_CurrentLeader(
|
|
484
|
+
&tbuf, mdi, changed_topic,
|
|
485
|
+
changed_partition, Partition->Partition,
|
|
486
|
+
&Partition->CurrentLeader);
|
|
487
|
+
changed_partition++;
|
|
488
|
+
}
|
|
489
|
+
changed_topic++;
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
rko = rd_kafka_op_new(RD_KAFKA_OP_METADATA_UPDATE);
|
|
493
|
+
rko->rko_u.metadata.md = md;
|
|
494
|
+
rko->rko_u.metadata.mdi = mdi;
|
|
495
|
+
rd_kafka_q_enq(rkb->rkb_rk->rk_ops, rko);
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
/**
|
|
500
|
+
* @brief Per-partition FetchResponse parsing and handling.
|
|
501
|
+
*
|
|
502
|
+
* @returns an error on buffer parse failure, else RD_KAFKA_RESP_ERR_NO_ERROR.
|
|
503
|
+
*/
|
|
504
|
+
static rd_kafka_resp_err_t rd_kafka_fetch_reply_handle_partition(
|
|
505
|
+
rd_kafka_broker_t *rkb,
|
|
506
|
+
const rd_kafkap_str_t *topic,
|
|
507
|
+
rd_kafka_topic_t *rkt /*possibly NULL*/,
|
|
508
|
+
rd_kafka_buf_t *rkbuf,
|
|
509
|
+
rd_kafka_buf_t *request,
|
|
510
|
+
int16_t ErrorCode,
|
|
511
|
+
rd_kafkap_Fetch_reply_tags_Topic_t *TopicTags,
|
|
512
|
+
rd_kafkap_Fetch_reply_tags_Partition_t *PartitionTags) {
|
|
513
|
+
const int log_decode_errors = LOG_ERR;
|
|
514
|
+
struct rd_kafka_toppar_ver *tver, tver_skel;
|
|
515
|
+
rd_kafka_toppar_t *rktp = NULL;
|
|
516
|
+
rd_kafka_aborted_txns_t *aborted_txns = NULL;
|
|
517
|
+
rd_slice_t save_slice;
|
|
518
|
+
int32_t fetch_version;
|
|
519
|
+
struct {
|
|
520
|
+
int32_t Partition;
|
|
521
|
+
int16_t ErrorCode;
|
|
522
|
+
int64_t HighwaterMarkOffset;
|
|
523
|
+
int64_t LastStableOffset; /* v4 */
|
|
524
|
+
int64_t LogStartOffset; /* v5 */
|
|
525
|
+
int32_t MessageSetSize;
|
|
526
|
+
int32_t PreferredReadReplica; /* v11 */
|
|
527
|
+
} hdr;
|
|
528
|
+
rd_kafka_resp_err_t err;
|
|
529
|
+
int64_t end_offset;
|
|
530
|
+
|
|
531
|
+
rd_kafka_buf_read_i32(rkbuf, &hdr.Partition);
|
|
532
|
+
rd_kafka_buf_read_i16(rkbuf, &hdr.ErrorCode);
|
|
533
|
+
if (PartitionTags)
|
|
534
|
+
PartitionTags->Partition = hdr.Partition;
|
|
535
|
+
if (ErrorCode)
|
|
536
|
+
hdr.ErrorCode = ErrorCode;
|
|
537
|
+
rd_kafka_buf_read_i64(rkbuf, &hdr.HighwaterMarkOffset);
|
|
538
|
+
|
|
539
|
+
end_offset = hdr.HighwaterMarkOffset;
|
|
540
|
+
|
|
541
|
+
hdr.LastStableOffset = RD_KAFKA_OFFSET_INVALID;
|
|
542
|
+
hdr.LogStartOffset = RD_KAFKA_OFFSET_INVALID;
|
|
543
|
+
if (rd_kafka_buf_ApiVersion(request) >= 4) {
|
|
544
|
+
int32_t AbortedTxnCnt;
|
|
545
|
+
int k;
|
|
546
|
+
rd_kafka_buf_read_i64(rkbuf, &hdr.LastStableOffset);
|
|
547
|
+
if (rd_kafka_buf_ApiVersion(request) >= 5)
|
|
548
|
+
rd_kafka_buf_read_i64(rkbuf, &hdr.LogStartOffset);
|
|
549
|
+
|
|
550
|
+
rd_kafka_buf_read_arraycnt(rkbuf, &AbortedTxnCnt,
|
|
551
|
+
RD_KAFKAP_ABORTED_TRANSACTIONS_MAX);
|
|
552
|
+
|
|
553
|
+
if (rkb->rkb_rk->rk_conf.isolation_level ==
|
|
554
|
+
RD_KAFKA_READ_UNCOMMITTED) {
|
|
555
|
+
|
|
556
|
+
if (unlikely(AbortedTxnCnt > 0)) {
|
|
557
|
+
rd_rkb_log(rkb, LOG_ERR, "FETCH",
|
|
558
|
+
"%.*s [%" PRId32
|
|
559
|
+
"]: "
|
|
560
|
+
"%" PRId32
|
|
561
|
+
" aborted transaction(s) "
|
|
562
|
+
"encountered in READ_UNCOMMITTED "
|
|
563
|
+
"fetch response: ignoring.",
|
|
564
|
+
RD_KAFKAP_STR_PR(topic),
|
|
565
|
+
hdr.Partition, AbortedTxnCnt);
|
|
566
|
+
for (k = 0; k < AbortedTxnCnt; k++) {
|
|
567
|
+
rd_kafka_buf_skip(rkbuf, (8 + 8));
|
|
568
|
+
/* AbortedTransaction tags */
|
|
569
|
+
rd_kafka_buf_skip_tags(rkbuf);
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
} else {
|
|
573
|
+
/* Older brokers may return LSO -1,
|
|
574
|
+
* in which case we use the HWM. */
|
|
575
|
+
if (hdr.LastStableOffset >= 0)
|
|
576
|
+
end_offset = hdr.LastStableOffset;
|
|
577
|
+
|
|
578
|
+
if (AbortedTxnCnt > 0) {
|
|
579
|
+
aborted_txns =
|
|
580
|
+
rd_kafka_aborted_txns_new(AbortedTxnCnt);
|
|
581
|
+
for (k = 0; k < AbortedTxnCnt; k++) {
|
|
582
|
+
int64_t PID;
|
|
583
|
+
int64_t FirstOffset;
|
|
584
|
+
rd_kafka_buf_read_i64(rkbuf, &PID);
|
|
585
|
+
rd_kafka_buf_read_i64(rkbuf,
|
|
586
|
+
&FirstOffset);
|
|
587
|
+
/* AbortedTransaction tags */
|
|
588
|
+
rd_kafka_buf_skip_tags(rkbuf);
|
|
589
|
+
rd_kafka_aborted_txns_add(
|
|
590
|
+
aborted_txns, PID, FirstOffset);
|
|
591
|
+
}
|
|
592
|
+
rd_kafka_aborted_txns_sort(aborted_txns);
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
if (rd_kafka_buf_ApiVersion(request) >= 11)
|
|
598
|
+
rd_kafka_buf_read_i32(rkbuf, &hdr.PreferredReadReplica);
|
|
599
|
+
else
|
|
600
|
+
hdr.PreferredReadReplica = -1;
|
|
601
|
+
/* Compact Records Array */
|
|
602
|
+
rd_kafka_buf_read_arraycnt(rkbuf, &hdr.MessageSetSize, -1);
|
|
603
|
+
|
|
604
|
+
if (unlikely(hdr.MessageSetSize < 0))
|
|
605
|
+
rd_kafka_buf_parse_fail(
|
|
606
|
+
rkbuf,
|
|
607
|
+
"%.*s [%" PRId32 "]: invalid MessageSetSize %" PRId32,
|
|
608
|
+
RD_KAFKAP_STR_PR(topic), hdr.Partition, hdr.MessageSetSize);
|
|
609
|
+
|
|
610
|
+
/* Look up topic+partition */
|
|
611
|
+
if (likely(rkt != NULL)) {
|
|
612
|
+
rd_kafka_topic_rdlock(rkt);
|
|
613
|
+
rktp = rd_kafka_toppar_get(rkt, hdr.Partition,
|
|
614
|
+
0 /*no ua-on-miss*/);
|
|
615
|
+
rd_kafka_topic_rdunlock(rkt);
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
if (unlikely(!rkt || !rktp)) {
|
|
619
|
+
rd_rkb_dbg(rkb, TOPIC, "UNKTOPIC",
|
|
620
|
+
"Received Fetch response (error %hu) for unknown "
|
|
621
|
+
"topic %.*s [%" PRId32 "]: ignoring",
|
|
622
|
+
hdr.ErrorCode, RD_KAFKAP_STR_PR(topic),
|
|
623
|
+
hdr.Partition);
|
|
624
|
+
rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize);
|
|
625
|
+
goto done;
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
rd_kafka_toppar_lock(rktp);
|
|
629
|
+
rktp->rktp_lo_offset = hdr.LogStartOffset;
|
|
630
|
+
rktp->rktp_hi_offset = hdr.HighwaterMarkOffset;
|
|
631
|
+
/* Let the LastStable offset be the effective
|
|
632
|
+
* end_offset based on protocol version, that is:
|
|
633
|
+
* if connected to a broker that does not support
|
|
634
|
+
* LastStableOffset we use the HighwaterMarkOffset. */
|
|
635
|
+
rktp->rktp_ls_offset = end_offset;
|
|
636
|
+
rd_kafka_toppar_unlock(rktp);
|
|
637
|
+
|
|
638
|
+
if (hdr.PreferredReadReplica != -1) {
|
|
639
|
+
|
|
640
|
+
rd_kafka_fetch_preferred_replica_handle(
|
|
641
|
+
rktp, rkbuf, rkb, hdr.PreferredReadReplica);
|
|
642
|
+
|
|
643
|
+
if (unlikely(hdr.MessageSetSize != 0)) {
|
|
644
|
+
rd_rkb_log(rkb, LOG_WARNING, "FETCH",
|
|
645
|
+
"%.*s [%" PRId32
|
|
646
|
+
"]: Fetch response has both preferred read "
|
|
647
|
+
"replica and non-zero message set size: "
|
|
648
|
+
"%" PRId32 ": skipping messages",
|
|
649
|
+
RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
|
|
650
|
+
rktp->rktp_partition, hdr.MessageSetSize);
|
|
651
|
+
rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize);
|
|
652
|
+
}
|
|
653
|
+
goto done;
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
rd_kafka_toppar_lock(rktp);
|
|
657
|
+
|
|
658
|
+
/* Make sure toppar hasn't moved to another broker
|
|
659
|
+
* during the lifetime of the request. */
|
|
660
|
+
if (unlikely(rktp->rktp_broker != rkb)) {
|
|
661
|
+
rd_kafka_toppar_unlock(rktp);
|
|
662
|
+
rd_rkb_dbg(rkb, MSG, "FETCH",
|
|
663
|
+
"%.*s [%" PRId32
|
|
664
|
+
"]: partition broker has changed: "
|
|
665
|
+
"discarding fetch response",
|
|
666
|
+
RD_KAFKAP_STR_PR(topic), hdr.Partition);
|
|
667
|
+
rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize);
|
|
668
|
+
goto done;
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
fetch_version = rktp->rktp_fetch_version;
|
|
672
|
+
rd_kafka_toppar_unlock(rktp);
|
|
673
|
+
|
|
674
|
+
/* Check if this Fetch is for an outdated fetch version,
|
|
675
|
+
* or the original rktp was removed and a new one
|
|
676
|
+
* created (due to partition count decreasing and
|
|
677
|
+
* then increasing again, which can happen in
|
|
678
|
+
* desynchronized clusters): if so ignore it. */
|
|
679
|
+
tver_skel.rktp = rktp;
|
|
680
|
+
tver = rd_list_find(request->rkbuf_rktp_vers, &tver_skel,
|
|
681
|
+
rd_kafka_toppar_ver_cmp);
|
|
682
|
+
rd_kafka_assert(NULL, tver);
|
|
683
|
+
if (tver->rktp != rktp || tver->version < fetch_version) {
|
|
684
|
+
rd_rkb_dbg(rkb, MSG, "DROP",
|
|
685
|
+
"%s [%" PRId32
|
|
686
|
+
"]: dropping outdated fetch response "
|
|
687
|
+
"(v%d < %d or old rktp)",
|
|
688
|
+
rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition,
|
|
689
|
+
tver->version, fetch_version);
|
|
690
|
+
rd_atomic64_add(&rktp->rktp_c.rx_ver_drops, 1);
|
|
691
|
+
rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize);
|
|
692
|
+
goto done;
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
rd_rkb_dbg(rkb, MSG, "FETCH",
|
|
696
|
+
"Topic %.*s [%" PRId32 "] MessageSet size %" PRId32
|
|
697
|
+
", error \"%s\", MaxOffset %" PRId64 ", LSO %" PRId64
|
|
698
|
+
", Ver %" PRId32 "/%" PRId32,
|
|
699
|
+
RD_KAFKAP_STR_PR(topic), hdr.Partition, hdr.MessageSetSize,
|
|
700
|
+
rd_kafka_err2str(hdr.ErrorCode), hdr.HighwaterMarkOffset,
|
|
701
|
+
hdr.LastStableOffset, tver->version, fetch_version);
|
|
702
|
+
|
|
703
|
+
/* If this is the last message of the queue,
|
|
704
|
+
* signal EOF back to the application. */
|
|
705
|
+
if (end_offset == rktp->rktp_offsets.fetch_pos.offset &&
|
|
706
|
+
rktp->rktp_offsets.eof_offset != end_offset) {
|
|
707
|
+
hdr.ErrorCode = RD_KAFKA_RESP_ERR__PARTITION_EOF;
|
|
708
|
+
rktp->rktp_offsets.eof_offset = end_offset;
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
if (unlikely(hdr.ErrorCode != RD_KAFKA_RESP_ERR_NO_ERROR)) {
|
|
712
|
+
/* Handle partition-level errors. */
|
|
713
|
+
rd_kafka_fetch_reply_handle_partition_error(
|
|
714
|
+
rkb, rktp, tver, hdr.ErrorCode, hdr.HighwaterMarkOffset);
|
|
715
|
+
|
|
716
|
+
rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize);
|
|
717
|
+
goto done;
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
/* No error, clear any previous fetch error. */
|
|
721
|
+
rktp->rktp_last_error = RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
722
|
+
|
|
723
|
+
if (unlikely(hdr.MessageSetSize <= 0))
|
|
724
|
+
goto done;
|
|
725
|
+
|
|
726
|
+
/**
|
|
727
|
+
* Parse MessageSet
|
|
728
|
+
*/
|
|
729
|
+
if (!rd_slice_narrow_relative(&rkbuf->rkbuf_reader, &save_slice,
|
|
730
|
+
(size_t)hdr.MessageSetSize))
|
|
731
|
+
rd_kafka_buf_check_len(rkbuf, hdr.MessageSetSize);
|
|
732
|
+
|
|
733
|
+
/* Parse messages */
|
|
734
|
+
err = rd_kafka_msgset_parse(rkbuf, request, rktp, aborted_txns, tver);
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
rd_slice_widen(&rkbuf->rkbuf_reader, &save_slice);
|
|
738
|
+
/* Continue with next partition regardless of
|
|
739
|
+
* parse errors (which are partition-specific) */
|
|
740
|
+
|
|
741
|
+
/* On error: back off the fetcher for this partition */
|
|
742
|
+
if (unlikely(err))
|
|
743
|
+
rd_kafka_toppar_fetch_backoff(rkb, rktp, err);
|
|
744
|
+
|
|
745
|
+
goto done;
|
|
746
|
+
|
|
747
|
+
err_parse:
|
|
748
|
+
if (aborted_txns)
|
|
749
|
+
rd_kafka_aborted_txns_destroy(aborted_txns);
|
|
750
|
+
if (rktp)
|
|
751
|
+
rd_kafka_toppar_destroy(rktp); /*from get()*/
|
|
752
|
+
return rkbuf->rkbuf_err;
|
|
753
|
+
|
|
754
|
+
done:
|
|
755
|
+
if (aborted_txns)
|
|
756
|
+
rd_kafka_aborted_txns_destroy(aborted_txns);
|
|
757
|
+
if (likely(rktp != NULL))
|
|
758
|
+
rd_kafka_toppar_destroy(rktp); /*from get()*/
|
|
759
|
+
|
|
760
|
+
if (PartitionTags) {
|
|
761
|
+
/* Set default LeaderId and LeaderEpoch */
|
|
762
|
+
PartitionTags->CurrentLeader.LeaderId = -1;
|
|
763
|
+
PartitionTags->CurrentLeader.LeaderEpoch = -1;
|
|
764
|
+
}
|
|
765
|
+
rd_kafka_buf_read_tags(rkbuf,
|
|
766
|
+
rd_kafkap_Fetch_reply_tags_partition_parse,
|
|
767
|
+
TopicTags, PartitionTags);
|
|
768
|
+
|
|
769
|
+
return RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
/**
|
|
773
|
+
* Parses and handles a Fetch reply.
|
|
774
|
+
* Returns 0 on success or an error code on failure.
|
|
775
|
+
*/
|
|
776
|
+
static rd_kafka_resp_err_t
|
|
777
|
+
rd_kafka_fetch_reply_handle(rd_kafka_broker_t *rkb,
|
|
778
|
+
rd_kafka_buf_t *rkbuf,
|
|
779
|
+
rd_kafka_buf_t *request) {
|
|
780
|
+
int32_t TopicArrayCnt;
|
|
781
|
+
int i;
|
|
782
|
+
const int log_decode_errors = LOG_ERR;
|
|
783
|
+
rd_kafka_topic_t *rkt = NULL;
|
|
784
|
+
int16_t ErrorCode = RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
785
|
+
rd_kafkap_Fetch_reply_tags_t FetchTags = RD_ZERO_INIT;
|
|
786
|
+
rd_bool_t has_fetch_tags = rd_false;
|
|
787
|
+
|
|
788
|
+
if (rd_kafka_buf_ApiVersion(request) >= 1) {
|
|
789
|
+
int32_t Throttle_Time;
|
|
790
|
+
rd_kafka_buf_read_i32(rkbuf, &Throttle_Time);
|
|
791
|
+
|
|
792
|
+
rd_kafka_op_throttle_time(rkb, rkb->rkb_rk->rk_rep,
|
|
793
|
+
Throttle_Time);
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
if (rd_kafka_buf_ApiVersion(request) >= 7) {
|
|
797
|
+
int32_t SessionId;
|
|
798
|
+
rd_kafka_buf_read_i16(rkbuf, &ErrorCode);
|
|
799
|
+
rd_kafka_buf_read_i32(rkbuf, &SessionId);
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
rd_kafka_buf_read_arraycnt(rkbuf, &TopicArrayCnt, RD_KAFKAP_TOPICS_MAX);
|
|
803
|
+
/* Verify that TopicArrayCnt seems to be in line with remaining size */
|
|
804
|
+
rd_kafka_buf_check_len(rkbuf,
|
|
805
|
+
TopicArrayCnt * (3 /*topic min size*/ +
|
|
806
|
+
4 /*PartitionArrayCnt*/ + 4 +
|
|
807
|
+
2 + 8 + 4 /*inner header*/));
|
|
808
|
+
|
|
809
|
+
if (rd_kafka_buf_ApiVersion(request) >= 12) {
|
|
810
|
+
has_fetch_tags = rd_true;
|
|
811
|
+
rd_kafkap_Fetch_reply_tags_set_topic_cnt(&FetchTags,
|
|
812
|
+
TopicArrayCnt);
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
for (i = 0; i < TopicArrayCnt; i++) {
|
|
816
|
+
rd_kafkap_str_t topic = RD_ZERO_INIT;
|
|
817
|
+
rd_kafka_Uuid_t topic_id = RD_KAFKA_UUID_ZERO;
|
|
818
|
+
int32_t PartitionArrayCnt;
|
|
819
|
+
int j;
|
|
820
|
+
|
|
821
|
+
if (rd_kafka_buf_ApiVersion(request) > 12) {
|
|
822
|
+
rd_kafka_buf_read_uuid(rkbuf, &topic_id);
|
|
823
|
+
rkt = rd_kafka_topic_find_by_topic_id(rkb->rkb_rk,
|
|
824
|
+
topic_id);
|
|
825
|
+
if (rkt)
|
|
826
|
+
topic = *rkt->rkt_topic;
|
|
827
|
+
} else {
|
|
828
|
+
rd_kafka_buf_read_str(rkbuf, &topic);
|
|
829
|
+
rkt = rd_kafka_topic_find0(rkb->rkb_rk, &topic);
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
rd_kafka_buf_read_arraycnt(rkbuf, &PartitionArrayCnt,
|
|
833
|
+
RD_KAFKAP_PARTITIONS_MAX);
|
|
834
|
+
if (rd_kafka_buf_ApiVersion(request) >= 12) {
|
|
835
|
+
rd_kafkap_Fetch_reply_tags_set_topic(
|
|
836
|
+
&FetchTags, i, topic_id, PartitionArrayCnt);
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
for (j = 0; j < PartitionArrayCnt; j++) {
|
|
840
|
+
if (rd_kafka_fetch_reply_handle_partition(
|
|
841
|
+
rkb, &topic, rkt, rkbuf, request, ErrorCode,
|
|
842
|
+
has_fetch_tags ? &FetchTags.Topics[i] : NULL,
|
|
843
|
+
has_fetch_tags
|
|
844
|
+
? &FetchTags.Topics[i].Partitions[j]
|
|
845
|
+
: NULL))
|
|
846
|
+
goto err_parse;
|
|
847
|
+
}
|
|
848
|
+
if (has_fetch_tags &&
|
|
849
|
+
FetchTags.Topics[i].partitions_with_leader_change_cnt) {
|
|
850
|
+
FetchTags.topics_with_leader_change_cnt++;
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
if (rkt) {
|
|
854
|
+
rd_kafka_topic_destroy0(rkt);
|
|
855
|
+
rkt = NULL;
|
|
856
|
+
}
|
|
857
|
+
/* Topic Tags */
|
|
858
|
+
rd_kafka_buf_skip_tags(rkbuf);
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
/* Top level tags */
|
|
862
|
+
rd_kafka_buf_read_tags(rkbuf, rd_kafkap_Fetch_reply_tags_parse,
|
|
863
|
+
&FetchTags);
|
|
864
|
+
|
|
865
|
+
if (rd_kafka_buf_read_remain(rkbuf) != 0) {
|
|
866
|
+
rd_kafka_buf_parse_fail(rkbuf,
|
|
867
|
+
"Remaining data after message set "
|
|
868
|
+
"parse: %" PRIusz " bytes",
|
|
869
|
+
rd_kafka_buf_read_remain(rkbuf));
|
|
870
|
+
RD_NOTREACHED();
|
|
871
|
+
}
|
|
872
|
+
rd_kafka_handle_Fetch_metadata_update(rkb, &FetchTags);
|
|
873
|
+
rd_kafkap_Fetch_reply_tags_destroy(&FetchTags);
|
|
874
|
+
|
|
875
|
+
return 0;
|
|
876
|
+
|
|
877
|
+
err_parse:
|
|
878
|
+
if (rkt)
|
|
879
|
+
rd_kafka_topic_destroy0(rkt);
|
|
880
|
+
rd_kafkap_Fetch_reply_tags_destroy(&FetchTags);
|
|
881
|
+
rd_rkb_dbg(rkb, MSG, "BADMSG",
|
|
882
|
+
"Bad message (Fetch v%d): "
|
|
883
|
+
"is broker.version.fallback incorrectly set?",
|
|
884
|
+
(int)request->rkbuf_reqhdr.ApiVersion);
|
|
885
|
+
return rkbuf->rkbuf_err;
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
void rd_kafka_share_filter_acquired_records_and_update_ack_type(
|
|
889
|
+
rd_kafka_q_t *temp_fetchq,
|
|
890
|
+
rd_list_t *filtered_msgs,
|
|
891
|
+
const int64_t *FirstOffsets,
|
|
892
|
+
const int64_t *LastOffsets,
|
|
893
|
+
const int16_t *DeliveryCounts,
|
|
894
|
+
int32_t AcquiredRecordsArrayCnt) {
|
|
895
|
+
|
|
896
|
+
rd_kafka_op_t *rko;
|
|
897
|
+
|
|
898
|
+
/* Iterate through all messages in temp_fetchq and forward
|
|
899
|
+
* only those whose offset falls within an acquired range.
|
|
900
|
+
* Also set the ack type based on the op type. */
|
|
901
|
+
while ((rko = rd_kafka_q_pop(temp_fetchq, RD_POLL_NOWAIT, 0)) != NULL) {
|
|
902
|
+
int64_t rko_offset = rd_kafka_op_get_offset(rko);
|
|
903
|
+
rd_bool_t in_acquired_range = rd_false;
|
|
904
|
+
int16_t delivery_count;
|
|
905
|
+
int32_t range_idx;
|
|
906
|
+
|
|
907
|
+
/* Check if this message's offset is within any acquired range
|
|
908
|
+
*/
|
|
909
|
+
for (range_idx = 0; range_idx < AcquiredRecordsArrayCnt;
|
|
910
|
+
range_idx++) {
|
|
911
|
+
if (rko_offset >= FirstOffsets[range_idx] &&
|
|
912
|
+
rko_offset <= LastOffsets[range_idx]) {
|
|
913
|
+
in_acquired_range = rd_true;
|
|
914
|
+
delivery_count = DeliveryCounts[range_idx];
|
|
915
|
+
break;
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
if (in_acquired_range) {
|
|
920
|
+
/* Set ack type based on op type */
|
|
921
|
+
rd_kafka_msg_t *rkm = NULL;
|
|
922
|
+
if (unlikely(rd_kafka_op_is_ctrl_msg(rko))) {
|
|
923
|
+
rd_kafka_op_destroy(rko);
|
|
924
|
+
continue;
|
|
925
|
+
}
|
|
926
|
+
if (rko->rko_type == RD_KAFKA_OP_FETCH) {
|
|
927
|
+
rkm = &rko->rko_u.fetch.rkm;
|
|
928
|
+
rkm->rkm_u.consumer.ack_type =
|
|
929
|
+
RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED;
|
|
930
|
+
rkm->rkm_u.consumer.delivery_count =
|
|
931
|
+
delivery_count;
|
|
932
|
+
} else if (rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR) {
|
|
933
|
+
rkm = &rko->rko_u.err.rkm;
|
|
934
|
+
/* Set ack_type to RELEASE only if not already
|
|
935
|
+
* set by rd_kafka_share_msgset_err_ops()
|
|
936
|
+
* (which sets REJECT for CRC/unsupported
|
|
937
|
+
* errors, RELEASE for decompression errors). */
|
|
938
|
+
if (rkm->rkm_u.consumer.ack_type ==
|
|
939
|
+
RD_KAFKA_SHARE_INTERNAL_ACK_GAP ||
|
|
940
|
+
rkm->rkm_u.consumer.ack_type ==
|
|
941
|
+
RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED)
|
|
942
|
+
rkm->rkm_u.consumer.ack_type =
|
|
943
|
+
RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE;
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
/* Add to filtered messages list */
|
|
947
|
+
rd_list_add(filtered_msgs, rko);
|
|
948
|
+
} else {
|
|
949
|
+
/* Discard message not in any acquired range */
|
|
950
|
+
rd_kafka_op_destroy(rko);
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
|
|
956
|
+
/**
|
|
957
|
+
* @brief Comparator for sorting ops by offset.
|
|
958
|
+
*
|
|
959
|
+
* Note: rd_list_sort uses rd_list_cmp_trampoline which dereferences
|
|
960
|
+
* the pointers before calling this comparator, so we receive
|
|
961
|
+
* rd_kafka_op_t* directly, not rd_kafka_op_t**.
|
|
962
|
+
*/
|
|
963
|
+
static int rd_kafka_op_offset_cmp(const void *_a, const void *_b) {
|
|
964
|
+
const rd_kafka_op_t *a = (const rd_kafka_op_t *)_a;
|
|
965
|
+
const rd_kafka_op_t *b = (const rd_kafka_op_t *)_b;
|
|
966
|
+
int64_t off_a = rd_kafka_op_get_offset(a);
|
|
967
|
+
int64_t off_b = rd_kafka_op_get_offset(b);
|
|
968
|
+
|
|
969
|
+
return (off_a > off_b) - (off_a < off_b);
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
|
|
973
|
+
/**
|
|
974
|
+
* @brief Check if op matches the given topic-partition.
|
|
975
|
+
*
|
|
976
|
+
* Compares the rktp (toppar) reference to ensure both topic and partition
|
|
977
|
+
* match, not just the partition number.
|
|
978
|
+
*/
|
|
979
|
+
static rd_bool_t rd_kafka_op_matches_toppar(const rd_kafka_op_t *rko,
|
|
980
|
+
const rd_kafka_toppar_t *rktp) {
|
|
981
|
+
return rko->rko_rktp == rktp;
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
/**
|
|
985
|
+
* @brief Find the batch entry that contains the given offset.
|
|
986
|
+
*
|
|
987
|
+
* @returns The entry, or NULL if no entry contains the offset.
|
|
988
|
+
*/
|
|
989
|
+
static rd_kafka_share_ack_batch_entry_t *
|
|
990
|
+
rd_kafka_share_find_entry_for_offset(rd_kafka_share_ack_batches_t *batches,
|
|
991
|
+
int64_t offset) {
|
|
992
|
+
rd_kafka_share_ack_batch_entry_t *entry;
|
|
993
|
+
int ei;
|
|
994
|
+
|
|
995
|
+
RD_LIST_FOREACH(entry, &batches->entries, ei) {
|
|
996
|
+
if (offset >= entry->start_offset &&
|
|
997
|
+
offset <= entry->end_offset)
|
|
998
|
+
return entry;
|
|
999
|
+
}
|
|
1000
|
+
return NULL;
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
/**
|
|
1004
|
+
* @brief Get ack type from a message op (FETCH or CONSUMER_ERR).
|
|
1005
|
+
*/
|
|
1006
|
+
static rd_kafka_share_internal_acknowledgement_type
|
|
1007
|
+
rd_kafka_share_ack_type_from_msg_op(rd_kafka_op_t *msg_rko) {
|
|
1008
|
+
rd_kafka_msg_t *rkm;
|
|
1009
|
+
|
|
1010
|
+
if (msg_rko->rko_type == RD_KAFKA_OP_FETCH)
|
|
1011
|
+
rkm = &msg_rko->rko_u.fetch.rkm;
|
|
1012
|
+
else
|
|
1013
|
+
rkm = &msg_rko->rko_u.err.rkm;
|
|
1014
|
+
return rkm->rkm_u.consumer.ack_type;
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
/**
|
|
1018
|
+
* @brief Build share fetch response RKO with messages and update inflight acks.
|
|
1019
|
+
*
|
|
1020
|
+
* All entry types are pre-initialized to GAP in the reply handler. This
|
|
1021
|
+
* function overwrites ACQUIRED/REJECT for each offset that has a message
|
|
1022
|
+
* (RD_KAFKA_OP_FETCH or RD_KAFKA_OP_CONSUMER_ERR), and adds those messages
|
|
1023
|
+
* to the response. Moves inflight_acks batches to the response RKO.
|
|
1024
|
+
*
|
|
1025
|
+
* @param rkb Broker handle
|
|
1026
|
+
* @param filtered_msgs List of filtered message ops (with ack types already
|
|
1027
|
+
* set)
|
|
1028
|
+
* @param inflight_acks List of rd_kafka_share_ack_batches_t* (ownership
|
|
1029
|
+
* transferred to RKO)
|
|
1030
|
+
*
|
|
1031
|
+
* @returns New rko containing messages and inflight_acks, or NULL if empty
|
|
1032
|
+
*/
|
|
1033
|
+
static rd_kafka_op_t *
|
|
1034
|
+
rd_kafka_share_build_response_rko(rd_kafka_broker_t *rkb,
|
|
1035
|
+
rd_list_t *filtered_msgs,
|
|
1036
|
+
rd_list_t *inflight_acks) {
|
|
1037
|
+
|
|
1038
|
+
rd_kafka_op_t *response_rko;
|
|
1039
|
+
rd_kafka_share_ack_batches_t *batches;
|
|
1040
|
+
int32_t msg_cnt = 0;
|
|
1041
|
+
int32_t inflight_acks_cnt = rd_list_cnt(inflight_acks);
|
|
1042
|
+
int pi, i;
|
|
1043
|
+
int total_msgs = rd_list_cnt(filtered_msgs);
|
|
1044
|
+
int msg_start_idx = 0;
|
|
1045
|
+
int64_t total_offsets = 0;
|
|
1046
|
+
|
|
1047
|
+
/* Create response rko */
|
|
1048
|
+
response_rko = rd_kafka_op_new(RD_KAFKA_OP_SHARE_FETCH_RESPONSE);
|
|
1049
|
+
response_rko->rko_rk = rkb->rkb_rk;
|
|
1050
|
+
|
|
1051
|
+
response_rko->rko_u.share_fetch_response.message_rkos =
|
|
1052
|
+
rd_list_new(0, NULL);
|
|
1053
|
+
response_rko->rko_u.share_fetch_response.inflight_acks =
|
|
1054
|
+
rd_list_new(0, rd_kafka_share_ack_batches_destroy_free);
|
|
1055
|
+
|
|
1056
|
+
/* Process each partition: set types for message offsets, add messages.
|
|
1057
|
+
* Messages in filtered_msgs are grouped by partition in the same order
|
|
1058
|
+
* as inflight_acks. Types are already GAP; we set ACQUIRED/REJECT only
|
|
1059
|
+
* where we have a message. */
|
|
1060
|
+
RD_LIST_FOREACH(batches, inflight_acks, pi) {
|
|
1061
|
+
|
|
1062
|
+
total_offsets += batches->response_acquired_offsets_count;
|
|
1063
|
+
|
|
1064
|
+
rd_kafka_topic_partition_private_t *parpriv =
|
|
1065
|
+
(rd_kafka_topic_partition_private_t *)
|
|
1066
|
+
batches->rktpar->_private;
|
|
1067
|
+
rd_kafka_toppar_t *rktp = parpriv->rktp;
|
|
1068
|
+
|
|
1069
|
+
rd_list_t partition_msgs;
|
|
1070
|
+
rd_list_init(&partition_msgs, 0, NULL);
|
|
1071
|
+
|
|
1072
|
+
while (msg_start_idx < total_msgs) {
|
|
1073
|
+
rd_kafka_op_t *candidate =
|
|
1074
|
+
rd_list_elem(filtered_msgs, msg_start_idx);
|
|
1075
|
+
if (!rd_kafka_op_matches_toppar(candidate, rktp))
|
|
1076
|
+
break;
|
|
1077
|
+
rd_list_add(&partition_msgs, candidate);
|
|
1078
|
+
msg_start_idx++;
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
rd_list_sort(&partition_msgs, rd_kafka_op_offset_cmp);
|
|
1082
|
+
|
|
1083
|
+
int partition_msg_cnt = rd_list_cnt(&partition_msgs);
|
|
1084
|
+
for (i = 0; i < partition_msg_cnt; i++) {
|
|
1085
|
+
rd_kafka_op_t *msg_rko =
|
|
1086
|
+
rd_list_elem(&partition_msgs, i);
|
|
1087
|
+
int64_t offset = rd_kafka_op_get_offset(msg_rko);
|
|
1088
|
+
rd_kafka_share_ack_batch_entry_t *entry =
|
|
1089
|
+
rd_kafka_share_find_entry_for_offset(batches,
|
|
1090
|
+
offset);
|
|
1091
|
+
|
|
1092
|
+
if (unlikely(!entry)) {
|
|
1093
|
+
rd_rkb_dbg(
|
|
1094
|
+
rkb, FETCH, "SHAREFETCH",
|
|
1095
|
+
"No ack entry found for offset %" PRId64
|
|
1096
|
+
" on %s [%" PRId32 "], skipping",
|
|
1097
|
+
offset, batches->rktpar->topic,
|
|
1098
|
+
batches->rktpar->partition);
|
|
1099
|
+
rd_kafka_op_destroy(msg_rko);
|
|
1100
|
+
continue;
|
|
1101
|
+
}
|
|
1102
|
+
|
|
1103
|
+
entry->types[offset - entry->start_offset] =
|
|
1104
|
+
rd_kafka_share_ack_type_from_msg_op(msg_rko);
|
|
1105
|
+
|
|
1106
|
+
/**
|
|
1107
|
+
* The per message error ops (Decompression error, CRC
|
|
1108
|
+
* error, or MagicByte Errors are tracked in the same
|
|
1109
|
+
* list of messages as the successful messages.
|
|
1110
|
+
* TODO KIP-932: Check if we need a new op for record
|
|
1111
|
+
* level message errors: RD_KAFKA_OP_CONSUMER_MSG_ERR.
|
|
1112
|
+
*/
|
|
1113
|
+
if (msg_rko->rko_type == RD_KAFKA_OP_FETCH ||
|
|
1114
|
+
msg_rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR) {
|
|
1115
|
+
rd_list_add(
|
|
1116
|
+
response_rko->rko_u.share_fetch_response
|
|
1117
|
+
.message_rkos,
|
|
1118
|
+
msg_rko);
|
|
1119
|
+
msg_cnt++;
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
rd_list_destroy(&partition_msgs);
|
|
1124
|
+
rd_list_add(
|
|
1125
|
+
response_rko->rko_u.share_fetch_response.inflight_acks,
|
|
1126
|
+
rd_kafka_share_ack_batches_copy(batches));
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
if (msg_cnt == 0 && inflight_acks_cnt == 0) {
|
|
1130
|
+
rd_kafka_op_destroy(response_rko);
|
|
1131
|
+
return NULL;
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
|
|
1135
|
+
"Built share fetch response rko with %d messages, "
|
|
1136
|
+
"%d gaps, and %d partitions",
|
|
1137
|
+
msg_cnt, (int32_t)(total_offsets - msg_cnt),
|
|
1138
|
+
inflight_acks_cnt);
|
|
1139
|
+
|
|
1140
|
+
return response_rko;
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
|
|
1144
|
+
/**
|
|
1145
|
+
* @brief Handle a per-partition fetch error from a ShareFetch response.
|
|
1146
|
+
*
|
|
1147
|
+
* @locality broker thread
|
|
1148
|
+
*/
|
|
1149
|
+
static void rd_kafka_share_fetch_reply_handle_partition_error(
|
|
1150
|
+
rd_kafka_broker_t *rkb,
|
|
1151
|
+
rd_kafka_toppar_t *rktp,
|
|
1152
|
+
const rd_kafkap_str_t *topic,
|
|
1153
|
+
int32_t partition,
|
|
1154
|
+
rd_kafka_resp_err_t err,
|
|
1155
|
+
const rd_kafkap_str_t *err_msg) {
|
|
1156
|
+
|
|
1157
|
+
/* TODO KIP-932: Verify whether the SURFACE-only arms below
|
|
1158
|
+
* (CORRUPT_MESSAGE, default unknown err) should also emit an
|
|
1159
|
+
* explicit warn-level log here. They currently rely on the
|
|
1160
|
+
* downstream OP_CONSUMER_ERR being surfaced to the app via
|
|
1161
|
+
* consume_batch. */
|
|
1162
|
+
/* TODO KIP-932: write test cases for each per-partition error
|
|
1163
|
+
* arm below once the mock cluster exposes a per-partition
|
|
1164
|
+
* error-injection API (e.g.
|
|
1165
|
+
* rd_kafka_mock_partition_push_share_fetch_error). Today only
|
|
1166
|
+
* the leader-change errors are exercised via
|
|
1167
|
+
* rd_kafka_mock_partition_set_leader; the remaining arms
|
|
1168
|
+
* (KAFKA_STORAGE_ERROR, OFFSET_NOT_AVAILABLE,
|
|
1169
|
+
* REPLICA_NOT_AVAILABLE, UNKNOWN_TOPIC_OR_PART,
|
|
1170
|
+
* UNKNOWN_TOPIC_ID, INCONSISTENT_TOPIC_ID,
|
|
1171
|
+
* TOPIC_AUTHORIZATION_FAILED, UNKNOWN_LEADER_EPOCH,
|
|
1172
|
+
* UNKNOWN_SERVER_ERROR, CORRUPT_MESSAGE, and the default)
|
|
1173
|
+
* have no deterministic mock trigger. */
|
|
1174
|
+
switch (err) {
|
|
1175
|
+
case RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER:
|
|
1176
|
+
case RD_KAFKA_RESP_ERR_FENCED_LEADER_EPOCH:
|
|
1177
|
+
case RD_KAFKA_RESP_ERR_KAFKA_STORAGE_ERROR:
|
|
1178
|
+
case RD_KAFKA_RESP_ERR_OFFSET_NOT_AVAILABLE:
|
|
1179
|
+
case RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE:
|
|
1180
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
|
|
1181
|
+
"%.*s [%" PRId32
|
|
1182
|
+
"]: ShareFetch failed: %s: %.*s: "
|
|
1183
|
+
"triggering metadata refresh",
|
|
1184
|
+
RD_KAFKAP_STR_PR(topic), partition,
|
|
1185
|
+
rd_kafka_err2name(err), RD_KAFKAP_STR_PR(err_msg));
|
|
1186
|
+
rd_kafka_toppar_leader_unavailable(rktp, "sharefetch", err);
|
|
1187
|
+
break;
|
|
1188
|
+
|
|
1189
|
+
case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART:
|
|
1190
|
+
case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID:
|
|
1191
|
+
case RD_KAFKA_RESP_ERR_INCONSISTENT_TOPIC_ID:
|
|
1192
|
+
/* No per-partition recovery action; left for the next
|
|
1193
|
+
* metadata refresh / heartbeat reconciliation to resolve. */
|
|
1194
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
|
|
1195
|
+
"%.*s [%" PRId32 "]: ShareFetch failed: %s: %.*s",
|
|
1196
|
+
RD_KAFKAP_STR_PR(topic), partition,
|
|
1197
|
+
rd_kafka_err2name(err), RD_KAFKAP_STR_PR(err_msg));
|
|
1198
|
+
break;
|
|
1199
|
+
|
|
1200
|
+
case RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED:
|
|
1201
|
+
rd_rkb_log(rkb, LOG_WARNING, "SHAREFETCH",
|
|
1202
|
+
"%.*s [%" PRId32 "]: Not authorized to read: %.*s",
|
|
1203
|
+
RD_KAFKAP_STR_PR(topic), partition,
|
|
1204
|
+
RD_KAFKAP_STR_PR(err_msg));
|
|
1205
|
+
rd_kafka_consumer_err(
|
|
1206
|
+
rkb->rkb_rk->rk_cgrp->rkcg_q, rd_kafka_broker_id(rkb), err,
|
|
1207
|
+
0, NULL, rktp, RD_KAFKA_OFFSET_INVALID,
|
|
1208
|
+
"ShareFetch failed for %.*s [%" PRId32 "]: %.*s",
|
|
1209
|
+
RD_KAFKAP_STR_PR(topic), partition,
|
|
1210
|
+
RD_KAFKAP_STR_PR(err_msg));
|
|
1211
|
+
break;
|
|
1212
|
+
|
|
1213
|
+
case RD_KAFKA_RESP_ERR_UNKNOWN_LEADER_EPOCH:
|
|
1214
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
|
|
1215
|
+
"%.*s [%" PRId32 "]: ShareFetch failed: %s: %.*s",
|
|
1216
|
+
RD_KAFKAP_STR_PR(topic), partition,
|
|
1217
|
+
rd_kafka_err2name(err), RD_KAFKAP_STR_PR(err_msg));
|
|
1218
|
+
break;
|
|
1219
|
+
|
|
1220
|
+
case RD_KAFKA_RESP_ERR_UNKNOWN:
|
|
1221
|
+
rd_rkb_log(rkb, LOG_WARNING, "SHAREFETCH",
|
|
1222
|
+
"%.*s [%" PRId32 "]: ShareFetch failed: %s: %.*s",
|
|
1223
|
+
RD_KAFKAP_STR_PR(topic), partition,
|
|
1224
|
+
rd_kafka_err2name(err), RD_KAFKAP_STR_PR(err_msg));
|
|
1225
|
+
break;
|
|
1226
|
+
|
|
1227
|
+
case RD_KAFKA_RESP_ERR_INVALID_MSG:
|
|
1228
|
+
rd_kafka_consumer_err(
|
|
1229
|
+
rkb->rkb_rk->rk_cgrp->rkcg_q, rd_kafka_broker_id(rkb), err,
|
|
1230
|
+
0, NULL, rktp, RD_KAFKA_OFFSET_INVALID,
|
|
1231
|
+
"Encountered corrupt message when fetching "
|
|
1232
|
+
"topic-partition %.*s-%" PRId32 ": %.*s",
|
|
1233
|
+
RD_KAFKAP_STR_PR(topic), partition,
|
|
1234
|
+
RD_KAFKAP_STR_PR(err_msg));
|
|
1235
|
+
break;
|
|
1236
|
+
|
|
1237
|
+
default:
|
|
1238
|
+
rd_kafka_consumer_err(
|
|
1239
|
+
rkb->rkb_rk->rk_cgrp->rkcg_q, rd_kafka_broker_id(rkb), err,
|
|
1240
|
+
0, NULL, rktp, RD_KAFKA_OFFSET_INVALID,
|
|
1241
|
+
"Unexpected error code %" PRId16
|
|
1242
|
+
" (%s) while fetching from topic-partition "
|
|
1243
|
+
"%.*s-%" PRId32 ": %.*s",
|
|
1244
|
+
(int16_t)err, rd_kafka_err2name(err),
|
|
1245
|
+
RD_KAFKAP_STR_PR(topic), partition,
|
|
1246
|
+
RD_KAFKAP_STR_PR(err_msg));
|
|
1247
|
+
break;
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
|
|
1252
|
+
/**
|
|
1253
|
+
* @brief Parse a partition from ShareFetch response and build inflight_acks.
|
|
1254
|
+
*
|
|
1255
|
+
* Creates rd_kafka_share_ack_batches_t directly with per-offset tracking.
|
|
1256
|
+
* All offsets are initialized to ACQUIRED state; types will be updated
|
|
1257
|
+
* later when matching with actual messages (GAP for missing, REJECT for
|
|
1258
|
+
* errors).
|
|
1259
|
+
*
|
|
1260
|
+
* @param rkb Broker handle
|
|
1261
|
+
* @param topic Topic name
|
|
1262
|
+
* @param topic_id Topic UUID
|
|
1263
|
+
* @param rkt Topic handle (possibly NULL)
|
|
1264
|
+
* @param rkbuf Response buffer
|
|
1265
|
+
* @param request Request buffer
|
|
1266
|
+
* @param temp_appq Queue to forward filtered messages
|
|
1267
|
+
* @param batches_out Output: inflight acks batches for this partition
|
|
1268
|
+
*
|
|
1269
|
+
* @returns Error code or RD_KAFKA_RESP_ERR_NO_ERROR on success
|
|
1270
|
+
*
|
|
1271
|
+
* TODO KIP-932: Check if we can reduce the number of args in this method.
|
|
1272
|
+
* Can we remove the topic field from the args, since it is a derived value
|
|
1273
|
+
* from rkt and initialize a topic variable inside this function?
|
|
1274
|
+
*/
|
|
1275
|
+
static rd_kafka_resp_err_t rd_kafka_share_fetch_reply_handle_partition(
|
|
1276
|
+
rd_kafka_broker_t *rkb,
|
|
1277
|
+
const rd_kafkap_str_t *topic,
|
|
1278
|
+
rd_kafka_Uuid_t topic_id,
|
|
1279
|
+
rd_kafka_topic_t *rkt /*possibly NULL*/,
|
|
1280
|
+
rd_kafka_buf_t *rkbuf,
|
|
1281
|
+
rd_kafka_buf_t *request,
|
|
1282
|
+
rd_list_t *filtered_msgs,
|
|
1283
|
+
rd_kafka_share_ack_batches_t *batches_out,
|
|
1284
|
+
rd_list_t *request_ack_details) {
|
|
1285
|
+
|
|
1286
|
+
int32_t PartitionId;
|
|
1287
|
+
int16_t PartitionFetchErrorCode;
|
|
1288
|
+
rd_kafkap_str_t PartitionFetchErrorStr =
|
|
1289
|
+
RD_KAFKAP_STR_INITIALIZER_EMPTY;
|
|
1290
|
+
int16_t AcknowledgementErrorCode;
|
|
1291
|
+
rd_kafkap_str_t AcknowledgementErrorStr =
|
|
1292
|
+
RD_KAFKAP_STR_INITIALIZER_EMPTY;
|
|
1293
|
+
rd_kafkap_CurrentLeader_t CurrentLeader;
|
|
1294
|
+
int32_t MessageSetSize;
|
|
1295
|
+
rd_kafka_toppar_t *rktp = NULL;
|
|
1296
|
+
struct rd_kafka_toppar_ver tver;
|
|
1297
|
+
rd_slice_t save_slice;
|
|
1298
|
+
const int log_decode_errors = LOG_ERR;
|
|
1299
|
+
rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
1300
|
+
int32_t AcquiredRecordsArrayCnt;
|
|
1301
|
+
int64_t *FirstOffsets = NULL;
|
|
1302
|
+
int64_t *LastOffsets = NULL;
|
|
1303
|
+
int16_t *DeliveryCounts = NULL;
|
|
1304
|
+
rd_kafka_q_t *temp_fetchq = rd_kafka_q_new(rkb->rkb_rk);
|
|
1305
|
+
int i;
|
|
1306
|
+
rd_bool_t is_sorted = rd_true;
|
|
1307
|
+
int64_t prev_end_offset = -1, size, j;
|
|
1308
|
+
rd_kafka_share_ack_batch_entry_t *entry;
|
|
1309
|
+
rd_kafka_topic_partition_private_t *parpriv;
|
|
1310
|
+
char *topic_str;
|
|
1311
|
+
|
|
1312
|
+
rd_kafka_buf_read_i32(rkbuf, &PartitionId); // Partition
|
|
1313
|
+
rd_kafka_buf_read_i16(rkbuf,
|
|
1314
|
+
&PartitionFetchErrorCode); // PartitionFetchError
|
|
1315
|
+
rd_kafka_buf_read_str(rkbuf, &PartitionFetchErrorStr); // ErrorString
|
|
1316
|
+
rd_kafka_buf_read_i16(
|
|
1317
|
+
rkbuf, &AcknowledgementErrorCode); // AcknowledgementError
|
|
1318
|
+
rd_kafka_buf_read_str(
|
|
1319
|
+
rkbuf, &AcknowledgementErrorStr); // AcknowledgementErrorString
|
|
1320
|
+
|
|
1321
|
+
/* Set the AcknowledgementErrorCode on the matching ack batch
|
|
1322
|
+
* in request_ack_details, so the main thread can map
|
|
1323
|
+
* per-partition ack errors to commit_sync results /
|
|
1324
|
+
* acknowledgement callback. PartitionFetchErrorCode is for
|
|
1325
|
+
* fetch-level errors and is handled separately. The partition
|
|
1326
|
+
* may be present in the response but absent from
|
|
1327
|
+
* request_ack_details (e.g. fetch-only partitions from
|
|
1328
|
+
* toppars_to_add) — silently skip those.
|
|
1329
|
+
*
|
|
1330
|
+
* Conditional on _IN_PROGRESS preserves any deliberately
|
|
1331
|
+
* pre-set err on the batch (e.g. INVALID_SHARE_SESSION_EPOCH
|
|
1332
|
+
* from the epoch-0 strip path) — those partitions were never
|
|
1333
|
+
* actually sent to the broker, so the broker's
|
|
1334
|
+
* AcknowledgementErrorCode (typically NO_ERROR) is not the
|
|
1335
|
+
* authoritative result for them. */
|
|
1336
|
+
if (request_ack_details) {
|
|
1337
|
+
rd_kafka_share_ack_batches_t *batch =
|
|
1338
|
+
rd_kafka_share_find_ack_batch_by_id(request_ack_details,
|
|
1339
|
+
topic_id, PartitionId);
|
|
1340
|
+
if (batch &&
|
|
1341
|
+
batch->rktpar->err == RD_KAFKA_RESP_ERR__IN_PROGRESS)
|
|
1342
|
+
batch->rktpar->err = AcknowledgementErrorCode;
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1345
|
+
if (AcknowledgementErrorCode != RD_KAFKA_RESP_ERR_NO_ERROR) {
|
|
1346
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREACK",
|
|
1347
|
+
"ShareFetch response for %.*s [%" PRId32
|
|
1348
|
+
"]: AcknowledgementError %" PRId16 " (%s)",
|
|
1349
|
+
RD_KAFKAP_STR_PR(topic), PartitionId,
|
|
1350
|
+
AcknowledgementErrorCode,
|
|
1351
|
+
rd_kafka_err2str(AcknowledgementErrorCode));
|
|
1352
|
+
}
|
|
1353
|
+
|
|
1354
|
+
rd_kafka_buf_read_CurrentLeader(rkbuf,
|
|
1355
|
+
&CurrentLeader); // CurrentLeader
|
|
1356
|
+
|
|
1357
|
+
/* Compact Records Array */
|
|
1358
|
+
rd_kafka_buf_read_arraycnt(rkbuf, &MessageSetSize, -1);
|
|
1359
|
+
|
|
1360
|
+
if (unlikely(MessageSetSize < 0))
|
|
1361
|
+
rd_kafka_buf_parse_fail(
|
|
1362
|
+
rkbuf,
|
|
1363
|
+
"%.*s [%" PRId32 "]: invalid MessageSetSize %" PRId32,
|
|
1364
|
+
RD_KAFKAP_STR_PR(topic), PartitionId, MessageSetSize);
|
|
1365
|
+
|
|
1366
|
+
/* Look up topic+partition */
|
|
1367
|
+
if (likely(rkt != NULL)) {
|
|
1368
|
+
rd_kafka_topic_rdlock(rkt);
|
|
1369
|
+
rktp =
|
|
1370
|
+
rd_kafka_toppar_get(rkt, PartitionId, 0 /*no ua-on-miss*/);
|
|
1371
|
+
rd_kafka_topic_rdunlock(rkt);
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
if (unlikely(!rkt || !rktp || PartitionFetchErrorCode)) {
|
|
1375
|
+
int64_t tmp_first, tmp_last;
|
|
1376
|
+
int16_t tmp_delivery;
|
|
1377
|
+
|
|
1378
|
+
if (!rkt) {
|
|
1379
|
+
/* TODO KIP-932: Recheck this branch once the
|
|
1380
|
+
* topic-recreate session-bookkeeping bug is fixed
|
|
1381
|
+
* (broker session may stop carrying old topic_ids
|
|
1382
|
+
* that no longer match any local rkt). */
|
|
1383
|
+
rd_rkb_dbg(rkb, TOPIC, "UNKTOPIC",
|
|
1384
|
+
"Received Fetch response (error %hu) for "
|
|
1385
|
+
"unknown topic %.*s [%" PRId32 "]: ignoring",
|
|
1386
|
+
PartitionFetchErrorCode,
|
|
1387
|
+
RD_KAFKAP_STR_PR(topic), PartitionId);
|
|
1388
|
+
} else if (!rktp) {
|
|
1389
|
+
/* TODO KIP-932: Verify this handling against expected
|
|
1390
|
+
* behavior. librdkafka requires rktp to route records;
|
|
1391
|
+
* partitions with no local rktp are silently dropped.
|
|
1392
|
+
*/
|
|
1393
|
+
rd_rkb_dbg(rkb, TOPIC, "UNKTOPIC",
|
|
1394
|
+
"Received Fetch response (error %hu) for "
|
|
1395
|
+
"unknown partition %.*s [%" PRId32
|
|
1396
|
+
"]: ignoring",
|
|
1397
|
+
PartitionFetchErrorCode,
|
|
1398
|
+
RD_KAFKAP_STR_PR(topic), PartitionId);
|
|
1399
|
+
} else {
|
|
1400
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
|
|
1401
|
+
"%.*s [%" PRId32
|
|
1402
|
+
"]: per-partition fetch error %s",
|
|
1403
|
+
RD_KAFKAP_STR_PR(topic), PartitionId,
|
|
1404
|
+
rd_kafka_err2name(PartitionFetchErrorCode));
|
|
1405
|
+
rd_kafka_share_fetch_reply_handle_partition_error(
|
|
1406
|
+
rkb, rktp, topic, PartitionId,
|
|
1407
|
+
PartitionFetchErrorCode, &PartitionFetchErrorStr);
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1410
|
+
rd_kafka_buf_skip(rkbuf, MessageSetSize);
|
|
1411
|
+
/* TODO KIP-932: Consider tracking total byte size of the
|
|
1412
|
+
* AcquiredRecords array in the protocol to allow a single
|
|
1413
|
+
* rd_kafka_buf_skip() here instead of per-entry parsing. */
|
|
1414
|
+
rd_kafka_buf_read_arraycnt(rkbuf, &AcquiredRecordsArrayCnt,
|
|
1415
|
+
-1); // AcquiredRecordsArrayCnt
|
|
1416
|
+
for (i = 0; i < AcquiredRecordsArrayCnt; i++) {
|
|
1417
|
+
rd_kafka_buf_read_i64(rkbuf,
|
|
1418
|
+
&tmp_first); // FirstOffset
|
|
1419
|
+
rd_kafka_buf_read_i64(rkbuf, &tmp_last); // LastOffset
|
|
1420
|
+
rd_kafka_buf_read_i16(rkbuf,
|
|
1421
|
+
&tmp_delivery); // DeliveryCount
|
|
1422
|
+
rd_kafka_buf_skip_tags(rkbuf); // AcquiredRecords tags
|
|
1423
|
+
}
|
|
1424
|
+
/* No records to track for this partition. Leave
|
|
1425
|
+
* batches_out->rktpar as NULL so the caller skips
|
|
1426
|
+
* adding this batch to inflight_acks. */
|
|
1427
|
+
rd_kafka_buf_skip_tags(rkbuf);
|
|
1428
|
+
goto done;
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1431
|
+
tver.rktp = rktp;
|
|
1432
|
+
/* There is no versioning/barrier of the records/partitions in
|
|
1433
|
+
* share consumer case. */
|
|
1434
|
+
tver.version = 0;
|
|
1435
|
+
|
|
1436
|
+
if (MessageSetSize > 0) {
|
|
1437
|
+
/**
|
|
1438
|
+
* Parse MessageSet
|
|
1439
|
+
*/
|
|
1440
|
+
if (!rd_slice_narrow_relative(&rkbuf->rkbuf_reader, &save_slice,
|
|
1441
|
+
(size_t)MessageSetSize))
|
|
1442
|
+
rd_kafka_buf_check_len(rkbuf, MessageSetSize);
|
|
1443
|
+
|
|
1444
|
+
/*
|
|
1445
|
+
* Parse messages
|
|
1446
|
+
*/
|
|
1447
|
+
err = rd_kafka_share_msgset_parse(rkbuf, rktp, NULL, &tver,
|
|
1448
|
+
temp_fetchq);
|
|
1449
|
+
|
|
1450
|
+
rd_slice_widen(&rkbuf->rkbuf_reader, &save_slice);
|
|
1451
|
+
/* Continue with next partition regardless of
|
|
1452
|
+
* parse errors (which are partition-specific) */
|
|
1453
|
+
|
|
1454
|
+
/**
|
|
1455
|
+
* Only inner-record parsing errors (RD_KAFKA_RESP_ERR__BAD_MSG
|
|
1456
|
+
* / RD_KAFKA_RESP_ERR__UNDERFLOW) propagate out of
|
|
1457
|
+
* rd_kafka_share_msgset_parse for share consumers — per-batch
|
|
1458
|
+
* codec / CRC / unsupported-MagicByte failures are handled
|
|
1459
|
+
* inline by emitting per-offset RELEASE/REJECT ops and are
|
|
1460
|
+
* swallowed by the v2 reader (msgset_reader.c). When a true
|
|
1461
|
+
* parse error bubbles up the wire data is corrupt and we can no
|
|
1462
|
+
* longer trust the buffer position or any per-batch metadata.
|
|
1463
|
+
* Stop processing the rest of this ShareFetch response and
|
|
1464
|
+
* propagate the error to the caller, which aborts the whole
|
|
1465
|
+
* response handling.
|
|
1466
|
+
*/
|
|
1467
|
+
if (err)
|
|
1468
|
+
goto done;
|
|
1469
|
+
}
|
|
1470
|
+
|
|
1471
|
+
rd_kafka_buf_read_arraycnt(rkbuf, &AcquiredRecordsArrayCnt,
|
|
1472
|
+
-1); // AcquiredRecordsArrayCnt
|
|
1473
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
|
|
1474
|
+
"%.*s [%" PRId32 "] : AcquiredRecordsArrayCnt: %d",
|
|
1475
|
+
RD_KAFKAP_STR_PR(topic), PartitionId,
|
|
1476
|
+
AcquiredRecordsArrayCnt);
|
|
1477
|
+
|
|
1478
|
+
/* Allocate and initialize the topic partition */
|
|
1479
|
+
topic_str = RD_KAFKAP_STR_DUP(topic);
|
|
1480
|
+
batches_out->rktpar =
|
|
1481
|
+
rd_kafka_topic_partition_new(topic_str, PartitionId);
|
|
1482
|
+
rd_free(topic_str);
|
|
1483
|
+
|
|
1484
|
+
/* Allocate and fill the private structure */
|
|
1485
|
+
parpriv = rd_kafka_topic_partition_private_new();
|
|
1486
|
+
parpriv->rktp = rd_kafka_toppar_keep(rktp);
|
|
1487
|
+
parpriv->topic_id = topic_id;
|
|
1488
|
+
batches_out->rktpar->_private = parpriv;
|
|
1489
|
+
|
|
1490
|
+
/* Record the broker from which records were acquired so the
|
|
1491
|
+
* acknowledgement is sent back to the same broker. The wire
|
|
1492
|
+
* CurrentLeader hint is only set when the broker signals a
|
|
1493
|
+
* leader change, so we use the responding broker. */
|
|
1494
|
+
batches_out->response_leader_id = rkb->rkb_nodeid;
|
|
1495
|
+
batches_out->response_acquired_offsets_count = 0;
|
|
1496
|
+
/* Pre-allocate capacity without re-initializing the list.
|
|
1497
|
+
* batches_out->entries was already initialized by
|
|
1498
|
+
* rd_kafka_share_ack_batches_new() with the proper
|
|
1499
|
+
* entry destructor. */
|
|
1500
|
+
if (AcquiredRecordsArrayCnt > 0)
|
|
1501
|
+
rd_list_grow(&batches_out->entries, AcquiredRecordsArrayCnt);
|
|
1502
|
+
|
|
1503
|
+
if (AcquiredRecordsArrayCnt > 0) {
|
|
1504
|
+
FirstOffsets =
|
|
1505
|
+
rd_malloc(sizeof(*FirstOffsets) * AcquiredRecordsArrayCnt);
|
|
1506
|
+
LastOffsets =
|
|
1507
|
+
rd_malloc(sizeof(*LastOffsets) * AcquiredRecordsArrayCnt);
|
|
1508
|
+
DeliveryCounts = rd_malloc(sizeof(*DeliveryCounts) *
|
|
1509
|
+
AcquiredRecordsArrayCnt);
|
|
1510
|
+
|
|
1511
|
+
/**
|
|
1512
|
+
* TODO KIP-932: There could be an improvement where we
|
|
1513
|
+
* segregate the records while reading the FirstOffsets and
|
|
1514
|
+
* LastOffsets, so that we can avoid mallocs above and directly
|
|
1515
|
+
* create the batches and fill the entries.
|
|
1516
|
+
*/
|
|
1517
|
+
for (i = 0; i < AcquiredRecordsArrayCnt; i++) {
|
|
1518
|
+
rd_kafka_buf_read_i64(rkbuf, &FirstOffsets[i]);
|
|
1519
|
+
rd_kafka_buf_read_i64(rkbuf, &LastOffsets[i]);
|
|
1520
|
+
rd_kafka_buf_read_i16(rkbuf, &DeliveryCounts[i]);
|
|
1521
|
+
rd_kafka_buf_skip_tags(rkbuf);
|
|
1522
|
+
|
|
1523
|
+
size = LastOffsets[i] - FirstOffsets[i] + 1;
|
|
1524
|
+
|
|
1525
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
|
|
1526
|
+
"%.*s [%" PRId32
|
|
1527
|
+
"]: Acquired Records from offset %" PRId64
|
|
1528
|
+
" to %" PRId64 ", DeliveryCount %" PRId16,
|
|
1529
|
+
RD_KAFKAP_STR_PR(topic), PartitionId,
|
|
1530
|
+
FirstOffsets[i], LastOffsets[i],
|
|
1531
|
+
DeliveryCounts[i]);
|
|
1532
|
+
|
|
1533
|
+
/* Track sortedness: entries must be non-overlapping
|
|
1534
|
+
* and in ascending order by start_offset. */
|
|
1535
|
+
if (is_sorted && FirstOffsets[i] <= prev_end_offset)
|
|
1536
|
+
is_sorted = rd_false;
|
|
1537
|
+
prev_end_offset = LastOffsets[i];
|
|
1538
|
+
|
|
1539
|
+
entry = rd_kafka_share_ack_batch_entry_new(
|
|
1540
|
+
FirstOffsets[i], LastOffsets[i], (int32_t)size,
|
|
1541
|
+
DeliveryCounts[i]);
|
|
1542
|
+
|
|
1543
|
+
/* Initialize all offsets to GAP; build_response_rko
|
|
1544
|
+
* will set ACQUIRED/REJECT for offsets that have a
|
|
1545
|
+
* message. */
|
|
1546
|
+
for (j = 0; j < size; j++) {
|
|
1547
|
+
entry->types[j] =
|
|
1548
|
+
RD_KAFKA_SHARE_INTERNAL_ACK_GAP;
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
rd_list_add(&batches_out->entries, entry);
|
|
1552
|
+
batches_out->response_acquired_offsets_count +=
|
|
1553
|
+
(int32_t)size;
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1556
|
+
/* Mark as sorted to enable binary search in rd_list_find(). */
|
|
1557
|
+
if (is_sorted)
|
|
1558
|
+
batches_out->entries.rl_flags |= RD_LIST_F_SORTED;
|
|
1559
|
+
|
|
1560
|
+
/* Filter and forward messages in acquired ranges */
|
|
1561
|
+
rd_kafka_share_filter_acquired_records_and_update_ack_type(
|
|
1562
|
+
temp_fetchq, filtered_msgs, FirstOffsets, LastOffsets,
|
|
1563
|
+
DeliveryCounts, AcquiredRecordsArrayCnt);
|
|
1564
|
+
} else {
|
|
1565
|
+
/* No acquired ranges: drop everything */
|
|
1566
|
+
rd_kafka_op_t *rko;
|
|
1567
|
+
while ((rko = rd_kafka_q_pop(temp_fetchq, RD_POLL_NOWAIT, 0)))
|
|
1568
|
+
rd_kafka_op_destroy(rko);
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1571
|
+
rd_kafka_buf_skip_tags(rkbuf); // Partition tags
|
|
1572
|
+
|
|
1573
|
+
goto done;
|
|
1574
|
+
|
|
1575
|
+
err_parse:
|
|
1576
|
+
err = rkbuf->rkbuf_err;
|
|
1577
|
+
|
|
1578
|
+
done:
|
|
1579
|
+
RD_IF_FREE(FirstOffsets, rd_free);
|
|
1580
|
+
RD_IF_FREE(LastOffsets, rd_free);
|
|
1581
|
+
RD_IF_FREE(DeliveryCounts, rd_free);
|
|
1582
|
+
RD_IF_FREE(temp_fetchq, rd_kafka_q_destroy_owner);
|
|
1583
|
+
if (rktp)
|
|
1584
|
+
rd_kafka_toppar_destroy(rktp); /* from toppar_get() */
|
|
1585
|
+
return err;
|
|
1586
|
+
}
|
|
1587
|
+
|
|
1588
|
+
|
|
1589
|
+
/**
|
|
1590
|
+
* Parses and handles a ShareFetch reply.
|
|
1591
|
+
* Returns 0 on success or an error code on failure.
|
|
1592
|
+
*
|
|
1593
|
+
* TODO KIP-932: Change return type to proper error with message. See
|
|
1594
|
+
* `rd_kafka_error_t *`.
|
|
1595
|
+
*/
|
|
1596
|
+
static rd_kafka_resp_err_t
|
|
1597
|
+
rd_kafka_share_fetch_reply_handle(rd_kafka_broker_t *rkb,
|
|
1598
|
+
rd_kafka_buf_t *rkbuf,
|
|
1599
|
+
rd_kafka_buf_t *request,
|
|
1600
|
+
rd_kafka_op_t **response_rko_out) {
|
|
1601
|
+
int32_t TopicArrayCnt;
|
|
1602
|
+
int i;
|
|
1603
|
+
const int log_decode_errors = LOG_ERR;
|
|
1604
|
+
rd_kafka_topic_t *rkt = NULL;
|
|
1605
|
+
int16_t ErrorCode = RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
1606
|
+
rd_kafkap_str_t ErrorStr = RD_KAFKAP_STR_INITIALIZER_EMPTY;
|
|
1607
|
+
int32_t AcquisitionLockTimeoutMs = 0;
|
|
1608
|
+
rd_kafkap_NodeEndpoints_t NodeEndpoints;
|
|
1609
|
+
NodeEndpoints.NodeEndpoints = NULL;
|
|
1610
|
+
NodeEndpoints.NodeEndpointCnt = 0;
|
|
1611
|
+
rd_list_t *filtered_msgs = NULL;
|
|
1612
|
+
rd_list_t *inflight_acks = NULL;
|
|
1613
|
+
rd_kafka_op_t *rko_orig = request->rkbuf_opaque;
|
|
1614
|
+
rd_kafka_op_t *response_rko = NULL;
|
|
1615
|
+
rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
1616
|
+
rd_kafka_op_t *fetch_size_rko;
|
|
1617
|
+
int fetch_size_idx;
|
|
1618
|
+
int64_t total_fetch_size_bytes = 0;
|
|
1619
|
+
|
|
1620
|
+
rd_kafka_buf_read_throttle_time(rkbuf);
|
|
1621
|
+
|
|
1622
|
+
rd_kafka_buf_read_i16(rkbuf, &ErrorCode);
|
|
1623
|
+
rd_kafka_buf_read_str(rkbuf, &ErrorStr);
|
|
1624
|
+
|
|
1625
|
+
if (ErrorCode) {
|
|
1626
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
|
|
1627
|
+
"ShareFetch response error %s: '%.*s'",
|
|
1628
|
+
rd_kafka_err2name(ErrorCode),
|
|
1629
|
+
RD_KAFKAP_STR_PR(&ErrorStr));
|
|
1630
|
+
return ErrorCode;
|
|
1631
|
+
}
|
|
1632
|
+
|
|
1633
|
+
/* Count this successful ShareFetch response for
|
|
1634
|
+
* consumer.share.fetch.manager.fetch.{total,rate}.
|
|
1635
|
+
* Fires only after top level error handling.
|
|
1636
|
+
*
|
|
1637
|
+
* TODO KIP-932: Revisit this sampling before GA. It fires here,
|
|
1638
|
+
* before the message set is parsed (rd_kafka_share_msgset_parse in
|
|
1639
|
+
* the partition loop below). If that parse fails with
|
|
1640
|
+
* RD_KAFKA_RESP_ERR__BAD_MSG / __UNDERFLOW the caller drops the
|
|
1641
|
+
* whole response, yet fetch.{total,rate} and fetch.latency have
|
|
1642
|
+
* already been counted here. */
|
|
1643
|
+
rd_atomic64_add(&rkb->rkb_rk->rk_telemetry.share_fetch_total, 1);
|
|
1644
|
+
|
|
1645
|
+
/* rkbuf_ts_sent is already the request RTT here */
|
|
1646
|
+
rd_avg_add(
|
|
1647
|
+
&rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_latency,
|
|
1648
|
+
request->rkbuf_ts_sent);
|
|
1649
|
+
|
|
1650
|
+
rd_kafka_buf_read_i32(rkbuf, &AcquisitionLockTimeoutMs);
|
|
1651
|
+
|
|
1652
|
+
rd_kafka_buf_read_arraycnt(rkbuf, &TopicArrayCnt, RD_KAFKAP_TOPICS_MAX);
|
|
1653
|
+
|
|
1654
|
+
filtered_msgs = rd_list_new(0, NULL);
|
|
1655
|
+
inflight_acks = rd_list_new(0, rd_kafka_share_ack_batches_destroy_free);
|
|
1656
|
+
|
|
1657
|
+
for (i = 0; i < TopicArrayCnt; i++) {
|
|
1658
|
+
rd_kafkap_str_t topic = RD_ZERO_INIT;
|
|
1659
|
+
rd_kafka_Uuid_t topic_id = RD_KAFKA_UUID_ZERO;
|
|
1660
|
+
int32_t PartitionArrayCnt;
|
|
1661
|
+
int j;
|
|
1662
|
+
|
|
1663
|
+
rd_kafka_buf_read_uuid(rkbuf, &topic_id);
|
|
1664
|
+
rkt = rd_kafka_topic_find_by_topic_id(rkb->rkb_rk, topic_id);
|
|
1665
|
+
if (rkt)
|
|
1666
|
+
topic = *rkt->rkt_topic;
|
|
1667
|
+
|
|
1668
|
+
rd_kafka_buf_read_arraycnt(rkbuf, &PartitionArrayCnt,
|
|
1669
|
+
RD_KAFKAP_PARTITIONS_MAX);
|
|
1670
|
+
|
|
1671
|
+
for (j = 0; j < PartitionArrayCnt; j++) {
|
|
1672
|
+
rd_kafka_share_ack_batches_t *batches =
|
|
1673
|
+
rd_kafka_share_ack_batches_new_empty();
|
|
1674
|
+
|
|
1675
|
+
err = rd_kafka_share_fetch_reply_handle_partition(
|
|
1676
|
+
rkb, &topic, topic_id, rkt, rkbuf, request,
|
|
1677
|
+
filtered_msgs, batches,
|
|
1678
|
+
rko_orig->rko_u.share_fetch.ack_details);
|
|
1679
|
+
|
|
1680
|
+
/*
|
|
1681
|
+
* Only inner-record parsing errors
|
|
1682
|
+
* (RD_KAFKA_RESP_ERR__BAD_MSG /
|
|
1683
|
+
* RD_KAFKA_RESP_ERR__UNDERFLOW) reach this point —
|
|
1684
|
+
* per-batch CRC / decompression / MagicByte failures
|
|
1685
|
+
* are handled inline inside
|
|
1686
|
+
* rd_kafka_share_msgset_parse by emitting per-offset
|
|
1687
|
+
* RELEASE/REJECT ops and are swallowed, and
|
|
1688
|
+
* per-partition broker errors are handled by
|
|
1689
|
+
* rd_kafka_share_fetch_reply_handle_partition_error
|
|
1690
|
+
* before MessageSet parsing begins. A parsing error
|
|
1691
|
+
* means the wire data is corrupt and the buffer
|
|
1692
|
+
* position can no longer be trusted for subsequent
|
|
1693
|
+
* partitions, so abort the whole response.
|
|
1694
|
+
*/
|
|
1695
|
+
if (err) {
|
|
1696
|
+
rd_kafka_share_ack_batches_destroy(batches);
|
|
1697
|
+
goto done;
|
|
1698
|
+
}
|
|
1699
|
+
|
|
1700
|
+
/* Skip unknown topics - don't add to inflight_acks */
|
|
1701
|
+
if (batches->rktpar == NULL) {
|
|
1702
|
+
rd_kafka_share_ack_batches_destroy(batches);
|
|
1703
|
+
continue;
|
|
1704
|
+
}
|
|
1705
|
+
|
|
1706
|
+
if (rd_list_cnt(&batches->entries) > 0) {
|
|
1707
|
+
rd_list_add(inflight_acks, batches);
|
|
1708
|
+
} else {
|
|
1709
|
+
rd_kafka_share_ack_batches_destroy(batches);
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
if (rkt) {
|
|
1714
|
+
rd_kafka_topic_destroy0(rkt);
|
|
1715
|
+
rkt = NULL;
|
|
1716
|
+
}
|
|
1717
|
+
/* Topic Tags */
|
|
1718
|
+
rd_kafka_buf_skip_tags(rkbuf);
|
|
1719
|
+
}
|
|
1720
|
+
|
|
1721
|
+
rd_kafka_buf_read_NodeEndpoints(rkbuf, &NodeEndpoints);
|
|
1722
|
+
|
|
1723
|
+
/* Sum per-record wire bytes across acquired records for
|
|
1724
|
+
* consumer.share.fetch.manager.fetch.size.{avg,max} and
|
|
1725
|
+
* .bytes.consumed.{total,rate}. */
|
|
1726
|
+
RD_LIST_FOREACH(fetch_size_rko, filtered_msgs, fetch_size_idx) {
|
|
1727
|
+
if (fetch_size_rko->rko_type == RD_KAFKA_OP_FETCH) {
|
|
1728
|
+
rd_kafka_msg_t *rkm = &fetch_size_rko->rko_u.fetch.rkm;
|
|
1729
|
+
total_fetch_size_bytes +=
|
|
1730
|
+
(int64_t)rkm->rkm_u.consumer.wire_size;
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
/* Update bytes per fetch metrics */
|
|
1734
|
+
rd_avg_add(&rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_size,
|
|
1735
|
+
total_fetch_size_bytes);
|
|
1736
|
+
|
|
1737
|
+
/* Update total bytes consumed */
|
|
1738
|
+
rd_atomic64_add(&rkb->rkb_rk->rk_telemetry.share_bytes_consumed_total,
|
|
1739
|
+
total_fetch_size_bytes);
|
|
1740
|
+
|
|
1741
|
+
/* Build response rko with messages and inflight_acks */
|
|
1742
|
+
response_rko = rd_kafka_share_build_response_rko(rkb, filtered_msgs,
|
|
1743
|
+
inflight_acks);
|
|
1744
|
+
|
|
1745
|
+
rd_list_destroy(inflight_acks);
|
|
1746
|
+
inflight_acks = NULL;
|
|
1747
|
+
rd_list_destroy(filtered_msgs);
|
|
1748
|
+
filtered_msgs = NULL;
|
|
1749
|
+
|
|
1750
|
+
/* Top level tags */
|
|
1751
|
+
rd_kafka_buf_skip_tags(rkbuf);
|
|
1752
|
+
|
|
1753
|
+
/* Return response_rko to the caller instead of enqueueing here.
|
|
1754
|
+
* The caller enqueues it on rkcg_q AFTER sending the reply to
|
|
1755
|
+
* rk_ops, ensuring the main thread processes the reply (and
|
|
1756
|
+
* resets share_fetch_more_records) before the app thread can
|
|
1757
|
+
* wake up, poll again, and enqueue a new FANOUT. */
|
|
1758
|
+
*response_rko_out = response_rko;
|
|
1759
|
+
|
|
1760
|
+
/* Signal main thread whether records were fetched */
|
|
1761
|
+
if (rko_orig)
|
|
1762
|
+
rko_orig->rko_u.share_fetch.records_fetched =
|
|
1763
|
+
response_rko ? rd_true : rd_false;
|
|
1764
|
+
|
|
1765
|
+
RD_IF_FREE(NodeEndpoints.NodeEndpoints, rd_free);
|
|
1766
|
+
RD_IF_FREE(rkt, rd_kafka_topic_destroy0);
|
|
1767
|
+
return RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
1768
|
+
|
|
1769
|
+
err_parse:
|
|
1770
|
+
err = rkbuf->rkbuf_err;
|
|
1771
|
+
|
|
1772
|
+
done:
|
|
1773
|
+
/* Lists are NULLed on the success path after their inline destroy;
|
|
1774
|
+
* here they're only non-NULL on the error path. filtered_msgs uses a
|
|
1775
|
+
* NULL destructor, so any ops accumulated from earlier partitions
|
|
1776
|
+
* before err_parse fired must be destroyed explicitly. */
|
|
1777
|
+
RD_IF_FREE(inflight_acks, rd_list_destroy);
|
|
1778
|
+
if (filtered_msgs) {
|
|
1779
|
+
rd_kafka_op_t *rko;
|
|
1780
|
+
int li;
|
|
1781
|
+
RD_LIST_FOREACH(rko, filtered_msgs, li)
|
|
1782
|
+
rd_kafka_op_destroy(rko);
|
|
1783
|
+
rd_list_destroy(filtered_msgs);
|
|
1784
|
+
}
|
|
1785
|
+
|
|
1786
|
+
RD_IF_FREE(response_rko, rd_kafka_op_destroy);
|
|
1787
|
+
*response_rko_out = NULL;
|
|
1788
|
+
|
|
1789
|
+
if (rkt)
|
|
1790
|
+
rd_kafka_topic_destroy0(rkt);
|
|
1791
|
+
rd_rkb_dbg(rkb, MSG, "BADMSG", "Bad message (ShareFetch v%d): %s",
|
|
1792
|
+
(int)request->rkbuf_reqhdr.ApiVersion,
|
|
1793
|
+
rd_kafka_err2str(err));
|
|
1794
|
+
return err;
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
|
|
1798
|
+
/**
|
|
1799
|
+
* @brief Reset the share fetch session for a broker.
|
|
1800
|
+
*
|
|
1801
|
+
* Called when the broker returns SHARE_SESSION_NOT_FOUND,
|
|
1802
|
+
* INVALID_SHARE_SESSION_EPOCH, or SHARE_SESSION_LIMIT_REACHED,
|
|
1803
|
+
* indicating the session is lost or cannot be created.
|
|
1804
|
+
* Resets the epoch to 0 and moves all toppars_in_session back
|
|
1805
|
+
* to toppars_to_add so the next request re-establishes the
|
|
1806
|
+
* full session.
|
|
1807
|
+
*
|
|
1808
|
+
* @param rkb Broker whose session is being reset.
|
|
1809
|
+
*
|
|
1810
|
+
* @locality broker thread
|
|
1811
|
+
*/
|
|
1812
|
+
static void rd_kafka_broker_session_reset(rd_kafka_broker_t *rkb) {
|
|
1813
|
+
rd_kafka_toppar_t *rktp;
|
|
1814
|
+
int i;
|
|
1815
|
+
|
|
1816
|
+
rd_rkb_dbg(
|
|
1817
|
+
rkb, FETCH, "SHARESESSION",
|
|
1818
|
+
"Resetting share session: epoch %" PRId32
|
|
1819
|
+
" -> 0, "
|
|
1820
|
+
"moving %d toppars_in_session to toppars_to_add",
|
|
1821
|
+
rkb->rkb_share_fetch_session.epoch,
|
|
1822
|
+
rd_list_cnt(rkb->rkb_share_fetch_session.toppars_in_session));
|
|
1823
|
+
|
|
1824
|
+
rkb->rkb_share_fetch_session.epoch = 0;
|
|
1825
|
+
|
|
1826
|
+
/* Remove toppars_to_forget from toppars_in_session — these
|
|
1827
|
+
* were pending removal and should not be re-added to the
|
|
1828
|
+
* new session. */
|
|
1829
|
+
if (rkb->rkb_share_fetch_session.toppars_to_forget) {
|
|
1830
|
+
rd_kafka_toppar_t *forget_rktp;
|
|
1831
|
+
|
|
1832
|
+
RD_LIST_FOREACH(forget_rktp,
|
|
1833
|
+
rkb->rkb_share_fetch_session.toppars_to_forget,
|
|
1834
|
+
i) {
|
|
1835
|
+
rktp = rd_list_remove(
|
|
1836
|
+
rkb->rkb_share_fetch_session.toppars_in_session,
|
|
1837
|
+
forget_rktp);
|
|
1838
|
+
if (rktp)
|
|
1839
|
+
rd_kafka_toppar_destroy(rktp);
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1842
|
+
rd_list_destroy(rkb->rkb_share_fetch_session.toppars_to_forget);
|
|
1843
|
+
rkb->rkb_share_fetch_session.toppars_to_forget = NULL;
|
|
1844
|
+
}
|
|
1845
|
+
|
|
1846
|
+
/* Move remaining toppars_in_session to toppars_to_add so they
|
|
1847
|
+
* get sent as new additions in the next request (epoch 0). */
|
|
1848
|
+
if (!rkb->rkb_share_fetch_session.toppars_to_add) {
|
|
1849
|
+
rkb->rkb_share_fetch_session.toppars_to_add =
|
|
1850
|
+
rkb->rkb_share_fetch_session.toppars_in_session;
|
|
1851
|
+
rkb->rkb_share_fetch_session.toppars_in_session =
|
|
1852
|
+
rd_list_new(0, rd_kafka_toppar_destroy_free);
|
|
1853
|
+
} else {
|
|
1854
|
+
while ((rktp = rd_list_pop(
|
|
1855
|
+
rkb->rkb_share_fetch_session.toppars_in_session))) {
|
|
1856
|
+
if (!rd_list_find(
|
|
1857
|
+
rkb->rkb_share_fetch_session.toppars_to_add,
|
|
1858
|
+
rktp, rd_list_cmp_ptr))
|
|
1859
|
+
rd_list_add(
|
|
1860
|
+
rkb->rkb_share_fetch_session.toppars_to_add,
|
|
1861
|
+
rktp);
|
|
1862
|
+
else
|
|
1863
|
+
rd_kafka_toppar_destroy(rktp);
|
|
1864
|
+
}
|
|
1865
|
+
}
|
|
1866
|
+
}
|
|
1867
|
+
|
|
1868
|
+
static void rd_kafka_broker_session_update_epoch(rd_kafka_broker_t *rkb) {
|
|
1869
|
+
int32_t prev_epoch = rkb->rkb_share_fetch_session.epoch;
|
|
1870
|
+
if (prev_epoch == -1) {
|
|
1871
|
+
rd_rkb_dbg(
|
|
1872
|
+
rkb, FETCH, "SHARESESSION",
|
|
1873
|
+
"Not updating next epoch for -1 as it should be -1 again.");
|
|
1874
|
+
return;
|
|
1875
|
+
}
|
|
1876
|
+
if (prev_epoch == INT32_MAX)
|
|
1877
|
+
rkb->rkb_share_fetch_session.epoch = 1;
|
|
1878
|
+
else
|
|
1879
|
+
rkb->rkb_share_fetch_session.epoch++;
|
|
1880
|
+
rd_rkb_dbg(rkb, FETCH, "SHARESESSION",
|
|
1881
|
+
"share-fetch session epoch %" PRId32 " -> %" PRId32
|
|
1882
|
+
" (wrap=%s)",
|
|
1883
|
+
prev_epoch, rkb->rkb_share_fetch_session.epoch,
|
|
1884
|
+
prev_epoch == INT32_MAX ? "yes" : "no");
|
|
1885
|
+
}
|
|
1886
|
+
|
|
1887
|
+
static void rd_kafka_broker_session_add_partition_to_toppars_in_session(
|
|
1888
|
+
rd_kafka_broker_t *rkb,
|
|
1889
|
+
rd_kafka_toppar_t *rktp) {
|
|
1890
|
+
if (rd_list_find(rkb->rkb_share_fetch_session.toppars_in_session, rktp,
|
|
1891
|
+
rd_list_cmp_ptr)) {
|
|
1892
|
+
rd_rkb_dbg(rkb, FETCH, "SHARESESSION",
|
|
1893
|
+
"%s [%" PRId32 "]: already in ShareFetch session",
|
|
1894
|
+
rktp->rktp_rkt->rkt_topic->str,
|
|
1895
|
+
rktp->rktp_partition);
|
|
1896
|
+
return;
|
|
1897
|
+
}
|
|
1898
|
+
rd_rkb_dbg(rkb, FETCH, "SHARESESSION",
|
|
1899
|
+
"%s [%" PRId32 "]: adding to ShareFetch session",
|
|
1900
|
+
rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition);
|
|
1901
|
+
rd_kafka_toppar_keep(rktp);
|
|
1902
|
+
rd_list_add(rkb->rkb_share_fetch_session.toppars_in_session, rktp);
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
void rd_kafka_broker_session_remove_partition_from_toppars_in_session(
|
|
1906
|
+
rd_kafka_broker_t *rkb,
|
|
1907
|
+
rd_kafka_toppar_t *rktp) {
|
|
1908
|
+
rd_kafka_toppar_t *removed_rktp;
|
|
1909
|
+
removed_rktp = rd_list_remove(
|
|
1910
|
+
rkb->rkb_share_fetch_session.toppars_in_session, rktp);
|
|
1911
|
+
if (removed_rktp) {
|
|
1912
|
+
rd_kafka_toppar_destroy(removed_rktp);
|
|
1913
|
+
rd_rkb_dbg(rkb, FETCH, "SHARESESSION",
|
|
1914
|
+
"%s [%" PRId32 "]: removed from ShareFetch session",
|
|
1915
|
+
rktp->rktp_rkt->rkt_topic->str,
|
|
1916
|
+
rktp->rktp_partition);
|
|
1917
|
+
} else {
|
|
1918
|
+
rd_rkb_dbg(rkb, FETCH, "SHARESESSION",
|
|
1919
|
+
"%s [%" PRId32 "]: not found in ShareFetch session",
|
|
1920
|
+
rktp->rktp_rkt->rkt_topic->str,
|
|
1921
|
+
rktp->rktp_partition);
|
|
1922
|
+
}
|
|
1923
|
+
}
|
|
1924
|
+
|
|
1925
|
+
static void
|
|
1926
|
+
rd_kafka_broker_session_update_toppars_in_session(rd_kafka_broker_t *rkb,
|
|
1927
|
+
rd_kafka_toppar_t *rktp,
|
|
1928
|
+
rd_bool_t add) {
|
|
1929
|
+
if (add)
|
|
1930
|
+
rd_kafka_broker_session_add_partition_to_toppars_in_session(
|
|
1931
|
+
rkb, rktp);
|
|
1932
|
+
else
|
|
1933
|
+
rd_kafka_broker_session_remove_partition_from_toppars_in_session(
|
|
1934
|
+
rkb, rktp);
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
static void rd_kafka_broker_session_update_toppars_list(
|
|
1938
|
+
rd_kafka_broker_t *rkb,
|
|
1939
|
+
rd_list_t **request_toppars_ptr,
|
|
1940
|
+
rd_list_t **toppars_to_remove_from_ptr,
|
|
1941
|
+
rd_bool_t add) {
|
|
1942
|
+
size_t i;
|
|
1943
|
+
rd_kafka_toppar_t *rktp, *removed_rktp;
|
|
1944
|
+
rd_list_t *request_toppars = *request_toppars_ptr;
|
|
1945
|
+
rd_list_t *toppars_to_remove_from = *toppars_to_remove_from_ptr;
|
|
1946
|
+
|
|
1947
|
+
if (request_toppars == NULL || rd_list_cnt(request_toppars) == 0)
|
|
1948
|
+
return;
|
|
1949
|
+
|
|
1950
|
+
rd_rkb_dbg(
|
|
1951
|
+
rkb, FETCH, "SHARESESSION",
|
|
1952
|
+
"%d toppars being %s the session:", rd_list_cnt(request_toppars),
|
|
1953
|
+
add ? "added to" : "removed from");
|
|
1954
|
+
|
|
1955
|
+
RD_LIST_FOREACH(rktp, request_toppars, i) {
|
|
1956
|
+
rd_kafka_broker_session_update_toppars_in_session(rkb, rktp,
|
|
1957
|
+
add);
|
|
1958
|
+
if (toppars_to_remove_from) {
|
|
1959
|
+
removed_rktp =
|
|
1960
|
+
rd_list_remove(toppars_to_remove_from, rktp);
|
|
1961
|
+
if (removed_rktp) {
|
|
1962
|
+
rd_kafka_toppar_destroy(
|
|
1963
|
+
removed_rktp); /* from partitions list */
|
|
1964
|
+
if (rd_list_empty(toppars_to_remove_from)) {
|
|
1965
|
+
rd_list_destroy(toppars_to_remove_from);
|
|
1966
|
+
toppars_to_remove_from = NULL;
|
|
1967
|
+
*toppars_to_remove_from_ptr = NULL;
|
|
1968
|
+
}
|
|
1969
|
+
}
|
|
1970
|
+
}
|
|
1971
|
+
}
|
|
1972
|
+
rd_list_destroy(request_toppars);
|
|
1973
|
+
*request_toppars_ptr = NULL;
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
static void
|
|
1977
|
+
rd_kafka_broker_session_update_added_partitions(rd_kafka_broker_t *rkb) {
|
|
1978
|
+
rd_kafka_broker_session_update_toppars_list(
|
|
1979
|
+
rkb, &rkb->rkb_share_fetch_session.adding_toppars,
|
|
1980
|
+
&rkb->rkb_share_fetch_session.toppars_to_add, rd_true);
|
|
1981
|
+
}
|
|
1982
|
+
|
|
1983
|
+
static void
|
|
1984
|
+
rd_kafka_broker_session_update_removed_partitions(rd_kafka_broker_t *rkb) {
|
|
1985
|
+
rd_kafka_broker_session_update_toppars_list(
|
|
1986
|
+
rkb, &rkb->rkb_share_fetch_session.forgetting_toppars,
|
|
1987
|
+
&rkb->rkb_share_fetch_session.toppars_to_forget, rd_false);
|
|
1988
|
+
}
|
|
1989
|
+
|
|
1990
|
+
static void rd_kafka_broker_session_update_partitions(rd_kafka_broker_t *rkb) {
|
|
1991
|
+
rd_rkb_dbg(
|
|
1992
|
+
rkb, FETCH, "SHARESESSION",
|
|
1993
|
+
"applying session updates: adding_toppars=%d "
|
|
1994
|
+
"forgetting_toppars=%d toppars_in_session=%d epoch=%" PRId32,
|
|
1995
|
+
rkb->rkb_share_fetch_session.adding_toppars
|
|
1996
|
+
? rd_list_cnt(rkb->rkb_share_fetch_session.adding_toppars)
|
|
1997
|
+
: 0,
|
|
1998
|
+
rkb->rkb_share_fetch_session.forgetting_toppars
|
|
1999
|
+
? rd_list_cnt(rkb->rkb_share_fetch_session.forgetting_toppars)
|
|
2000
|
+
: 0,
|
|
2001
|
+
rkb->rkb_share_fetch_session.toppars_in_session
|
|
2002
|
+
? rd_list_cnt(rkb->rkb_share_fetch_session.toppars_in_session)
|
|
2003
|
+
: 0,
|
|
2004
|
+
rkb->rkb_share_fetch_session.epoch);
|
|
2005
|
+
rd_kafka_broker_session_update_added_partitions(rkb);
|
|
2006
|
+
rd_kafka_broker_session_update_removed_partitions(rkb);
|
|
2007
|
+
}
|
|
2008
|
+
|
|
2009
|
+
|
|
2010
|
+
/**
|
|
2011
|
+
* Update ShareFetch session state after a Fetch or ShareFetch response.
|
|
2012
|
+
* TODO KIP-932: Improve efficiency of this function.
|
|
2013
|
+
*/
|
|
2014
|
+
static void rd_kafka_broker_session_update(rd_kafka_broker_t *rkb) {
|
|
2015
|
+
rd_kafka_broker_session_update_epoch(rkb);
|
|
2016
|
+
rd_kafka_broker_session_update_partitions(rkb);
|
|
2017
|
+
}
|
|
2018
|
+
|
|
2019
|
+
/**
|
|
2020
|
+
* @brief Whether \p err is a per-partition ShareAcknowledge error
|
|
2021
|
+
* whose CurrentLeader hint should trigger an inline metadata
|
|
2022
|
+
* update.
|
|
2023
|
+
*/
|
|
2024
|
+
static rd_bool_t
|
|
2025
|
+
rd_kafka_share_ack_err_is_leader_change(rd_kafka_resp_err_t err) {
|
|
2026
|
+
switch (err) {
|
|
2027
|
+
case RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER:
|
|
2028
|
+
case RD_KAFKA_RESP_ERR_FENCED_LEADER_EPOCH:
|
|
2029
|
+
case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART:
|
|
2030
|
+
case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID:
|
|
2031
|
+
return rd_true;
|
|
2032
|
+
default:
|
|
2033
|
+
return rd_false;
|
|
2034
|
+
}
|
|
2035
|
+
}
|
|
2036
|
+
|
|
2037
|
+
/**
|
|
2038
|
+
* @brief Parse a ShareAcknowledge response.
|
|
2039
|
+
*
|
|
2040
|
+
* ShareAcknowledge response contains per-partition error codes for
|
|
2041
|
+
* acknowledgement results. Returns the top-level error code and sets
|
|
2042
|
+
* the err field on each matching batch in \p ack_details.
|
|
2043
|
+
*
|
|
2044
|
+
* TODO KIP-932: Consider using rd_kafka_error_t instead of
|
|
2045
|
+
* rd_kafka_resp_err_t for the per-partition result and
|
|
2046
|
+
* acknowledgement callback.
|
|
2047
|
+
*
|
|
2048
|
+
* @param rkb Broker handle.
|
|
2049
|
+
* @param rkbuf Response buffer.
|
|
2050
|
+
* @param request Original request buffer.
|
|
2051
|
+
* @param ack_details List of acknowledgement batches sent in the
|
|
2052
|
+
* request. Each matched batch's rktpar->err is
|
|
2053
|
+
* updated with the partition-level error code
|
|
2054
|
+
* from the response.
|
|
2055
|
+
*
|
|
2056
|
+
* @returns Top-level error code.
|
|
2057
|
+
* @locality broker thread
|
|
2058
|
+
*/
|
|
2059
|
+
static rd_kafka_resp_err_t
|
|
2060
|
+
rd_kafka_share_acknowledge_reply_handle(rd_kafka_broker_t *rkb,
|
|
2061
|
+
rd_kafka_buf_t *rkbuf,
|
|
2062
|
+
rd_kafka_buf_t *request,
|
|
2063
|
+
rd_list_t *ack_details) {
|
|
2064
|
+
int32_t TopicArrayCnt;
|
|
2065
|
+
int i;
|
|
2066
|
+
const int log_decode_errors = LOG_ERR;
|
|
2067
|
+
int16_t ErrorCode = RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
2068
|
+
rd_kafkap_str_t ErrorStr = RD_KAFKAP_STR_INITIALIZER_EMPTY;
|
|
2069
|
+
rd_kafkap_NodeEndpoints_t NodeEndpoints;
|
|
2070
|
+
rd_list_t *leader_changes = NULL;
|
|
2071
|
+
NodeEndpoints.NodeEndpoints = NULL;
|
|
2072
|
+
NodeEndpoints.NodeEndpointCnt = 0;
|
|
2073
|
+
|
|
2074
|
+
rd_kafka_buf_read_throttle_time(rkbuf);
|
|
2075
|
+
|
|
2076
|
+
rd_kafka_buf_read_i16(rkbuf, &ErrorCode);
|
|
2077
|
+
rd_kafka_buf_read_str(rkbuf, &ErrorStr);
|
|
2078
|
+
|
|
2079
|
+
if (ErrorCode) {
|
|
2080
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREACK",
|
|
2081
|
+
"ShareAcknowledge response error %s: '%.*s'",
|
|
2082
|
+
rd_kafka_err2name(ErrorCode),
|
|
2083
|
+
RD_KAFKAP_STR_PR(&ErrorStr));
|
|
2084
|
+
return ErrorCode;
|
|
2085
|
+
}
|
|
2086
|
+
|
|
2087
|
+
rd_kafka_buf_read_arraycnt(rkbuf, &TopicArrayCnt, RD_KAFKAP_TOPICS_MAX);
|
|
2088
|
+
|
|
2089
|
+
for (i = 0; i < TopicArrayCnt; i++) {
|
|
2090
|
+
rd_kafka_Uuid_t topic_id = RD_KAFKA_UUID_ZERO;
|
|
2091
|
+
int32_t PartitionArrayCnt;
|
|
2092
|
+
int j;
|
|
2093
|
+
|
|
2094
|
+
rd_kafka_buf_read_uuid(rkbuf, &topic_id);
|
|
2095
|
+
|
|
2096
|
+
rd_kafka_buf_read_arraycnt(rkbuf, &PartitionArrayCnt,
|
|
2097
|
+
RD_KAFKAP_PARTITIONS_MAX);
|
|
2098
|
+
|
|
2099
|
+
for (j = 0; j < PartitionArrayCnt; j++) {
|
|
2100
|
+
int32_t Partition;
|
|
2101
|
+
int16_t PartErrorCode;
|
|
2102
|
+
rd_kafkap_str_t PartErrorStr;
|
|
2103
|
+
rd_kafkap_CurrentLeader_t CurrentLeader;
|
|
2104
|
+
rd_kafka_share_ack_batches_t *batch;
|
|
2105
|
+
rd_kafkap_share_leader_change_t *lc;
|
|
2106
|
+
|
|
2107
|
+
rd_kafka_buf_read_i32(rkbuf, &Partition);
|
|
2108
|
+
rd_kafka_buf_read_i16(rkbuf, &PartErrorCode);
|
|
2109
|
+
rd_kafka_buf_read_str(rkbuf, &PartErrorStr);
|
|
2110
|
+
|
|
2111
|
+
/* CurrentLeader */
|
|
2112
|
+
rd_kafka_buf_read_CurrentLeader(rkbuf, &CurrentLeader);
|
|
2113
|
+
|
|
2114
|
+
batch = rd_kafka_share_find_ack_batch_by_id(
|
|
2115
|
+
ack_details, topic_id, Partition);
|
|
2116
|
+
if (batch) {
|
|
2117
|
+
/* Conditional on _IN_PROGRESS as a
|
|
2118
|
+
* defensive safety net — the broker's
|
|
2119
|
+
* per-partition PartErrorCode is the
|
|
2120
|
+
* authoritative result for partitions
|
|
2121
|
+
* actually sent in the request. */
|
|
2122
|
+
if (batch->rktpar->err ==
|
|
2123
|
+
RD_KAFKA_RESP_ERR__IN_PROGRESS)
|
|
2124
|
+
batch->rktpar->err = PartErrorCode;
|
|
2125
|
+
} else {
|
|
2126
|
+
rd_rkb_log(
|
|
2127
|
+
rkb, LOG_ERR, "SHAREACK",
|
|
2128
|
+
"Invalid partition %" PRId32
|
|
2129
|
+
" received in ShareAcknowledge response",
|
|
2130
|
+
Partition);
|
|
2131
|
+
}
|
|
2132
|
+
|
|
2133
|
+
if (PartErrorCode) {
|
|
2134
|
+
/* TODO KIP-932: write test cases for each
|
|
2135
|
+
* per-partition ShareAcknowledge error code
|
|
2136
|
+
* once the mock cluster exposes a
|
|
2137
|
+
* per-partition error-injection API for
|
|
2138
|
+
* ShareAcknowledge responses. Today only
|
|
2139
|
+
* NOT_LEADER_OR_FOLLOWER (via
|
|
2140
|
+
* rd_kafka_mock_partition_set_leader) is
|
|
2141
|
+
* deterministically reachable; the other
|
|
2142
|
+
* leader-change errors
|
|
2143
|
+
* (FENCED_LEADER_EPOCH,
|
|
2144
|
+
* UNKNOWN_TOPIC_OR_PART,
|
|
2145
|
+
* UNKNOWN_TOPIC_ID),
|
|
2146
|
+
* INVALID_RECORD_STATE,
|
|
2147
|
+
* INVALID_REQUEST, KAFKA_STORAGE_ERROR
|
|
2148
|
+
* and the inline-leader-update path have
|
|
2149
|
+
* no deterministic mock trigger. */
|
|
2150
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREACK",
|
|
2151
|
+
"ShareAcknowledge partition %" PRId32
|
|
2152
|
+
" error %s: '%.*s'",
|
|
2153
|
+
Partition,
|
|
2154
|
+
rd_kafka_err2name(PartErrorCode),
|
|
2155
|
+
RD_KAFKAP_STR_PR(&PartErrorStr));
|
|
2156
|
+
|
|
2157
|
+
if (rd_kafka_share_ack_err_is_leader_change(
|
|
2158
|
+
PartErrorCode) &&
|
|
2159
|
+
CurrentLeader.LeaderId != -1 &&
|
|
2160
|
+
CurrentLeader.LeaderEpoch != -1) {
|
|
2161
|
+
if (!leader_changes)
|
|
2162
|
+
leader_changes =
|
|
2163
|
+
rd_list_new(0, rd_free);
|
|
2164
|
+
lc = rd_calloc(1, sizeof(*lc));
|
|
2165
|
+
lc->topic_id = topic_id;
|
|
2166
|
+
lc->partition = Partition;
|
|
2167
|
+
lc->current_leader = CurrentLeader;
|
|
2168
|
+
rd_list_add(leader_changes, lc);
|
|
2169
|
+
}
|
|
2170
|
+
}
|
|
2171
|
+
|
|
2172
|
+
/* Partition tags */
|
|
2173
|
+
rd_kafka_buf_skip_tags(rkbuf);
|
|
2174
|
+
}
|
|
2175
|
+
|
|
2176
|
+
/* Topic tags */
|
|
2177
|
+
rd_kafka_buf_skip_tags(rkbuf);
|
|
2178
|
+
}
|
|
2179
|
+
|
|
2180
|
+
rd_kafka_buf_read_NodeEndpoints(rkbuf, &NodeEndpoints);
|
|
2181
|
+
|
|
2182
|
+
/* Top level tags */
|
|
2183
|
+
rd_kafka_buf_skip_tags(rkbuf);
|
|
2184
|
+
|
|
2185
|
+
rd_kafkap_share_leader_changes_apply(rkb, leader_changes,
|
|
2186
|
+
&NodeEndpoints);
|
|
2187
|
+
|
|
2188
|
+
RD_IF_FREE(leader_changes, rd_list_destroy);
|
|
2189
|
+
RD_IF_FREE(NodeEndpoints.NodeEndpoints, rd_free);
|
|
2190
|
+
return RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
2191
|
+
|
|
2192
|
+
err_parse:
|
|
2193
|
+
RD_IF_FREE(leader_changes, rd_list_destroy);
|
|
2194
|
+
RD_IF_FREE(NodeEndpoints.NodeEndpoints, rd_free);
|
|
2195
|
+
rd_rkb_dbg(rkb, MSG, "BADMSG",
|
|
2196
|
+
"Bad ShareAcknowledge response (v%d): parse error",
|
|
2197
|
+
(int)request->rkbuf_reqhdr.ApiVersion);
|
|
2198
|
+
return rkbuf->rkbuf_err;
|
|
2199
|
+
}
|
|
2200
|
+
|
|
2201
|
+
|
|
2202
|
+
/**
|
|
2203
|
+
* @brief ShareAcknowledge response handling callback.
|
|
2204
|
+
*
|
|
2205
|
+
* @locality broker thread (or any thread if err == __DESTROY).
|
|
2206
|
+
*/
|
|
2207
|
+
static void rd_kafka_broker_share_acknowledge_reply(rd_kafka_t *rk,
|
|
2208
|
+
rd_kafka_broker_t *rkb,
|
|
2209
|
+
rd_kafka_resp_err_t err,
|
|
2210
|
+
rd_kafka_buf_t *reply,
|
|
2211
|
+
rd_kafka_buf_t *request,
|
|
2212
|
+
void *opaque) {
|
|
2213
|
+
rd_kafka_op_t *rko_orig = opaque;
|
|
2214
|
+
|
|
2215
|
+
/* Parse and handle the response (unless the request errored).
|
|
2216
|
+
* The parser writes per-partition err on matching batches in
|
|
2217
|
+
* ack_details (only on batches still at the _IN_PROGRESS
|
|
2218
|
+
* sentinel as a defensive safety net). DESTROY case falls
|
|
2219
|
+
* through here and is handled by the generic top-level error
|
|
2220
|
+
* path below.
|
|
2221
|
+
*
|
|
2222
|
+
* If the parser ran and succeeded (err remains 0), convert
|
|
2223
|
+
* any batch still at _IN_PROGRESS to INVALID_RECORD_STATE —
|
|
2224
|
+
* those partitions were sent in the request but missing from
|
|
2225
|
+
* the response.
|
|
2226
|
+
*
|
|
2227
|
+
* If the parser was skipped or failed (err != 0), leave
|
|
2228
|
+
* batches at _IN_PROGRESS so the helper at the end can
|
|
2229
|
+
* propagate the top-level err to them. */
|
|
2230
|
+
if (!err && reply) {
|
|
2231
|
+
err = rd_kafka_share_acknowledge_reply_handle(
|
|
2232
|
+
rkb, reply, request,
|
|
2233
|
+
rko_orig->rko_u.share_fetch.ack_details);
|
|
2234
|
+
if (!err && rko_orig->rko_u.share_fetch.ack_details) {
|
|
2235
|
+
rd_kafka_share_ack_batches_t *batch;
|
|
2236
|
+
int i;
|
|
2237
|
+
RD_LIST_FOREACH(
|
|
2238
|
+
batch, rko_orig->rko_u.share_fetch.ack_details, i) {
|
|
2239
|
+
if (batch->rktpar->err ==
|
|
2240
|
+
RD_KAFKA_RESP_ERR__IN_PROGRESS)
|
|
2241
|
+
batch->rktpar->err =
|
|
2242
|
+
RD_KAFKA_RESP_ERR_INVALID_RECORD_STATE;
|
|
2243
|
+
}
|
|
2244
|
+
}
|
|
2245
|
+
}
|
|
2246
|
+
|
|
2247
|
+
rd_kafka_broker_session_update(rkb);
|
|
2248
|
+
|
|
2249
|
+
if (unlikely(err)) {
|
|
2250
|
+
rd_rkb_log(rkb, LOG_INFO, "SHAREACK",
|
|
2251
|
+
"ShareAcknowledge reply error: %s",
|
|
2252
|
+
rd_kafka_err2str(err));
|
|
2253
|
+
switch (err) {
|
|
2254
|
+
case RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND:
|
|
2255
|
+
case RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH:
|
|
2256
|
+
case RD_KAFKA_RESP_ERR__TRANSPORT:
|
|
2257
|
+
case RD_KAFKA_RESP_ERR__TIMED_OUT:
|
|
2258
|
+
/* __TRANSPORT means connection is already dead.
|
|
2259
|
+
* On __TIMED_OUT the connection is torn down by the
|
|
2260
|
+
* request-timeout scan (socket.max.fails is forced to
|
|
2261
|
+
* 1 for share consumers and cannot be changed) and the
|
|
2262
|
+
* broker reconnects via the share-serve persistent
|
|
2263
|
+
* connection driver; here we only reset the session so
|
|
2264
|
+
* it re-establishes at epoch 0. */
|
|
2265
|
+
rd_kafka_broker_session_reset(rkb);
|
|
2266
|
+
break;
|
|
2267
|
+
|
|
2268
|
+
case RD_KAFKA_RESP_ERR__BAD_MSG:
|
|
2269
|
+
case RD_KAFKA_RESP_ERR__UNDERFLOW:
|
|
2270
|
+
/* Wire-level parse failure: response envelope or
|
|
2271
|
+
* an inner MessageSet/record was malformed or
|
|
2272
|
+
* truncated. Indicates broker bug, on-the-wire
|
|
2273
|
+
* corruption, or version mismatch — log loudly so
|
|
2274
|
+
* the user notices, since the LOG_INFO top-level
|
|
2275
|
+
* log can be easy to miss in production output. */
|
|
2276
|
+
rd_rkb_log(
|
|
2277
|
+
rkb, LOG_ERR, "SHAREACK",
|
|
2278
|
+
"ShareAcknowledge response parse failure: %s "
|
|
2279
|
+
"(ApiVersion %hd) — broker bug, wire "
|
|
2280
|
+
"corruption, or version mismatch; "
|
|
2281
|
+
"this response is being dropped",
|
|
2282
|
+
rd_kafka_err2str(err),
|
|
2283
|
+
request->rkbuf_reqhdr.ApiVersion);
|
|
2284
|
+
break;
|
|
2285
|
+
|
|
2286
|
+
default:
|
|
2287
|
+
/* No retry for ShareAcknowledge RPC-level
|
|
2288
|
+
* errors. The error semantics (partition
|
|
2289
|
+
* moved, session state lost, etc.) make a
|
|
2290
|
+
* blind retry unlikely to succeed and risk
|
|
2291
|
+
* duplicating side effects. */
|
|
2292
|
+
break;
|
|
2293
|
+
}
|
|
2294
|
+
}
|
|
2295
|
+
|
|
2296
|
+
if (rko_orig->rko_u.share_fetch.should_leave)
|
|
2297
|
+
rd_kafka_broker_share_fetch_session_clear(rkb);
|
|
2298
|
+
|
|
2299
|
+
/* ack_details is owned by the op and freed by the op destructor
|
|
2300
|
+
* after the main thread has processed the reply. */
|
|
2301
|
+
rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err(rko_orig,
|
|
2302
|
+
err);
|
|
2303
|
+
}
|
|
2304
|
+
|
|
2305
|
+
|
|
2306
|
+
/**
|
|
2307
|
+
* @broker ShareFetchResponse handling.
|
|
2308
|
+
*
|
|
2309
|
+
* @locality broker thread (or any thread if err == __DESTROY).
|
|
2310
|
+
*/
|
|
2311
|
+
static void rd_kafka_broker_share_fetch_reply(rd_kafka_t *rk,
|
|
2312
|
+
rd_kafka_broker_t *rkb,
|
|
2313
|
+
rd_kafka_resp_err_t err,
|
|
2314
|
+
rd_kafka_buf_t *reply,
|
|
2315
|
+
rd_kafka_buf_t *request,
|
|
2316
|
+
void *opaque) {
|
|
2317
|
+
|
|
2318
|
+
rd_kafka_op_t *rko_orig = opaque;
|
|
2319
|
+
rd_kafka_op_t *response_rko = NULL;
|
|
2320
|
+
|
|
2321
|
+
/* Parse the response only if the network/broker layer didn't
|
|
2322
|
+
* report an error. If err is set (e.g. __TRANSPORT,
|
|
2323
|
+
* __TIMED_OUT, __DESTROY), the reply buffer is unusable so
|
|
2324
|
+
* we skip parsing. The parser writes per-partition err on
|
|
2325
|
+
* matching batches in ack_details (only on batches still at
|
|
2326
|
+
* the _IN_PROGRESS sentinel, so any deliberately pre-set
|
|
2327
|
+
* value such as INVALID_SHARE_SESSION_EPOCH is preserved).
|
|
2328
|
+
* The DESTROY/network error case falls through to the
|
|
2329
|
+
* generic top-level error path below.
|
|
2330
|
+
*
|
|
2331
|
+
* If the parser ran and succeeded (err remains 0), convert
|
|
2332
|
+
* any batch still at _IN_PROGRESS to INVALID_RECORD_STATE —
|
|
2333
|
+
* those partitions were sent in the request but missing from
|
|
2334
|
+
* the response.
|
|
2335
|
+
*
|
|
2336
|
+
* If the parser was skipped or failed (err != 0), leave
|
|
2337
|
+
* batches at _IN_PROGRESS so the helper at the end can
|
|
2338
|
+
* propagate the top-level err to them. */
|
|
2339
|
+
if (!err && reply) {
|
|
2340
|
+
err = rd_kafka_share_fetch_reply_handle(rkb, reply, request,
|
|
2341
|
+
&response_rko);
|
|
2342
|
+
if (!err && rko_orig->rko_u.share_fetch.ack_details) {
|
|
2343
|
+
rd_kafka_share_ack_batches_t *batch;
|
|
2344
|
+
int i;
|
|
2345
|
+
RD_LIST_FOREACH(
|
|
2346
|
+
batch, rko_orig->rko_u.share_fetch.ack_details, i) {
|
|
2347
|
+
if (batch->rktpar->err ==
|
|
2348
|
+
RD_KAFKA_RESP_ERR__IN_PROGRESS)
|
|
2349
|
+
batch->rktpar->err =
|
|
2350
|
+
RD_KAFKA_RESP_ERR_INVALID_RECORD_STATE;
|
|
2351
|
+
}
|
|
2352
|
+
}
|
|
2353
|
+
}
|
|
2354
|
+
|
|
2355
|
+
rd_kafka_broker_session_update(rkb);
|
|
2356
|
+
|
|
2357
|
+
if (unlikely(err)) {
|
|
2358
|
+
rd_rkb_log(rkb, LOG_INFO, "SHAREFETCH",
|
|
2359
|
+
"ShareFetch reply error: %s", rd_kafka_err2str(err));
|
|
2360
|
+
switch (err) {
|
|
2361
|
+
case RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND:
|
|
2362
|
+
case RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH:
|
|
2363
|
+
case RD_KAFKA_RESP_ERR_SHARE_SESSION_LIMIT_REACHED:
|
|
2364
|
+
case RD_KAFKA_RESP_ERR__TRANSPORT:
|
|
2365
|
+
case RD_KAFKA_RESP_ERR__TIMED_OUT:
|
|
2366
|
+
/* Session is invalid, lost, cannot be created,
|
|
2367
|
+
* or connection/request failed.
|
|
2368
|
+
* Reset session state so the next request
|
|
2369
|
+
* re-establishes a new session (epoch 0).
|
|
2370
|
+
* __TRANSPORT means connection is already dead.
|
|
2371
|
+
* On __TIMED_OUT the connection is torn down by the
|
|
2372
|
+
* request-timeout scan (socket.max.fails is forced to
|
|
2373
|
+
* 1 for share consumers and cannot be changed) and the
|
|
2374
|
+
* broker reconnects via the share-serve persistent
|
|
2375
|
+
* connection driver. */
|
|
2376
|
+
rd_kafka_broker_session_reset(rkb);
|
|
2377
|
+
break;
|
|
2378
|
+
|
|
2379
|
+
case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID: {
|
|
2380
|
+
char tmp[128];
|
|
2381
|
+
rd_snprintf(tmp, sizeof(tmp), "ShareFetch failed: %s",
|
|
2382
|
+
rd_kafka_err2str(err));
|
|
2383
|
+
rd_kafka_metadata_refresh_known_topics(
|
|
2384
|
+
rkb->rkb_rk, NULL, rd_false /*!force*/, tmp);
|
|
2385
|
+
break;
|
|
2386
|
+
}
|
|
2387
|
+
|
|
2388
|
+
case RD_KAFKA_RESP_ERR__BAD_MSG:
|
|
2389
|
+
case RD_KAFKA_RESP_ERR__UNDERFLOW:
|
|
2390
|
+
/* Wire-level parse failure: response envelope or
|
|
2391
|
+
* an inner MessageSet/record was malformed or
|
|
2392
|
+
* truncated. Indicates broker bug, on-the-wire
|
|
2393
|
+
* corruption, or version mismatch — log loudly so
|
|
2394
|
+
* the user notices, since the LOG_INFO top-level
|
|
2395
|
+
* log can be easy to miss in production output. */
|
|
2396
|
+
rd_rkb_log(rkb, LOG_ERR, "SHAREFETCH",
|
|
2397
|
+
"ShareFetch response parse failure: %s "
|
|
2398
|
+
"(ApiVersion %hd) — broker bug, wire "
|
|
2399
|
+
"corruption, or version mismatch; "
|
|
2400
|
+
"this response is being dropped",
|
|
2401
|
+
rd_kafka_err2str(err),
|
|
2402
|
+
request->rkbuf_reqhdr.ApiVersion);
|
|
2403
|
+
break;
|
|
2404
|
+
|
|
2405
|
+
default:
|
|
2406
|
+
/* No error-specific handling at the request
|
|
2407
|
+
* level. Non-session top-level errors are
|
|
2408
|
+
* treated as transient — the main thread
|
|
2409
|
+
* retries by selecting another broker. */
|
|
2410
|
+
break;
|
|
2411
|
+
}
|
|
2412
|
+
|
|
2413
|
+
/* There is no retry for ShareFetch RPC at the broker
|
|
2414
|
+
* thread level. */
|
|
2415
|
+
}
|
|
2416
|
+
|
|
2417
|
+
/* ack_details is owned by the op and freed by the op destructor
|
|
2418
|
+
* after the main thread has processed the reply. */
|
|
2419
|
+
rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err(rko_orig,
|
|
2420
|
+
err);
|
|
2421
|
+
|
|
2422
|
+
/* Enqueue the response for the app thread AFTER sending the
|
|
2423
|
+
* reply to the main thread. This ensures the main thread
|
|
2424
|
+
* processes the reply (resetting share_fetch_more_records)
|
|
2425
|
+
* before the app thread wakes up and enqueues a new FANOUT. */
|
|
2426
|
+
if (response_rko)
|
|
2427
|
+
rd_kafka_q_enq(rkb->rkb_rk->rk_cgrp->rkcg_q, response_rko);
|
|
2428
|
+
}
|
|
2429
|
+
|
|
2430
|
+
/**
|
|
2431
|
+
* @broker FetchResponse handling.
|
|
2432
|
+
*
|
|
2433
|
+
* @locality broker thread (or any thread if err == __DESTROY).
|
|
2434
|
+
*/
|
|
2435
|
+
static void rd_kafka_broker_fetch_reply(rd_kafka_t *rk,
|
|
2436
|
+
rd_kafka_broker_t *rkb,
|
|
2437
|
+
rd_kafka_resp_err_t err,
|
|
2438
|
+
rd_kafka_buf_t *reply,
|
|
2439
|
+
rd_kafka_buf_t *request,
|
|
2440
|
+
void *opaque) {
|
|
2441
|
+
|
|
2442
|
+
if (err == RD_KAFKA_RESP_ERR__DESTROY)
|
|
2443
|
+
return; /* Terminating */
|
|
2444
|
+
|
|
2445
|
+
rd_kafka_assert(rkb->rkb_rk, rkb->rkb_fetching > 0);
|
|
2446
|
+
rkb->rkb_fetching = 0;
|
|
2447
|
+
|
|
2448
|
+
/* Parse and handle the messages (unless the request errored) */
|
|
2449
|
+
if (!err && reply)
|
|
2450
|
+
err = rd_kafka_fetch_reply_handle(rkb, reply, request);
|
|
2451
|
+
|
|
2452
|
+
if (unlikely(err)) {
|
|
2453
|
+
char tmp[128];
|
|
2454
|
+
|
|
2455
|
+
rd_rkb_dbg(rkb, MSG, "FETCH", "Fetch reply: %s",
|
|
2456
|
+
rd_kafka_err2str(err));
|
|
2457
|
+
switch (err) {
|
|
2458
|
+
case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART:
|
|
2459
|
+
case RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE:
|
|
2460
|
+
case RD_KAFKA_RESP_ERR_NOT_LEADER_FOR_PARTITION:
|
|
2461
|
+
case RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE:
|
|
2462
|
+
case RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE:
|
|
2463
|
+
case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID:
|
|
2464
|
+
/* Request metadata information update */
|
|
2465
|
+
rd_snprintf(tmp, sizeof(tmp), "FetchRequest failed: %s",
|
|
2466
|
+
rd_kafka_err2str(err));
|
|
2467
|
+
rd_kafka_metadata_refresh_known_topics(
|
|
2468
|
+
rkb->rkb_rk, NULL, rd_true /*force*/, tmp);
|
|
2469
|
+
/* FALLTHRU */
|
|
2470
|
+
|
|
2471
|
+
case RD_KAFKA_RESP_ERR__TRANSPORT:
|
|
2472
|
+
case RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT:
|
|
2473
|
+
case RD_KAFKA_RESP_ERR__MSG_TIMED_OUT:
|
|
2474
|
+
/* The fetch is already intervalled from
|
|
2475
|
+
* consumer_serve() so dont retry. */
|
|
2476
|
+
break;
|
|
2477
|
+
|
|
2478
|
+
default:
|
|
2479
|
+
break;
|
|
2480
|
+
}
|
|
2481
|
+
|
|
2482
|
+
rd_kafka_broker_fetch_backoff(rkb, err);
|
|
2483
|
+
/* FALLTHRU */
|
|
2484
|
+
}
|
|
2485
|
+
}
|
|
2486
|
+
|
|
2487
|
+
/**
|
|
2488
|
+
* @brief Check if any toppars have a zero topic id.
|
|
2489
|
+
*
|
|
2490
|
+
*/
|
|
2491
|
+
static rd_bool_t can_use_topic_ids(rd_kafka_broker_t *rkb) {
|
|
2492
|
+
rd_kafka_toppar_t *rktp = rkb->rkb_active_toppar_next;
|
|
2493
|
+
do {
|
|
2494
|
+
if (RD_KAFKA_UUID_IS_ZERO(rktp->rktp_rkt->rkt_topic_id))
|
|
2495
|
+
return rd_false;
|
|
2496
|
+
} while ((rktp = CIRCLEQ_LOOP_NEXT(&rkb->rkb_active_toppars, rktp,
|
|
2497
|
+
rktp_activelink)) !=
|
|
2498
|
+
rkb->rkb_active_toppar_next);
|
|
2499
|
+
|
|
2500
|
+
return rd_true;
|
|
2501
|
+
}
|
|
2502
|
+
|
|
2503
|
+
|
|
2504
|
+
/**
|
|
2505
|
+
* @brief Sum the total acknowledgement entries and records across all
|
|
2506
|
+
* per-partition batches in \p ack_details.
|
|
2507
|
+
*
|
|
2508
|
+
* @param ack_details List of rd_kafka_share_ack_batches_t, or NULL.
|
|
2509
|
+
* @param entries_out Set to the total number of ack entries (offset ranges).
|
|
2510
|
+
* @param records_out Set to the total number of acknowledged records
|
|
2511
|
+
* (sum of entry->size across all entries).
|
|
2512
|
+
*/
|
|
2513
|
+
static void rd_kafka_share_ack_details_totals(rd_list_t *ack_details,
|
|
2514
|
+
int *entries_out,
|
|
2515
|
+
int64_t *records_out) {
|
|
2516
|
+
rd_kafka_share_ack_batches_t *batches;
|
|
2517
|
+
rd_kafka_share_ack_batch_entry_t *entry;
|
|
2518
|
+
int k, j;
|
|
2519
|
+
int entries = 0;
|
|
2520
|
+
int64_t records = 0;
|
|
2521
|
+
|
|
2522
|
+
if (ack_details) {
|
|
2523
|
+
RD_LIST_FOREACH(batches, ack_details, k) {
|
|
2524
|
+
entries += rd_list_cnt(&batches->entries);
|
|
2525
|
+
RD_LIST_FOREACH(entry, &batches->entries, j) {
|
|
2526
|
+
records += entry->size;
|
|
2527
|
+
}
|
|
2528
|
+
}
|
|
2529
|
+
}
|
|
2530
|
+
|
|
2531
|
+
*entries_out = entries;
|
|
2532
|
+
*records_out = records;
|
|
2533
|
+
}
|
|
2534
|
+
|
|
2535
|
+
|
|
2536
|
+
void rd_kafka_ShareFetchRequest(rd_kafka_broker_t *rkb,
|
|
2537
|
+
const rd_kafkap_str_t *group_id,
|
|
2538
|
+
const rd_kafkap_str_t *member_id,
|
|
2539
|
+
int32_t share_session_epoch,
|
|
2540
|
+
int32_t wait_max_ms,
|
|
2541
|
+
int32_t min_bytes,
|
|
2542
|
+
int32_t max_bytes,
|
|
2543
|
+
int32_t max_records,
|
|
2544
|
+
int32_t batch_size,
|
|
2545
|
+
rd_list_t *toppars_to_add,
|
|
2546
|
+
rd_list_t *toppars_to_forget,
|
|
2547
|
+
rd_kafka_op_t *rko_orig,
|
|
2548
|
+
rd_ts_t now) {
|
|
2549
|
+
rd_kafka_toppar_t *rktp;
|
|
2550
|
+
rd_kafka_buf_t *rkbuf;
|
|
2551
|
+
int cnt = 0;
|
|
2552
|
+
size_t of_TopicArrayCnt = 0;
|
|
2553
|
+
int TopicArrayCnt = 0;
|
|
2554
|
+
size_t of_PartitionArrayCnt = 0;
|
|
2555
|
+
int PartitionArrayCnt = 0;
|
|
2556
|
+
rd_kafka_topic_t *rkt_last = NULL;
|
|
2557
|
+
int16_t ApiVersion = 0;
|
|
2558
|
+
size_t rkbuf_size = 0;
|
|
2559
|
+
int toppars_to_add_cnt =
|
|
2560
|
+
toppars_to_add ? rd_list_cnt(toppars_to_add) : 0;
|
|
2561
|
+
int i;
|
|
2562
|
+
rd_list_t *ack_details =
|
|
2563
|
+
rko_orig ? rko_orig->rko_u.share_fetch.ack_details : NULL;
|
|
2564
|
+
int ack_details_cnt = ack_details ? rd_list_cnt(ack_details) : 0;
|
|
2565
|
+
/* TODO KIP-932: Ensure there is no intersection between toppars_to_add
|
|
2566
|
+
* and ack_details. A toppar should not appear in both lists. */
|
|
2567
|
+
int total_ack_entries = 0;
|
|
2568
|
+
int64_t total_ack_records = 0;
|
|
2569
|
+
int toppars_to_forget_cnt =
|
|
2570
|
+
toppars_to_forget ? rd_list_cnt(toppars_to_forget) : 0;
|
|
2571
|
+
rd_bool_t is_fetching_messages = max_records > 0 ? rd_true : rd_false;
|
|
2572
|
+
/* FirstOffset + LastOffset + AcknowledgementType per ack entry */
|
|
2573
|
+
size_t acknowledgement_size = 8 + 8 + 1;
|
|
2574
|
+
|
|
2575
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
|
|
2576
|
+
"toppars_to_add_cnt=%d, ack_details_cnt=%d, "
|
|
2577
|
+
"total_ack_entries=%d, toppars_to_forget_cnt=%d, "
|
|
2578
|
+
"is_fetching_messages=%d",
|
|
2579
|
+
toppars_to_add_cnt, ack_details_cnt, total_ack_entries,
|
|
2580
|
+
toppars_to_forget_cnt, is_fetching_messages);
|
|
2581
|
+
|
|
2582
|
+
/* Calculate buffer size */
|
|
2583
|
+
if (group_id)
|
|
2584
|
+
rkbuf_size += RD_KAFKAP_STR_SIZE(group_id);
|
|
2585
|
+
if (member_id)
|
|
2586
|
+
rkbuf_size += RD_KAFKAP_STR_SIZE(member_id);
|
|
2587
|
+
/* ShareSessionEpoch + WaitMaxMs + MinBytes + MaxBytes + MaxRecords +
|
|
2588
|
+
* BatchSize + TopicArrayCnt*/
|
|
2589
|
+
rkbuf_size += 4 + 4 + 4 + 4 + 4 + 4 + 4;
|
|
2590
|
+
/* N x (topic id + partition id) for topics to add */
|
|
2591
|
+
rkbuf_size += (toppars_to_add_cnt * (32 + 4));
|
|
2592
|
+
/* M x (topic id + partition id) for ack details partitions +
|
|
2593
|
+
* E x acknowledgement entries */
|
|
2594
|
+
|
|
2595
|
+
rkbuf_size += (ack_details_cnt * (32 + 4));
|
|
2596
|
+
/* Sum total ack entries and total records across all
|
|
2597
|
+
* ack_details batches. */
|
|
2598
|
+
rd_kafka_share_ack_details_totals(ack_details, &total_ack_entries,
|
|
2599
|
+
&total_ack_records);
|
|
2600
|
+
/* Accumulate piggybacked record-level acknowledgements for
|
|
2601
|
+
* consumer.share.fetch.manager.acknowledgements.send.{total,rate}. */
|
|
2602
|
+
if (total_ack_records > 0)
|
|
2603
|
+
rd_atomic64_add(
|
|
2604
|
+
&rkb->rkb_rk->rk_telemetry.acknowledgements_send_total,
|
|
2605
|
+
total_ack_records);
|
|
2606
|
+
|
|
2607
|
+
rkbuf_size += (total_ack_entries * acknowledgement_size);
|
|
2608
|
+
|
|
2609
|
+
/* F x (topic id + partition id) for topics to forget */
|
|
2610
|
+
rkbuf_size += (toppars_to_forget_cnt * (32 + 4));
|
|
2611
|
+
|
|
2612
|
+
ApiVersion = rd_kafka_broker_ApiVersion_supported(
|
|
2613
|
+
rkb, RD_KAFKAP_ShareFetch, 1, 1, NULL);
|
|
2614
|
+
|
|
2615
|
+
rkbuf = rd_kafka_buf_new_flexver_request(rkb, RD_KAFKAP_ShareFetch, 1,
|
|
2616
|
+
rkbuf_size, rd_true);
|
|
2617
|
+
|
|
2618
|
+
if (rkb->rkb_features & RD_KAFKA_FEATURE_MSGVER2)
|
|
2619
|
+
rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion,
|
|
2620
|
+
RD_KAFKA_FEATURE_MSGVER2);
|
|
2621
|
+
else if (rkb->rkb_features & RD_KAFKA_FEATURE_MSGVER1)
|
|
2622
|
+
rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion,
|
|
2623
|
+
RD_KAFKA_FEATURE_MSGVER1);
|
|
2624
|
+
else if (rkb->rkb_features & RD_KAFKA_FEATURE_THROTTLETIME)
|
|
2625
|
+
rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion,
|
|
2626
|
+
RD_KAFKA_FEATURE_THROTTLETIME);
|
|
2627
|
+
|
|
2628
|
+
/* GroupId */
|
|
2629
|
+
rd_kafka_buf_write_kstr(rkbuf, group_id);
|
|
2630
|
+
|
|
2631
|
+
/* MemberId */
|
|
2632
|
+
rd_kafka_buf_write_kstr(rkbuf, member_id);
|
|
2633
|
+
|
|
2634
|
+
// printf(" ---------------------------------------
|
|
2635
|
+
// rd_kafka_ShareFetchRequest: member_id=%.*s\n",
|
|
2636
|
+
// RD_KAFKAP_STR_PR(member_id));
|
|
2637
|
+
|
|
2638
|
+
/* ShareSessionEpoch */
|
|
2639
|
+
rd_kafka_buf_write_i32(rkbuf, share_session_epoch);
|
|
2640
|
+
|
|
2641
|
+
/* WaitMaxMs */
|
|
2642
|
+
rd_kafka_buf_write_i32(rkbuf, wait_max_ms);
|
|
2643
|
+
|
|
2644
|
+
/* MinBytes */
|
|
2645
|
+
rd_kafka_buf_write_i32(rkbuf, min_bytes);
|
|
2646
|
+
|
|
2647
|
+
/* MaxBytes */
|
|
2648
|
+
rd_kafka_buf_write_i32(rkbuf, max_bytes);
|
|
2649
|
+
|
|
2650
|
+
/* MaxRecords */
|
|
2651
|
+
rd_kafka_buf_write_i32(rkbuf, max_records);
|
|
2652
|
+
|
|
2653
|
+
/* BatchSize */
|
|
2654
|
+
rd_kafka_buf_write_i32(rkbuf, batch_size);
|
|
2655
|
+
|
|
2656
|
+
/* Write zero TopicArrayCnt but store pointer for later update */
|
|
2657
|
+
of_TopicArrayCnt = rd_kafka_buf_write_arraycnt_pos(rkbuf);
|
|
2658
|
+
|
|
2659
|
+
/* TODO KIP-932: Ensure toppars_to_add and ack_details don't have
|
|
2660
|
+
* common rktps. A toppar should only appear in one list.
|
|
2661
|
+
* Also merge toppars_to_add and ack_details as both can have
|
|
2662
|
+
* the same topic but different partitions. */
|
|
2663
|
+
|
|
2664
|
+
/* Write toppars_to_add: new toppars being added to session
|
|
2665
|
+
* (no acknowledgements for newly added toppars) */
|
|
2666
|
+
if (toppars_to_add) {
|
|
2667
|
+
/* TODO KIP-932: This condition will cause partitions of same
|
|
2668
|
+
topics to be inside single instance of the topic as
|
|
2669
|
+
toppars_to_add is not sorted. Eg: T1 0, T1 1, T2 0, T1 3, T1
|
|
2670
|
+
5, T2 1 will translate to T1 (0,1), T2 (0), T1 (3, 5), T2
|
|
2671
|
+
(1) instead it should be T1 (0,1,3,5) T2(0,1) Fix this. */
|
|
2672
|
+
RD_LIST_FOREACH(rktp, toppars_to_add, i) {
|
|
2673
|
+
if (rkt_last != rktp->rktp_rkt) {
|
|
2674
|
+
if (rkt_last != NULL) {
|
|
2675
|
+
/* Update PartitionArrayCnt */
|
|
2676
|
+
rd_kafka_buf_finalize_arraycnt(
|
|
2677
|
+
rkbuf, of_PartitionArrayCnt,
|
|
2678
|
+
PartitionArrayCnt);
|
|
2679
|
+
/* Topic tags */
|
|
2680
|
+
rd_kafka_buf_write_tags_empty(rkbuf);
|
|
2681
|
+
}
|
|
2682
|
+
|
|
2683
|
+
rd_kafka_topic_rdlock(rktp->rktp_rkt);
|
|
2684
|
+
/* Topic ID */
|
|
2685
|
+
rd_kafka_buf_write_uuid(
|
|
2686
|
+
rkbuf, &rktp->rktp_rkt->rkt_topic_id);
|
|
2687
|
+
rd_kafka_topic_rdunlock(rktp->rktp_rkt);
|
|
2688
|
+
|
|
2689
|
+
TopicArrayCnt++;
|
|
2690
|
+
rkt_last = rktp->rktp_rkt;
|
|
2691
|
+
/* Partition count */
|
|
2692
|
+
of_PartitionArrayCnt =
|
|
2693
|
+
rd_kafka_buf_write_arraycnt_pos(rkbuf);
|
|
2694
|
+
PartitionArrayCnt = 0;
|
|
2695
|
+
}
|
|
2696
|
+
|
|
2697
|
+
PartitionArrayCnt++;
|
|
2698
|
+
|
|
2699
|
+
/* Partition */
|
|
2700
|
+
rd_kafka_buf_write_i32(rkbuf, rktp->rktp_partition);
|
|
2701
|
+
|
|
2702
|
+
/* No acknowledgements for newly added toppars */
|
|
2703
|
+
rd_kafka_buf_write_arraycnt(rkbuf, 0);
|
|
2704
|
+
|
|
2705
|
+
/* Partition tags */
|
|
2706
|
+
rd_kafka_buf_write_tags_empty(rkbuf);
|
|
2707
|
+
|
|
2708
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
|
|
2709
|
+
"Share Fetch adding topic %.*s [%" PRId32
|
|
2710
|
+
"]",
|
|
2711
|
+
RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
|
|
2712
|
+
rktp->rktp_partition);
|
|
2713
|
+
|
|
2714
|
+
cnt++;
|
|
2715
|
+
}
|
|
2716
|
+
|
|
2717
|
+
/* Finalize last topic from toppars_to_add */
|
|
2718
|
+
if (rkt_last != NULL) {
|
|
2719
|
+
rd_kafka_buf_finalize_arraycnt(
|
|
2720
|
+
rkbuf, of_PartitionArrayCnt, PartitionArrayCnt);
|
|
2721
|
+
rd_kafka_buf_write_tags_empty(rkbuf);
|
|
2722
|
+
}
|
|
2723
|
+
}
|
|
2724
|
+
|
|
2725
|
+
/* Write ack_details: toppars with acknowledgement batches.
|
|
2726
|
+
* Uses rktpar directly (no rktp/lock needed). */
|
|
2727
|
+
if (ack_details) {
|
|
2728
|
+
rd_kafka_Uuid_t *topic_id_last = NULL;
|
|
2729
|
+
rd_kafka_share_ack_batches_t *batches;
|
|
2730
|
+
rd_kafka_topic_partition_private_t *parpriv;
|
|
2731
|
+
rd_kafka_share_ack_batch_entry_t *entry;
|
|
2732
|
+
int k, m, entries_cnt;
|
|
2733
|
+
|
|
2734
|
+
/* TODO KIP-932: This condition will cause partitions of same
|
|
2735
|
+
topics to be inside single instance of the topic as
|
|
2736
|
+
ack_details is not sorted. Eg: T1 0, T1 1, T2 0, T1 3, T1
|
|
2737
|
+
5, T2 1 will translate to T1 (0,1), T2 (0), T1 (3, 5), T2
|
|
2738
|
+
(1) instead it should be T1 (0,1,3,5) T2(0,1) Fix this. */
|
|
2739
|
+
PartitionArrayCnt = 0;
|
|
2740
|
+
|
|
2741
|
+
RD_LIST_FOREACH(batches, ack_details, k) {
|
|
2742
|
+
/* TODO KIP-932: Ensure rktpar and rktp are in sync,
|
|
2743
|
+
* leader has not changed in between, and rktp is
|
|
2744
|
+
* present before reaching this point.
|
|
2745
|
+
* Also ensure batches->entries are not empty. */
|
|
2746
|
+
/* _private should always be present here as it was
|
|
2747
|
+
* set when building the ack batches from the
|
|
2748
|
+
* inflight map. */
|
|
2749
|
+
parpriv = (rd_kafka_topic_partition_private_t *)
|
|
2750
|
+
batches->rktpar->_private;
|
|
2751
|
+
rd_dassert(parpriv != NULL);
|
|
2752
|
+
|
|
2753
|
+
if (topic_id_last == NULL ||
|
|
2754
|
+
rd_kafka_Uuid_cmp(*topic_id_last,
|
|
2755
|
+
parpriv->topic_id) != 0) {
|
|
2756
|
+
if (topic_id_last != NULL) {
|
|
2757
|
+
/* Update PartitionArrayCnt */
|
|
2758
|
+
rd_kafka_buf_finalize_arraycnt(
|
|
2759
|
+
rkbuf, of_PartitionArrayCnt,
|
|
2760
|
+
PartitionArrayCnt);
|
|
2761
|
+
/* Topic tags */
|
|
2762
|
+
rd_kafka_buf_write_tags_empty(rkbuf);
|
|
2763
|
+
}
|
|
2764
|
+
|
|
2765
|
+
/* Topic ID */
|
|
2766
|
+
rd_kafka_buf_write_uuid(rkbuf,
|
|
2767
|
+
&parpriv->topic_id);
|
|
2768
|
+
|
|
2769
|
+
TopicArrayCnt++;
|
|
2770
|
+
topic_id_last = &parpriv->topic_id;
|
|
2771
|
+
/* Partition count */
|
|
2772
|
+
of_PartitionArrayCnt =
|
|
2773
|
+
rd_kafka_buf_write_arraycnt_pos(rkbuf);
|
|
2774
|
+
PartitionArrayCnt = 0;
|
|
2775
|
+
}
|
|
2776
|
+
|
|
2777
|
+
PartitionArrayCnt++;
|
|
2778
|
+
|
|
2779
|
+
/* Partition */
|
|
2780
|
+
rd_kafka_buf_write_i32(rkbuf,
|
|
2781
|
+
batches->rktpar->partition);
|
|
2782
|
+
|
|
2783
|
+
/* Write acknowledgement batches */
|
|
2784
|
+
entries_cnt = rd_list_cnt(&batches->entries);
|
|
2785
|
+
rd_kafka_buf_write_arraycnt(rkbuf, entries_cnt);
|
|
2786
|
+
|
|
2787
|
+
RD_LIST_FOREACH(entry, &batches->entries, m) {
|
|
2788
|
+
/* FirstOffset */
|
|
2789
|
+
rd_kafka_buf_write_i64(rkbuf,
|
|
2790
|
+
entry->start_offset);
|
|
2791
|
+
/* LastOffset */
|
|
2792
|
+
rd_kafka_buf_write_i64(rkbuf,
|
|
2793
|
+
entry->end_offset);
|
|
2794
|
+
/* AcknowledgeTypes */
|
|
2795
|
+
rd_kafka_buf_write_arraycnt(rkbuf, 1);
|
|
2796
|
+
rd_kafka_buf_write_i8(rkbuf,
|
|
2797
|
+
(int8_t)entry->types[0]);
|
|
2798
|
+
/* Acknowledgement tags */
|
|
2799
|
+
rd_kafka_buf_write_tags_empty(rkbuf);
|
|
2800
|
+
}
|
|
2801
|
+
|
|
2802
|
+
/* Partition tags */
|
|
2803
|
+
rd_kafka_buf_write_tags_empty(rkbuf);
|
|
2804
|
+
|
|
2805
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
|
|
2806
|
+
"Share Fetch ack for topic %s [%" PRId32
|
|
2807
|
+
"] with %d entries",
|
|
2808
|
+
batches->rktpar->topic,
|
|
2809
|
+
batches->rktpar->partition, entries_cnt);
|
|
2810
|
+
|
|
2811
|
+
cnt++;
|
|
2812
|
+
}
|
|
2813
|
+
|
|
2814
|
+
/* Finalize last topic from ack_details */
|
|
2815
|
+
if (topic_id_last != NULL) {
|
|
2816
|
+
rd_kafka_buf_finalize_arraycnt(
|
|
2817
|
+
rkbuf, of_PartitionArrayCnt, PartitionArrayCnt);
|
|
2818
|
+
rd_kafka_buf_write_tags_empty(rkbuf);
|
|
2819
|
+
}
|
|
2820
|
+
}
|
|
2821
|
+
|
|
2822
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
|
|
2823
|
+
"Share Fetch Request (epoch %" PRId32
|
|
2824
|
+
") with %d toppars on %d topics",
|
|
2825
|
+
rkb->rkb_share_fetch_session.epoch, cnt, TopicArrayCnt);
|
|
2826
|
+
|
|
2827
|
+
/* Update TopicArrayCnt */
|
|
2828
|
+
rd_kafka_buf_finalize_arraycnt(rkbuf, of_TopicArrayCnt, TopicArrayCnt);
|
|
2829
|
+
|
|
2830
|
+
if (toppars_to_forget) {
|
|
2831
|
+
TopicArrayCnt = 0;
|
|
2832
|
+
PartitionArrayCnt = 0;
|
|
2833
|
+
rkt_last = NULL;
|
|
2834
|
+
/* Write zero TopicArrayCnt but store pointer for later update
|
|
2835
|
+
*/
|
|
2836
|
+
of_TopicArrayCnt = rd_kafka_buf_write_arraycnt_pos(rkbuf);
|
|
2837
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", "Forgetting %d toppars",
|
|
2838
|
+
toppars_to_forget_cnt);
|
|
2839
|
+
RD_LIST_FOREACH(rktp, toppars_to_forget, i) {
|
|
2840
|
+
/* TODO KIP-932: This condition will cause partitions of
|
|
2841
|
+
same topics to be inside single instance of the topic as
|
|
2842
|
+
toppars_to_forget is not sorted. Eg: T1 0, T1 1, T2 0,
|
|
2843
|
+
T1 3, T1 5, T2 1 will translate to T1 (0,1), T2 (0),
|
|
2844
|
+
T1 (3, 5), T2 (1) instead it should be T1 (0,1,3,5)
|
|
2845
|
+
T2(0,1) Fix this. */
|
|
2846
|
+
if (rkt_last != rktp->rktp_rkt) {
|
|
2847
|
+
if (rkt_last != NULL) {
|
|
2848
|
+
/* Update PartitionArrayCnt */
|
|
2849
|
+
rd_kafka_buf_finalize_arraycnt(
|
|
2850
|
+
rkbuf, of_PartitionArrayCnt,
|
|
2851
|
+
PartitionArrayCnt);
|
|
2852
|
+
/* Topic tags */
|
|
2853
|
+
rd_kafka_buf_write_tags_empty(rkbuf);
|
|
2854
|
+
}
|
|
2855
|
+
|
|
2856
|
+
rd_kafka_topic_rdlock(rktp->rktp_rkt);
|
|
2857
|
+
/* Topic ID */
|
|
2858
|
+
rd_kafka_buf_write_uuid(
|
|
2859
|
+
rkbuf, &rktp->rktp_rkt->rkt_topic_id);
|
|
2860
|
+
rd_kafka_topic_rdunlock(rktp->rktp_rkt);
|
|
2861
|
+
|
|
2862
|
+
TopicArrayCnt++;
|
|
2863
|
+
rkt_last = rktp->rktp_rkt;
|
|
2864
|
+
/* Partition count */
|
|
2865
|
+
of_PartitionArrayCnt =
|
|
2866
|
+
rd_kafka_buf_write_arraycnt_pos(rkbuf);
|
|
2867
|
+
PartitionArrayCnt = 0;
|
|
2868
|
+
}
|
|
2869
|
+
|
|
2870
|
+
PartitionArrayCnt++;
|
|
2871
|
+
|
|
2872
|
+
/* Partition */
|
|
2873
|
+
rd_kafka_buf_write_i32(rkbuf, rktp->rktp_partition);
|
|
2874
|
+
|
|
2875
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
|
|
2876
|
+
"Forgetting Fetch partition %.*s [%" PRId32
|
|
2877
|
+
"]",
|
|
2878
|
+
RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
|
|
2879
|
+
rktp->rktp_partition);
|
|
2880
|
+
}
|
|
2881
|
+
if (rkt_last != NULL) {
|
|
2882
|
+
/* Update last topic's PartitionArrayCnt */
|
|
2883
|
+
rd_kafka_buf_finalize_arraycnt(
|
|
2884
|
+
rkbuf, of_PartitionArrayCnt, PartitionArrayCnt);
|
|
2885
|
+
/* Topic tags */
|
|
2886
|
+
rd_kafka_buf_write_tags_empty(rkbuf);
|
|
2887
|
+
}
|
|
2888
|
+
/* Update TopicArrayCnt */
|
|
2889
|
+
rd_kafka_buf_finalize_arraycnt(rkbuf, of_TopicArrayCnt,
|
|
2890
|
+
TopicArrayCnt);
|
|
2891
|
+
} else {
|
|
2892
|
+
/* ForgottenToppars */
|
|
2893
|
+
rd_kafka_buf_write_arraycnt(rkbuf, 0);
|
|
2894
|
+
}
|
|
2895
|
+
|
|
2896
|
+
/* Consider Fetch requests blocking if fetch.wait.max.ms >= 1s */
|
|
2897
|
+
if (rkb->rkb_rk->rk_conf.fetch_wait_max_ms >= 1000)
|
|
2898
|
+
rkbuf->rkbuf_flags |= RD_KAFKA_OP_F_BLOCKING;
|
|
2899
|
+
|
|
2900
|
+
/* Use configured timeout */
|
|
2901
|
+
rd_kafka_buf_set_timeout(rkbuf,
|
|
2902
|
+
rkb->rkb_rk->rk_conf.socket_timeout_ms +
|
|
2903
|
+
rkb->rkb_rk->rk_conf.fetch_wait_max_ms,
|
|
2904
|
+
now);
|
|
2905
|
+
|
|
2906
|
+
/* Copy toppars_to_add/toppars_to_forget into adding/forgetting lists
|
|
2907
|
+
* just before sending. On response, session_update() will move
|
|
2908
|
+
* adding_toppars into toppars_in_session and remove from
|
|
2909
|
+
* toppars_to_add. */
|
|
2910
|
+
if (rkb->rkb_share_fetch_session.toppars_to_add)
|
|
2911
|
+
rkb->rkb_share_fetch_session.adding_toppars =
|
|
2912
|
+
rd_list_copy(rkb->rkb_share_fetch_session.toppars_to_add,
|
|
2913
|
+
rd_kafka_toppar_list_copy, NULL);
|
|
2914
|
+
if (rkb->rkb_share_fetch_session.toppars_to_forget)
|
|
2915
|
+
rkb->rkb_share_fetch_session.forgetting_toppars =
|
|
2916
|
+
rd_list_copy(rkb->rkb_share_fetch_session.toppars_to_forget,
|
|
2917
|
+
rd_kafka_toppar_list_copy, NULL);
|
|
2918
|
+
|
|
2919
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
|
|
2920
|
+
"Issuing ShareFetch request (max wait %dms, min %d bytes, "
|
|
2921
|
+
"max %d bytes, max %d records) with %d toppars",
|
|
2922
|
+
wait_max_ms, min_bytes, max_bytes, max_records, cnt);
|
|
2923
|
+
rd_kafka_broker_buf_enq1(rkb, rkbuf, rd_kafka_broker_share_fetch_reply,
|
|
2924
|
+
rko_orig);
|
|
2925
|
+
|
|
2926
|
+
return;
|
|
2927
|
+
}
|
|
2928
|
+
|
|
2929
|
+
/**
|
|
2930
|
+
* @brief Build and send a ShareAcknowledge request.
|
|
2931
|
+
*
|
|
2932
|
+
* Used for ack-only requests (no fetching) and session close (epoch=-1).
|
|
2933
|
+
* ShareAcknowledge carries GroupId, MemberId, ShareSessionEpoch, and
|
|
2934
|
+
* acknowledgement batches only — no fetch parameters or forgotten topics.
|
|
2935
|
+
*
|
|
2936
|
+
* @param rkb Broker to send request to.
|
|
2937
|
+
* @param group_id Consumer group id.
|
|
2938
|
+
* @param member_id Consumer member id.
|
|
2939
|
+
* @param share_session_epoch Session epoch (-1 for close).
|
|
2940
|
+
* @param rko_orig The originating SHARE_FETCH op (carries ack_details).
|
|
2941
|
+
* @param now Current timestamp.
|
|
2942
|
+
*
|
|
2943
|
+
* @locality broker thread
|
|
2944
|
+
*/
|
|
2945
|
+
void rd_kafka_ShareAcknowledgeRequest(rd_kafka_broker_t *rkb,
|
|
2946
|
+
const rd_kafkap_str_t *group_id,
|
|
2947
|
+
const rd_kafkap_str_t *member_id,
|
|
2948
|
+
int32_t share_session_epoch,
|
|
2949
|
+
rd_kafka_op_t *rko_orig,
|
|
2950
|
+
rd_ts_t now) {
|
|
2951
|
+
rd_kafka_buf_t *rkbuf;
|
|
2952
|
+
int cnt = 0;
|
|
2953
|
+
size_t of_TopicArrayCnt = 0;
|
|
2954
|
+
int TopicArrayCnt = 0;
|
|
2955
|
+
size_t of_PartitionArrayCnt = 0;
|
|
2956
|
+
int PartitionArrayCnt = 0;
|
|
2957
|
+
size_t rkbuf_size = 0;
|
|
2958
|
+
rd_list_t *ack_details =
|
|
2959
|
+
rko_orig ? rko_orig->rko_u.share_fetch.ack_details : NULL;
|
|
2960
|
+
int ack_details_cnt = ack_details ? rd_list_cnt(ack_details) : 0;
|
|
2961
|
+
int total_ack_entries = 0;
|
|
2962
|
+
int64_t total_ack_records = 0;
|
|
2963
|
+
/* FirstOffset + LastOffset + AcknowledgementType per ack entry */
|
|
2964
|
+
size_t acknowledgement_size = 8 + 8 + 1;
|
|
2965
|
+
|
|
2966
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREACK",
|
|
2967
|
+
"ack_details_cnt=%d, share_session_epoch=%" PRId32,
|
|
2968
|
+
ack_details_cnt, share_session_epoch);
|
|
2969
|
+
|
|
2970
|
+
/* Calculate buffer size */
|
|
2971
|
+
if (group_id)
|
|
2972
|
+
rkbuf_size += RD_KAFKAP_STR_SIZE(group_id);
|
|
2973
|
+
if (member_id)
|
|
2974
|
+
rkbuf_size += RD_KAFKAP_STR_SIZE(member_id);
|
|
2975
|
+
/* ShareSessionEpoch + TopicArrayCnt */
|
|
2976
|
+
rkbuf_size += 4 + 4;
|
|
2977
|
+
/* N x (topic id + partition id) for ack details */
|
|
2978
|
+
rkbuf_size += (ack_details_cnt * (32 + 4));
|
|
2979
|
+
/* Sum total ack entries and total records across all
|
|
2980
|
+
* ack_details batches. */
|
|
2981
|
+
rd_kafka_share_ack_details_totals(ack_details, &total_ack_entries,
|
|
2982
|
+
&total_ack_records);
|
|
2983
|
+
/* Accumulate standalone record-level acknowledgements for
|
|
2984
|
+
* consumer.share.fetch.manager.acknowledgements.send.{total,rate}. */
|
|
2985
|
+
if (total_ack_records > 0)
|
|
2986
|
+
rd_atomic64_add(
|
|
2987
|
+
&rkb->rkb_rk->rk_telemetry.acknowledgements_send_total,
|
|
2988
|
+
total_ack_records);
|
|
2989
|
+
|
|
2990
|
+
rkbuf_size += (total_ack_entries * acknowledgement_size);
|
|
2991
|
+
|
|
2992
|
+
rkbuf = rd_kafka_buf_new_flexver_request(
|
|
2993
|
+
rkb, RD_KAFKAP_ShareAcknowledge, 1, rkbuf_size, rd_true);
|
|
2994
|
+
|
|
2995
|
+
rd_kafka_buf_ApiVersion_set(
|
|
2996
|
+
rkbuf,
|
|
2997
|
+
rd_kafka_broker_ApiVersion_supported(
|
|
2998
|
+
rkb, RD_KAFKAP_ShareAcknowledge, 1, 1, NULL),
|
|
2999
|
+
0);
|
|
3000
|
+
|
|
3001
|
+
/* GroupId */
|
|
3002
|
+
rd_kafka_buf_write_kstr(rkbuf, group_id);
|
|
3003
|
+
|
|
3004
|
+
/* MemberId */
|
|
3005
|
+
rd_kafka_buf_write_kstr(rkbuf, member_id);
|
|
3006
|
+
|
|
3007
|
+
/* ShareSessionEpoch */
|
|
3008
|
+
rd_kafka_buf_write_i32(rkbuf, share_session_epoch);
|
|
3009
|
+
|
|
3010
|
+
/* Topics array with acknowledgement batches */
|
|
3011
|
+
of_TopicArrayCnt = rd_kafka_buf_write_arraycnt_pos(rkbuf);
|
|
3012
|
+
|
|
3013
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREACK",
|
|
3014
|
+
"Building ShareAcknowledge request with %d ack toppars "
|
|
3015
|
+
"and %d total ack entries",
|
|
3016
|
+
ack_details_cnt, total_ack_entries);
|
|
3017
|
+
|
|
3018
|
+
if (ack_details) {
|
|
3019
|
+
rd_kafka_Uuid_t *topic_id_last = NULL;
|
|
3020
|
+
rd_kafka_share_ack_batches_t *batches;
|
|
3021
|
+
rd_kafka_topic_partition_private_t *parpriv;
|
|
3022
|
+
rd_kafka_share_ack_batch_entry_t *entry;
|
|
3023
|
+
int k, m, entries_cnt;
|
|
3024
|
+
|
|
3025
|
+
RD_LIST_FOREACH(batches, ack_details, k) {
|
|
3026
|
+
parpriv = (rd_kafka_topic_partition_private_t *)
|
|
3027
|
+
batches->rktpar->_private;
|
|
3028
|
+
rd_dassert(parpriv != NULL);
|
|
3029
|
+
|
|
3030
|
+
if (topic_id_last == NULL ||
|
|
3031
|
+
rd_kafka_Uuid_cmp(*topic_id_last,
|
|
3032
|
+
parpriv->topic_id) != 0) {
|
|
3033
|
+
if (topic_id_last != NULL) {
|
|
3034
|
+
rd_kafka_buf_finalize_arraycnt(
|
|
3035
|
+
rkbuf, of_PartitionArrayCnt,
|
|
3036
|
+
PartitionArrayCnt);
|
|
3037
|
+
rd_kafka_buf_write_tags_empty(rkbuf);
|
|
3038
|
+
}
|
|
3039
|
+
|
|
3040
|
+
/* Topic ID */
|
|
3041
|
+
rd_kafka_buf_write_uuid(rkbuf,
|
|
3042
|
+
&parpriv->topic_id);
|
|
3043
|
+
|
|
3044
|
+
TopicArrayCnt++;
|
|
3045
|
+
topic_id_last = &parpriv->topic_id;
|
|
3046
|
+
of_PartitionArrayCnt =
|
|
3047
|
+
rd_kafka_buf_write_arraycnt_pos(rkbuf);
|
|
3048
|
+
PartitionArrayCnt = 0;
|
|
3049
|
+
}
|
|
3050
|
+
|
|
3051
|
+
PartitionArrayCnt++;
|
|
3052
|
+
|
|
3053
|
+
/* Partition */
|
|
3054
|
+
rd_kafka_buf_write_i32(rkbuf,
|
|
3055
|
+
batches->rktpar->partition);
|
|
3056
|
+
|
|
3057
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREACK",
|
|
3058
|
+
"Adding ack for topic %s [%" PRId32
|
|
3059
|
+
"] with %d entries",
|
|
3060
|
+
batches->rktpar->topic,
|
|
3061
|
+
batches->rktpar->partition,
|
|
3062
|
+
rd_list_cnt(&batches->entries));
|
|
3063
|
+
|
|
3064
|
+
/* Write acknowledgement batches */
|
|
3065
|
+
entries_cnt = rd_list_cnt(&batches->entries);
|
|
3066
|
+
rd_kafka_buf_write_arraycnt(rkbuf, entries_cnt);
|
|
3067
|
+
|
|
3068
|
+
RD_LIST_FOREACH(entry, &batches->entries, m) {
|
|
3069
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREACK",
|
|
3070
|
+
"Adding ack entry with start offset "
|
|
3071
|
+
"%" PRId64 ", end offset %" PRId64
|
|
3072
|
+
", type %d",
|
|
3073
|
+
entry->start_offset,
|
|
3074
|
+
entry->end_offset, entry->types[0]);
|
|
3075
|
+
/* FirstOffset */
|
|
3076
|
+
rd_kafka_buf_write_i64(rkbuf,
|
|
3077
|
+
entry->start_offset);
|
|
3078
|
+
/* LastOffset */
|
|
3079
|
+
rd_kafka_buf_write_i64(rkbuf,
|
|
3080
|
+
entry->end_offset);
|
|
3081
|
+
/* AcknowledgeTypes */
|
|
3082
|
+
rd_kafka_buf_write_arraycnt(rkbuf, 1);
|
|
3083
|
+
rd_kafka_buf_write_i8(rkbuf,
|
|
3084
|
+
(int8_t)entry->types[0]);
|
|
3085
|
+
/* Acknowledgement tags */
|
|
3086
|
+
rd_kafka_buf_write_tags_empty(rkbuf);
|
|
3087
|
+
}
|
|
3088
|
+
|
|
3089
|
+
/* Partition tags */
|
|
3090
|
+
rd_kafka_buf_write_tags_empty(rkbuf);
|
|
3091
|
+
|
|
3092
|
+
cnt++;
|
|
3093
|
+
}
|
|
3094
|
+
|
|
3095
|
+
/* Finalize last topic from ack_details */
|
|
3096
|
+
if (topic_id_last != NULL) {
|
|
3097
|
+
rd_kafka_buf_finalize_arraycnt(
|
|
3098
|
+
rkbuf, of_PartitionArrayCnt, PartitionArrayCnt);
|
|
3099
|
+
rd_kafka_buf_write_tags_empty(rkbuf);
|
|
3100
|
+
}
|
|
3101
|
+
}
|
|
3102
|
+
|
|
3103
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREACK",
|
|
3104
|
+
"ShareAcknowledge Request with %d toppars on %d topics", cnt,
|
|
3105
|
+
TopicArrayCnt);
|
|
3106
|
+
|
|
3107
|
+
/* Update TopicArrayCnt */
|
|
3108
|
+
rd_kafka_buf_finalize_arraycnt(rkbuf, of_TopicArrayCnt, TopicArrayCnt);
|
|
3109
|
+
|
|
3110
|
+
/* Wire RPC timeout is socket.timeout.ms, decoupled from any
|
|
3111
|
+
* caller-side commit_sync deadline. Interactions covered by
|
|
3112
|
+
* tests/0182-share_consumer_error_handling_mock.c
|
|
3113
|
+
* (do_test_socket_timeout_full_ack_then_more). */
|
|
3114
|
+
rd_kafka_buf_set_timeout(rkbuf, rkb->rkb_rk->rk_conf.socket_timeout_ms,
|
|
3115
|
+
now);
|
|
3116
|
+
|
|
3117
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREACK",
|
|
3118
|
+
"Issuing ShareAcknowledge request with %d toppars", cnt);
|
|
3119
|
+
rd_kafka_broker_buf_enq1(
|
|
3120
|
+
rkb, rkbuf, rd_kafka_broker_share_acknowledge_reply, rko_orig);
|
|
3121
|
+
}
|
|
3122
|
+
|
|
3123
|
+
|
|
3124
|
+
void rd_kafka_broker_share_fetch_session_clear(rd_kafka_broker_t *rkb) {
|
|
3125
|
+
/* Clear toppars in session */
|
|
3126
|
+
rd_rkb_dbg(
|
|
3127
|
+
rkb, BROKER, "SHARESESSION",
|
|
3128
|
+
"Clearing %d toppars from ShareFetch session",
|
|
3129
|
+
rd_list_cnt(rkb->rkb_share_fetch_session.toppars_in_session));
|
|
3130
|
+
rd_list_clear(rkb->rkb_share_fetch_session.toppars_in_session);
|
|
3131
|
+
|
|
3132
|
+
/* Clear toppars to add */
|
|
3133
|
+
if (rkb->rkb_share_fetch_session.toppars_to_add) {
|
|
3134
|
+
rd_rkb_dbg(
|
|
3135
|
+
rkb, BROKER, "SHARESESSION",
|
|
3136
|
+
"Clearing %d toppars to add from ShareFetch session on "
|
|
3137
|
+
"clear",
|
|
3138
|
+
rd_list_cnt(rkb->rkb_share_fetch_session.toppars_to_add));
|
|
3139
|
+
rd_list_destroy(rkb->rkb_share_fetch_session.toppars_to_add);
|
|
3140
|
+
rkb->rkb_share_fetch_session.toppars_to_add = NULL;
|
|
3141
|
+
}
|
|
3142
|
+
|
|
3143
|
+
/* Clear toppars to forget */
|
|
3144
|
+
if (rkb->rkb_share_fetch_session.toppars_to_forget) {
|
|
3145
|
+
rd_rkb_dbg(rkb, BROKER, "SHARESESSION",
|
|
3146
|
+
"Clearing %d toppars to forget from ShareFetch "
|
|
3147
|
+
"session on clear",
|
|
3148
|
+
rd_list_cnt(
|
|
3149
|
+
rkb->rkb_share_fetch_session.toppars_to_forget));
|
|
3150
|
+
rd_list_destroy(rkb->rkb_share_fetch_session.toppars_to_forget);
|
|
3151
|
+
rkb->rkb_share_fetch_session.toppars_to_forget = NULL;
|
|
3152
|
+
}
|
|
3153
|
+
|
|
3154
|
+
/* Clear adding toppars */
|
|
3155
|
+
if (rkb->rkb_share_fetch_session.adding_toppars) {
|
|
3156
|
+
rd_rkb_dbg(
|
|
3157
|
+
rkb, BROKER, "SHARESESSION",
|
|
3158
|
+
"Clearing %d adding toppars from ShareFetch session on "
|
|
3159
|
+
"clear",
|
|
3160
|
+
rd_list_cnt(rkb->rkb_share_fetch_session.adding_toppars));
|
|
3161
|
+
rd_list_destroy(rkb->rkb_share_fetch_session.adding_toppars);
|
|
3162
|
+
rkb->rkb_share_fetch_session.adding_toppars = NULL;
|
|
3163
|
+
}
|
|
3164
|
+
|
|
3165
|
+
/* Clear forgetting toppars */
|
|
3166
|
+
if (rkb->rkb_share_fetch_session.forgetting_toppars) {
|
|
3167
|
+
rd_rkb_dbg(
|
|
3168
|
+
rkb, BROKER, "SHARESESSION",
|
|
3169
|
+
"Clearing %d forgetting toppars from ShareFetch session on "
|
|
3170
|
+
"clear",
|
|
3171
|
+
rd_list_cnt(
|
|
3172
|
+
rkb->rkb_share_fetch_session.forgetting_toppars));
|
|
3173
|
+
rd_list_destroy(
|
|
3174
|
+
rkb->rkb_share_fetch_session.forgetting_toppars);
|
|
3175
|
+
rkb->rkb_share_fetch_session.forgetting_toppars = NULL;
|
|
3176
|
+
}
|
|
3177
|
+
|
|
3178
|
+
/*
|
|
3179
|
+
* This allows us to avoid future changes to the closed share session
|
|
3180
|
+
* Toppar add/remove functions do not allow updates if epoch is -1
|
|
3181
|
+
* Avoid future changes to the closed share session */
|
|
3182
|
+
rkb->rkb_share_fetch_session.epoch = -1;
|
|
3183
|
+
}
|
|
3184
|
+
|
|
3185
|
+
void rd_kafka_broker_share_fetch_session_leave(rd_kafka_broker_t *rkb,
|
|
3186
|
+
rd_kafka_op_t *rko_orig,
|
|
3187
|
+
rd_ts_t now) {
|
|
3188
|
+
rd_kafka_cgrp_t *rkcg = rkb->rkb_rk->rk_cgrp;
|
|
3189
|
+
|
|
3190
|
+
if (rkb->rkb_share_fetch_session.epoch > 0) {
|
|
3191
|
+
rd_rkb_dbg(rkb, BROKER, "SHARESESSION",
|
|
3192
|
+
"Processing SHARE_FETCH op: "
|
|
3193
|
+
"should_leave = true");
|
|
3194
|
+
|
|
3195
|
+
/* Set epoch to -1 to signal session close before sending
|
|
3196
|
+
* request. This ensures session_update_epoch() skips
|
|
3197
|
+
* incrementing on reply. */
|
|
3198
|
+
rkb->rkb_share_fetch_session.epoch = -1;
|
|
3199
|
+
rd_kafka_ShareAcknowledgeRequest(
|
|
3200
|
+
rkb, rkcg->rkcg_group_id, rkcg->rkcg_member_id,
|
|
3201
|
+
-1, /* epoch=-1 signals session close */
|
|
3202
|
+
rko_orig, now);
|
|
3203
|
+
} else {
|
|
3204
|
+
rd_rkb_dbg(rkb, BROKER, "SHARESESSION",
|
|
3205
|
+
"Ignoring SHARE_FETCH op with "
|
|
3206
|
+
"should_leave = true: "
|
|
3207
|
+
"no active session");
|
|
3208
|
+
|
|
3209
|
+
if (rkb->rkb_share_fetch_session.epoch == 0)
|
|
3210
|
+
/* Required as it is possible that we were about
|
|
3211
|
+
* to establish a session */
|
|
3212
|
+
rd_kafka_broker_share_fetch_session_clear(rkb);
|
|
3213
|
+
rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err(
|
|
3214
|
+
rko_orig, RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND);
|
|
3215
|
+
}
|
|
3216
|
+
}
|
|
3217
|
+
|
|
3218
|
+
void rd_kafka_broker_share_rpc(rd_kafka_broker_t *rkb,
|
|
3219
|
+
rd_kafka_op_t *rko_orig,
|
|
3220
|
+
rd_ts_t now) {
|
|
3221
|
+
|
|
3222
|
+
rd_kafka_cgrp_t *rkcg = rkb->rkb_rk->rk_cgrp;
|
|
3223
|
+
int32_t max_records = 0;
|
|
3224
|
+
rd_list_t *ack_details = rko_orig->rko_u.share_fetch.ack_details;
|
|
3225
|
+
rd_bool_t has_ack_details = ack_details && rd_list_cnt(ack_details) > 0;
|
|
3226
|
+
rd_list_t *toppars_to_add = NULL;
|
|
3227
|
+
rd_list_t *toppars_to_forget = NULL;
|
|
3228
|
+
rd_list_t *stripped_acks = NULL;
|
|
3229
|
+
|
|
3230
|
+
if (!rko_orig->rko_u.share_fetch.should_fetch && !has_ack_details) {
|
|
3231
|
+
rd_rkb_dbg(rkb, FETCH, "SHARERPC",
|
|
3232
|
+
"Not sending Share RPC: "
|
|
3233
|
+
"no fetch requested and no acknowledgements");
|
|
3234
|
+
rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err(
|
|
3235
|
+
rko_orig, RD_KAFKA_RESP_ERR__NOOP);
|
|
3236
|
+
return;
|
|
3237
|
+
}
|
|
3238
|
+
|
|
3239
|
+
if (!rkcg->rkcg_member_id) {
|
|
3240
|
+
rd_rkb_dbg(rkb, FETCH, "SHARERPC",
|
|
3241
|
+
"Share RPC requested without member_id");
|
|
3242
|
+
rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err(
|
|
3243
|
+
rko_orig, RD_KAFKA_RESP_ERR__INVALID_ARG);
|
|
3244
|
+
return;
|
|
3245
|
+
}
|
|
3246
|
+
|
|
3247
|
+
if (!rko_orig->rko_u.share_fetch.should_fetch) {
|
|
3248
|
+
/* Ack-only: use ShareAcknowledge RPC.
|
|
3249
|
+
*
|
|
3250
|
+
* If session epoch is 0 (new consumer or post-reset),
|
|
3251
|
+
* the broker has no session state to acknowledge
|
|
3252
|
+
* against. Fail the acks locally with
|
|
3253
|
+
* INVALID_SHARE_SESSION_EPOCH instead of sending,
|
|
3254
|
+
* since the broker would reject the request. */
|
|
3255
|
+
if (rkb->rkb_share_fetch_session.epoch == 0) {
|
|
3256
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREACK",
|
|
3257
|
+
"Failing %d ack batches locally: "
|
|
3258
|
+
"session epoch is 0 (no session)",
|
|
3259
|
+
rd_list_cnt(ack_details));
|
|
3260
|
+
rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err(
|
|
3261
|
+
rko_orig,
|
|
3262
|
+
RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH);
|
|
3263
|
+
return;
|
|
3264
|
+
}
|
|
3265
|
+
|
|
3266
|
+
/* TODO KIP-932: add a defensive per-batch leader-stale
|
|
3267
|
+
* strip here (mirroring the segregation-time check) to
|
|
3268
|
+
* cover the window where the cached leader changes
|
|
3269
|
+
* between segregation on the main thread and this RPC
|
|
3270
|
+
* send on the broker thread. It must run AFTER the
|
|
3271
|
+
* epoch==0 check above so the local-only failure path
|
|
3272
|
+
* stays identical to the broker-roundtrip flow. */
|
|
3273
|
+
|
|
3274
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREACK",
|
|
3275
|
+
"Sending ShareAcknowledge Request with"
|
|
3276
|
+
" acknowledgements");
|
|
3277
|
+
|
|
3278
|
+
rd_kafka_ShareAcknowledgeRequest(
|
|
3279
|
+
rkb, rkcg->rkcg_group_id, rkcg->rkcg_member_id,
|
|
3280
|
+
rkb->rkb_share_fetch_session.epoch, rko_orig, now);
|
|
3281
|
+
return;
|
|
3282
|
+
}
|
|
3283
|
+
|
|
3284
|
+
max_records = rkb->rkb_rk->rk_conf.share.max_poll_records;
|
|
3285
|
+
toppars_to_add = rkb->rkb_share_fetch_session.toppars_to_add;
|
|
3286
|
+
toppars_to_forget = rkb->rkb_share_fetch_session.toppars_to_forget;
|
|
3287
|
+
|
|
3288
|
+
/* If session epoch is 0 (new/reset session) and we have
|
|
3289
|
+
* piggybacked acks, the broker has no session state to ack
|
|
3290
|
+
* against. Fail the acks locally with
|
|
3291
|
+
* INVALID_SHARE_SESSION_EPOCH and strip them from the wire
|
|
3292
|
+
* request. The ShareFetch itself still goes out to establish
|
|
3293
|
+
* the session. */
|
|
3294
|
+
if (rkb->rkb_share_fetch_session.epoch == 0 && has_ack_details) {
|
|
3295
|
+
rd_kafka_share_ack_batches_t *batch;
|
|
3296
|
+
int i;
|
|
3297
|
+
|
|
3298
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
|
|
3299
|
+
"Stripping %d piggybacked ack batches: "
|
|
3300
|
+
"session epoch is 0 (no session)",
|
|
3301
|
+
rd_list_cnt(ack_details));
|
|
3302
|
+
|
|
3303
|
+
/* Pre-set each batch's err. Conditional buf-cb init
|
|
3304
|
+
* preserves this; main reply handler propagates to
|
|
3305
|
+
* commit_sync results / acknowledgement callback. */
|
|
3306
|
+
RD_LIST_FOREACH(batch, ack_details, i) {
|
|
3307
|
+
batch->rktpar->err =
|
|
3308
|
+
RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH;
|
|
3309
|
+
}
|
|
3310
|
+
|
|
3311
|
+
/* Detach so ShareFetchRequest builds without the ack
|
|
3312
|
+
* data section. Restored after the request is built
|
|
3313
|
+
* so the buf reply handler / main thread reply
|
|
3314
|
+
* handler can still read the per-batch err. */
|
|
3315
|
+
stripped_acks = rko_orig->rko_u.share_fetch.ack_details;
|
|
3316
|
+
rko_orig->rko_u.share_fetch.ack_details = NULL;
|
|
3317
|
+
has_ack_details = rd_false;
|
|
3318
|
+
}
|
|
3319
|
+
|
|
3320
|
+
/* TODO KIP-932: add a defensive per-batch leader-stale strip
|
|
3321
|
+
* here (mirroring the segregation-time check) to cover the
|
|
3322
|
+
* window where the cached leader changes between segregation
|
|
3323
|
+
* on the main thread and this RPC send on the broker thread.
|
|
3324
|
+
* It must run AFTER the epoch==0 strip above so the local-only
|
|
3325
|
+
* failure path stays identical to the broker-roundtrip flow. */
|
|
3326
|
+
|
|
3327
|
+
rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
|
|
3328
|
+
"Sending ShareFetch Request (epoch %" PRId32
|
|
3329
|
+
") with%s%s%s fetching messages",
|
|
3330
|
+
rkb->rkb_share_fetch_session.epoch,
|
|
3331
|
+
has_ack_details ? " acknowledgements," : "",
|
|
3332
|
+
toppars_to_add ? " new topics," : "",
|
|
3333
|
+
toppars_to_forget ? " forgotten toppars," : "");
|
|
3334
|
+
|
|
3335
|
+
rd_kafka_ShareFetchRequest(
|
|
3336
|
+
rkb, rkcg->rkcg_group_id, /* group_id */
|
|
3337
|
+
rkcg->rkcg_member_id, /* member_id */
|
|
3338
|
+
rkb->rkb_share_fetch_session.epoch, /* share_session_epoch */
|
|
3339
|
+
rkb->rkb_rk->rk_conf.fetch_wait_max_ms,
|
|
3340
|
+
rkb->rkb_rk->rk_conf.fetch_min_bytes,
|
|
3341
|
+
rkb->rkb_rk->rk_conf.fetch_max_bytes, max_records,
|
|
3342
|
+
max_records, /* batch_size: same value as max_records,
|
|
3343
|
+
* both sourced from max.poll.records */
|
|
3344
|
+
toppars_to_add, /* toppars to add to session */
|
|
3345
|
+
toppars_to_forget, /* forgetting toppars */
|
|
3346
|
+
rko_orig, /* rko (carries ack_details) */
|
|
3347
|
+
now);
|
|
3348
|
+
|
|
3349
|
+
/* Restore ack_details so the buf reply handler / main thread
|
|
3350
|
+
* reply handler can read the pre-set per-batch err. */
|
|
3351
|
+
if (stripped_acks)
|
|
3352
|
+
rko_orig->rko_u.share_fetch.ack_details = stripped_acks;
|
|
3353
|
+
}
|
|
3354
|
+
|
|
3355
|
+
/**
|
|
3356
|
+
* @brief Build and send a Fetch request message for all underflowed toppars
|
|
3357
|
+
* for a specific broker.
|
|
3358
|
+
*
|
|
3359
|
+
* @returns the number of partitions included in the FetchRequest, if any.
|
|
3360
|
+
*
|
|
3361
|
+
* @locality broker thread
|
|
3362
|
+
*/
|
|
3363
|
+
int rd_kafka_broker_fetch_toppars(rd_kafka_broker_t *rkb, rd_ts_t now) {
|
|
3364
|
+
rd_kafka_toppar_t *rktp;
|
|
3365
|
+
rd_kafka_buf_t *rkbuf;
|
|
3366
|
+
int cnt = 0;
|
|
3367
|
+
size_t of_TopicArrayCnt = 0;
|
|
3368
|
+
int TopicArrayCnt = 0;
|
|
3369
|
+
size_t of_PartitionArrayCnt = 0;
|
|
3370
|
+
int PartitionArrayCnt = 0;
|
|
3371
|
+
rd_kafka_topic_t *rkt_last = NULL;
|
|
3372
|
+
int16_t ApiVersion = 0;
|
|
3373
|
+
|
|
3374
|
+
/* Create buffer and segments:
|
|
3375
|
+
* 1 x ReplicaId MaxWaitTime MinBytes TopicArrayCnt
|
|
3376
|
+
* N x topic name
|
|
3377
|
+
* N x PartitionArrayCnt Partition FetchOffset MaxBytes
|
|
3378
|
+
* where N = number of toppars.
|
|
3379
|
+
* Since we dont keep track of the number of topics served by
|
|
3380
|
+
* this broker, only the partition count, we do a worst-case calc
|
|
3381
|
+
* when allocating and assume each partition is on its own topic
|
|
3382
|
+
*/
|
|
3383
|
+
|
|
3384
|
+
if (unlikely(rkb->rkb_active_toppar_cnt == 0))
|
|
3385
|
+
return 0;
|
|
3386
|
+
|
|
3387
|
+
ApiVersion = rd_kafka_broker_ApiVersion_supported(rkb, RD_KAFKAP_Fetch,
|
|
3388
|
+
0, 16, NULL);
|
|
3389
|
+
|
|
3390
|
+
/* Fallback to version 12 if topic id is null which can happen if
|
|
3391
|
+
* inter.broker.protocol.version is < 2.8 */
|
|
3392
|
+
if (ApiVersion > 12 && !can_use_topic_ids(rkb))
|
|
3393
|
+
ApiVersion = 12;
|
|
3394
|
+
|
|
3395
|
+
rkbuf = rd_kafka_buf_new_flexver_request(
|
|
3396
|
+
rkb, RD_KAFKAP_Fetch, 1,
|
|
3397
|
+
/* MaxWaitTime+MinBytes+MaxBytes+IsolationLevel+
|
|
3398
|
+
* SessionId+Epoch+TopicCnt */
|
|
3399
|
+
4 + 4 + 4 + 1 + 4 + 4 + 4 +
|
|
3400
|
+
/* N x PartCnt+Partition+CurrentLeaderEpoch+FetchOffset+
|
|
3401
|
+
* LastFetchedEpoch+LogStartOffset+MaxBytes+?TopicNameLen?*/
|
|
3402
|
+
(rkb->rkb_active_toppar_cnt *
|
|
3403
|
+
(4 + 4 + 4 + 8 + 4 + 8 + 4 + 40)) +
|
|
3404
|
+
/* ForgottenTopicsCnt */
|
|
3405
|
+
4 +
|
|
3406
|
+
/* N x ForgottenTopicsData */
|
|
3407
|
+
0,
|
|
3408
|
+
ApiVersion >= 12);
|
|
3409
|
+
|
|
3410
|
+
if (rkb->rkb_features & RD_KAFKA_FEATURE_MSGVER2)
|
|
3411
|
+
rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion,
|
|
3412
|
+
RD_KAFKA_FEATURE_MSGVER2);
|
|
3413
|
+
else if (rkb->rkb_features & RD_KAFKA_FEATURE_MSGVER1)
|
|
3414
|
+
rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion,
|
|
3415
|
+
RD_KAFKA_FEATURE_MSGVER1);
|
|
3416
|
+
else if (rkb->rkb_features & RD_KAFKA_FEATURE_THROTTLETIME)
|
|
3417
|
+
rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion,
|
|
3418
|
+
RD_KAFKA_FEATURE_THROTTLETIME);
|
|
3419
|
+
|
|
3420
|
+
|
|
3421
|
+
/* FetchRequest header */
|
|
3422
|
+
if (rd_kafka_buf_ApiVersion(rkbuf) <= 14)
|
|
3423
|
+
/* ReplicaId */
|
|
3424
|
+
rd_kafka_buf_write_i32(rkbuf, -1);
|
|
3425
|
+
|
|
3426
|
+
/* MaxWaitTime */
|
|
3427
|
+
rd_kafka_buf_write_i32(rkbuf, rkb->rkb_rk->rk_conf.fetch_wait_max_ms);
|
|
3428
|
+
/* MinBytes */
|
|
3429
|
+
rd_kafka_buf_write_i32(rkbuf, rkb->rkb_rk->rk_conf.fetch_min_bytes);
|
|
3430
|
+
|
|
3431
|
+
if (rd_kafka_buf_ApiVersion(rkbuf) >= 3)
|
|
3432
|
+
/* MaxBytes */
|
|
3433
|
+
rd_kafka_buf_write_i32(rkbuf,
|
|
3434
|
+
rkb->rkb_rk->rk_conf.fetch_max_bytes);
|
|
3435
|
+
|
|
3436
|
+
if (rd_kafka_buf_ApiVersion(rkbuf) >= 4)
|
|
3437
|
+
/* IsolationLevel */
|
|
3438
|
+
rd_kafka_buf_write_i8(rkbuf,
|
|
3439
|
+
rkb->rkb_rk->rk_conf.isolation_level);
|
|
3440
|
+
|
|
3441
|
+
if (rd_kafka_buf_ApiVersion(rkbuf) >= 7) {
|
|
3442
|
+
/* SessionId */
|
|
3443
|
+
rd_kafka_buf_write_i32(rkbuf, 0);
|
|
3444
|
+
/* Epoch */
|
|
3445
|
+
rd_kafka_buf_write_i32(rkbuf, -1);
|
|
3446
|
+
}
|
|
3447
|
+
|
|
3448
|
+
/* Write zero TopicArrayCnt but store pointer for later update */
|
|
3449
|
+
of_TopicArrayCnt = rd_kafka_buf_write_arraycnt_pos(rkbuf);
|
|
3450
|
+
|
|
3451
|
+
/* Prepare map for storing the fetch version for each partition,
|
|
3452
|
+
* this will later be checked in Fetch response to purge outdated
|
|
3453
|
+
* responses (e.g., after a seek). */
|
|
3454
|
+
rkbuf->rkbuf_rktp_vers =
|
|
3455
|
+
rd_list_new(0, (void *)rd_kafka_toppar_ver_destroy);
|
|
3456
|
+
rd_list_prealloc_elems(rkbuf->rkbuf_rktp_vers,
|
|
3457
|
+
sizeof(struct rd_kafka_toppar_ver),
|
|
3458
|
+
rkb->rkb_active_toppar_cnt, 0);
|
|
3459
|
+
|
|
3460
|
+
/* Round-robin start of the list. */
|
|
3461
|
+
rktp = rkb->rkb_active_toppar_next;
|
|
3462
|
+
do {
|
|
3463
|
+
struct rd_kafka_toppar_ver *tver;
|
|
3464
|
+
|
|
3465
|
+
if (rkt_last != rktp->rktp_rkt) {
|
|
3466
|
+
if (rkt_last != NULL) {
|
|
3467
|
+
/* Update PartitionArrayCnt */
|
|
3468
|
+
rd_kafka_buf_finalize_arraycnt(
|
|
3469
|
+
rkbuf, of_PartitionArrayCnt,
|
|
3470
|
+
PartitionArrayCnt);
|
|
3471
|
+
/* Topic tags */
|
|
3472
|
+
rd_kafka_buf_write_tags_empty(rkbuf);
|
|
3473
|
+
}
|
|
3474
|
+
|
|
3475
|
+
/* TODO: This is not thread safe as topic can
|
|
3476
|
+
be recreated in which case topic id is
|
|
3477
|
+
updated from the main thread and we are
|
|
3478
|
+
sending topic id from broker thread.*/
|
|
3479
|
+
if (rd_kafka_buf_ApiVersion(rkbuf) > 12) {
|
|
3480
|
+
/* Topic id must be non-zero here */
|
|
3481
|
+
rd_dassert(!RD_KAFKA_UUID_IS_ZERO(
|
|
3482
|
+
rktp->rktp_rkt->rkt_topic_id));
|
|
3483
|
+
/* Topic ID */
|
|
3484
|
+
rd_kafka_buf_write_uuid(
|
|
3485
|
+
rkbuf, &rktp->rktp_rkt->rkt_topic_id);
|
|
3486
|
+
} else {
|
|
3487
|
+
/* Topic name */
|
|
3488
|
+
rd_kafka_buf_write_kstr(
|
|
3489
|
+
rkbuf, rktp->rktp_rkt->rkt_topic);
|
|
3490
|
+
}
|
|
3491
|
+
|
|
3492
|
+
TopicArrayCnt++;
|
|
3493
|
+
rkt_last = rktp->rktp_rkt;
|
|
3494
|
+
/* Partition count */
|
|
3495
|
+
of_PartitionArrayCnt =
|
|
3496
|
+
rd_kafka_buf_write_arraycnt_pos(rkbuf);
|
|
3497
|
+
PartitionArrayCnt = 0;
|
|
3498
|
+
}
|
|
3499
|
+
|
|
3500
|
+
PartitionArrayCnt++;
|
|
3501
|
+
|
|
3502
|
+
/* Partition */
|
|
3503
|
+
rd_kafka_buf_write_i32(rkbuf, rktp->rktp_partition);
|
|
3504
|
+
|
|
3505
|
+
if (rd_kafka_buf_ApiVersion(rkbuf) >= 9) {
|
|
3506
|
+
/* CurrentLeaderEpoch */
|
|
3507
|
+
if (rktp->rktp_leader_epoch < 0 &&
|
|
3508
|
+
rd_kafka_has_reliable_leader_epochs(rkb)) {
|
|
3509
|
+
/* If current leader epoch is set to -1 and
|
|
3510
|
+
* the broker has reliable leader epochs,
|
|
3511
|
+
* send 0 instead, so that epoch is checked
|
|
3512
|
+
* and optionally metadata is refreshed.
|
|
3513
|
+
* This can happen if metadata is read initially
|
|
3514
|
+
* without an existing topic (see
|
|
3515
|
+
* rd_kafka_topic_metadata_update2).
|
|
3516
|
+
*/
|
|
3517
|
+
rd_kafka_buf_write_i32(rkbuf, 0);
|
|
3518
|
+
} else {
|
|
3519
|
+
rd_kafka_buf_write_i32(rkbuf,
|
|
3520
|
+
rktp->rktp_leader_epoch);
|
|
3521
|
+
}
|
|
3522
|
+
}
|
|
3523
|
+
/* FetchOffset */
|
|
3524
|
+
rd_kafka_buf_write_i64(rkbuf,
|
|
3525
|
+
rktp->rktp_offsets.fetch_pos.offset);
|
|
3526
|
+
if (rd_kafka_buf_ApiVersion(rkbuf) >= 12)
|
|
3527
|
+
/* LastFetchedEpoch - only used by follower replica */
|
|
3528
|
+
rd_kafka_buf_write_i32(rkbuf, -1);
|
|
3529
|
+
if (rd_kafka_buf_ApiVersion(rkbuf) >= 5)
|
|
3530
|
+
/* LogStartOffset - only used by follower replica */
|
|
3531
|
+
rd_kafka_buf_write_i64(rkbuf, -1);
|
|
3532
|
+
|
|
3533
|
+
/* MaxBytes */
|
|
3534
|
+
rd_kafka_buf_write_i32(rkbuf, rktp->rktp_fetch_msg_max_bytes);
|
|
3535
|
+
|
|
3536
|
+
/* Partition tags */
|
|
3537
|
+
rd_kafka_buf_write_tags_empty(rkbuf);
|
|
3538
|
+
|
|
3539
|
+
rd_rkb_dbg(rkb, FETCH, "FETCH",
|
|
3540
|
+
"Fetch topic %.*s [%" PRId32 "] at offset %" PRId64
|
|
3541
|
+
" (leader epoch %" PRId32
|
|
3542
|
+
", current leader epoch %" PRId32 ", v%d)",
|
|
3543
|
+
RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
|
|
3544
|
+
rktp->rktp_partition,
|
|
3545
|
+
rktp->rktp_offsets.fetch_pos.offset,
|
|
3546
|
+
rktp->rktp_offsets.fetch_pos.leader_epoch,
|
|
3547
|
+
rktp->rktp_leader_epoch, rktp->rktp_fetch_version);
|
|
3548
|
+
|
|
3549
|
+
/* We must have a valid fetch offset when we get here */
|
|
3550
|
+
rd_dassert(rktp->rktp_offsets.fetch_pos.offset >= 0);
|
|
3551
|
+
|
|
3552
|
+
/* Add toppar + op version mapping. */
|
|
3553
|
+
tver = rd_list_add(rkbuf->rkbuf_rktp_vers, NULL);
|
|
3554
|
+
tver->rktp = rd_kafka_toppar_keep(rktp);
|
|
3555
|
+
tver->version = rktp->rktp_fetch_version;
|
|
3556
|
+
|
|
3557
|
+
cnt++;
|
|
3558
|
+
} while ((rktp = CIRCLEQ_LOOP_NEXT(&rkb->rkb_active_toppars, rktp,
|
|
3559
|
+
rktp_activelink)) !=
|
|
3560
|
+
rkb->rkb_active_toppar_next);
|
|
3561
|
+
|
|
3562
|
+
/* Update next toppar to fetch in round-robin list. */
|
|
3563
|
+
rd_kafka_broker_active_toppar_next(
|
|
3564
|
+
rkb, rktp ? CIRCLEQ_LOOP_NEXT(&rkb->rkb_active_toppars, rktp,
|
|
3565
|
+
rktp_activelink)
|
|
3566
|
+
: NULL);
|
|
3567
|
+
|
|
3568
|
+
rd_rkb_dbg(rkb, FETCH, "FETCH", "Fetch %i/%i/%i toppar(s)", cnt,
|
|
3569
|
+
rkb->rkb_active_toppar_cnt, rkb->rkb_toppar_cnt);
|
|
3570
|
+
if (!cnt) {
|
|
3571
|
+
rd_kafka_buf_destroy(rkbuf);
|
|
3572
|
+
return cnt;
|
|
3573
|
+
}
|
|
3574
|
+
|
|
3575
|
+
if (rkt_last != NULL) {
|
|
3576
|
+
/* Update last topic's PartitionArrayCnt */
|
|
3577
|
+
rd_kafka_buf_finalize_arraycnt(rkbuf, of_PartitionArrayCnt,
|
|
3578
|
+
PartitionArrayCnt);
|
|
3579
|
+
/* Topic tags */
|
|
3580
|
+
rd_kafka_buf_write_tags_empty(rkbuf);
|
|
3581
|
+
}
|
|
3582
|
+
|
|
3583
|
+
/* Update TopicArrayCnt */
|
|
3584
|
+
rd_kafka_buf_finalize_arraycnt(rkbuf, of_TopicArrayCnt, TopicArrayCnt);
|
|
3585
|
+
|
|
3586
|
+
|
|
3587
|
+
if (rd_kafka_buf_ApiVersion(rkbuf) >= 7)
|
|
3588
|
+
/* Length of the ForgottenTopics list (KIP-227). Broker
|
|
3589
|
+
* use only - not used by the consumer. */
|
|
3590
|
+
rd_kafka_buf_write_arraycnt(rkbuf, 0);
|
|
3591
|
+
|
|
3592
|
+
if (rd_kafka_buf_ApiVersion(rkbuf) >= 11)
|
|
3593
|
+
/* RackId */
|
|
3594
|
+
rd_kafka_buf_write_kstr(rkbuf,
|
|
3595
|
+
rkb->rkb_rk->rk_conf.client_rack);
|
|
3596
|
+
|
|
3597
|
+
/* Consider Fetch requests blocking if fetch.wait.max.ms >= 1s */
|
|
3598
|
+
if (rkb->rkb_rk->rk_conf.fetch_wait_max_ms >= 1000)
|
|
3599
|
+
rkbuf->rkbuf_flags |= RD_KAFKA_OP_F_BLOCKING;
|
|
3600
|
+
|
|
3601
|
+
/* Use configured timeout */
|
|
3602
|
+
rd_kafka_buf_set_timeout(rkbuf,
|
|
3603
|
+
rkb->rkb_rk->rk_conf.socket_timeout_ms +
|
|
3604
|
+
rkb->rkb_rk->rk_conf.fetch_wait_max_ms,
|
|
3605
|
+
now);
|
|
3606
|
+
|
|
3607
|
+
/* Sort toppar versions for quicker lookups in Fetch response. */
|
|
3608
|
+
rd_list_sort(rkbuf->rkbuf_rktp_vers, rd_kafka_toppar_ver_cmp);
|
|
3609
|
+
|
|
3610
|
+
rkb->rkb_fetching = 1;
|
|
3611
|
+
rd_kafka_broker_buf_enq1(rkb, rkbuf, rd_kafka_broker_fetch_reply, NULL);
|
|
3612
|
+
|
|
3613
|
+
return cnt;
|
|
3614
|
+
}
|
|
3615
|
+
|
|
3616
|
+
/**
|
|
3617
|
+
* @brief Decide whether it should start fetching from next fetch start
|
|
3618
|
+
* or continue with current fetch pos.
|
|
3619
|
+
*
|
|
3620
|
+
* @param rktp the toppar
|
|
3621
|
+
*
|
|
3622
|
+
* @returns rd_true if it should start fetching from next fetch start,
|
|
3623
|
+
* rd_false otherwise.
|
|
3624
|
+
*
|
|
3625
|
+
* @locality any
|
|
3626
|
+
* @locks toppar_lock() MUST be held
|
|
3627
|
+
*/
|
|
3628
|
+
static rd_bool_t rd_kafka_toppar_fetch_decide_start_from_next_fetch_start(
|
|
3629
|
+
rd_kafka_toppar_t *rktp) {
|
|
3630
|
+
return rktp->rktp_op_version > rktp->rktp_fetch_version ||
|
|
3631
|
+
rd_kafka_fetch_pos_cmp(&rktp->rktp_next_fetch_start,
|
|
3632
|
+
&rktp->rktp_last_next_fetch_start) ||
|
|
3633
|
+
rktp->rktp_offsets.fetch_pos.offset == RD_KAFKA_OFFSET_INVALID;
|
|
3634
|
+
}
|
|
3635
|
+
|
|
3636
|
+
/**
|
|
3637
|
+
* @brief Return next fetch start position:
|
|
3638
|
+
* if it should start fetching from next fetch start
|
|
3639
|
+
* or continue with current fetch pos.
|
|
3640
|
+
*
|
|
3641
|
+
* @param rktp The toppar
|
|
3642
|
+
*
|
|
3643
|
+
* @returns Next fetch start position
|
|
3644
|
+
*
|
|
3645
|
+
* @locality any
|
|
3646
|
+
* @locks toppar_lock() MUST be held
|
|
3647
|
+
*/
|
|
3648
|
+
rd_kafka_fetch_pos_t
|
|
3649
|
+
rd_kafka_toppar_fetch_decide_next_fetch_start_pos(rd_kafka_toppar_t *rktp) {
|
|
3650
|
+
if (rd_kafka_toppar_fetch_decide_start_from_next_fetch_start(rktp))
|
|
3651
|
+
return rktp->rktp_next_fetch_start;
|
|
3652
|
+
else
|
|
3653
|
+
return rktp->rktp_offsets.fetch_pos;
|
|
3654
|
+
}
|
|
3655
|
+
|
|
3656
|
+
/**
|
|
3657
|
+
* @brief Decide whether this toppar should be on the fetch list or not.
|
|
3658
|
+
*
|
|
3659
|
+
* Also:
|
|
3660
|
+
* - update toppar's op version (for broker thread's copy)
|
|
3661
|
+
* - finalize statistics (move rktp_offsets to rktp_offsets_fin)
|
|
3662
|
+
*
|
|
3663
|
+
* @returns the partition's Fetch backoff timestamp, or 0 if no backoff.
|
|
3664
|
+
*
|
|
3665
|
+
* @locality broker thread
|
|
3666
|
+
* @locks none
|
|
3667
|
+
*/
|
|
3668
|
+
rd_ts_t rd_kafka_toppar_fetch_decide(rd_kafka_toppar_t *rktp,
|
|
3669
|
+
rd_kafka_broker_t *rkb,
|
|
3670
|
+
int force_remove) {
|
|
3671
|
+
int should_fetch = 1;
|
|
3672
|
+
const char *reason = "";
|
|
3673
|
+
int32_t version;
|
|
3674
|
+
rd_ts_t ts_backoff = 0;
|
|
3675
|
+
rd_bool_t lease_expired = rd_false;
|
|
3676
|
+
|
|
3677
|
+
rd_kafka_toppar_lock(rktp);
|
|
3678
|
+
|
|
3679
|
+
/* Check for preferred replica lease expiry */
|
|
3680
|
+
lease_expired = rktp->rktp_leader_id != rktp->rktp_broker_id &&
|
|
3681
|
+
rd_interval(&rktp->rktp_lease_intvl,
|
|
3682
|
+
5 * 60 * 1000 * 1000 /*5 minutes*/, 0) > 0;
|
|
3683
|
+
if (lease_expired) {
|
|
3684
|
+
/* delegate_to_leader() requires no locks to be held */
|
|
3685
|
+
rd_kafka_toppar_unlock(rktp);
|
|
3686
|
+
rd_kafka_toppar_delegate_to_leader(rktp);
|
|
3687
|
+
rd_kafka_toppar_lock(rktp);
|
|
3688
|
+
|
|
3689
|
+
reason = "preferred replica lease expired";
|
|
3690
|
+
should_fetch = 0;
|
|
3691
|
+
goto done;
|
|
3692
|
+
}
|
|
3693
|
+
|
|
3694
|
+
/* Forced removal from fetch list */
|
|
3695
|
+
if (unlikely(force_remove)) {
|
|
3696
|
+
reason = "forced removal";
|
|
3697
|
+
should_fetch = 0;
|
|
3698
|
+
goto done;
|
|
3699
|
+
}
|
|
3700
|
+
|
|
3701
|
+
if (unlikely((rktp->rktp_flags & RD_KAFKA_TOPPAR_F_REMOVE) != 0)) {
|
|
3702
|
+
reason = "partition removed";
|
|
3703
|
+
should_fetch = 0;
|
|
3704
|
+
goto done;
|
|
3705
|
+
}
|
|
3706
|
+
|
|
3707
|
+
/* Skip toppars not in active fetch state */
|
|
3708
|
+
if (rktp->rktp_fetch_state != RD_KAFKA_TOPPAR_FETCH_ACTIVE) {
|
|
3709
|
+
reason = "not in active fetch state";
|
|
3710
|
+
should_fetch = 0;
|
|
3711
|
+
goto done;
|
|
3712
|
+
}
|
|
3713
|
+
|
|
3714
|
+
/* Update broker thread's fetch op version */
|
|
3715
|
+
version = rktp->rktp_op_version;
|
|
3716
|
+
if (rd_kafka_toppar_fetch_decide_start_from_next_fetch_start(rktp)) {
|
|
3717
|
+
/* New version barrier, something was modified from the
|
|
3718
|
+
* control plane. Reset and start over.
|
|
3719
|
+
* Alternatively only the next_offset changed but not the
|
|
3720
|
+
* barrier, which is the case when automatically triggering
|
|
3721
|
+
* offset.reset (such as on PARTITION_EOF or
|
|
3722
|
+
* OFFSET_OUT_OF_RANGE). */
|
|
3723
|
+
|
|
3724
|
+
rd_kafka_dbg(
|
|
3725
|
+
rktp->rktp_rkt->rkt_rk, TOPIC, "FETCHDEC",
|
|
3726
|
+
"Topic %s [%" PRId32
|
|
3727
|
+
"]: fetch decide: "
|
|
3728
|
+
"updating to version %d (was %d) at %s "
|
|
3729
|
+
"(was %s)",
|
|
3730
|
+
rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition,
|
|
3731
|
+
version, rktp->rktp_fetch_version,
|
|
3732
|
+
rd_kafka_fetch_pos2str(rktp->rktp_next_fetch_start),
|
|
3733
|
+
rd_kafka_fetch_pos2str(rktp->rktp_offsets.fetch_pos));
|
|
3734
|
+
|
|
3735
|
+
rd_kafka_offset_stats_reset(&rktp->rktp_offsets);
|
|
3736
|
+
|
|
3737
|
+
/* New start offset */
|
|
3738
|
+
rktp->rktp_offsets.fetch_pos = rktp->rktp_next_fetch_start;
|
|
3739
|
+
rktp->rktp_last_next_fetch_start = rktp->rktp_next_fetch_start;
|
|
3740
|
+
|
|
3741
|
+
rktp->rktp_fetch_version = version;
|
|
3742
|
+
|
|
3743
|
+
/* Clear last error to propagate new fetch
|
|
3744
|
+
* errors if encountered. */
|
|
3745
|
+
rktp->rktp_last_error = RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
3746
|
+
|
|
3747
|
+
rd_kafka_q_purge_toppar_version(rktp->rktp_fetchq, rktp,
|
|
3748
|
+
version);
|
|
3749
|
+
}
|
|
3750
|
+
|
|
3751
|
+
|
|
3752
|
+
if (RD_KAFKA_TOPPAR_IS_PAUSED(rktp)) {
|
|
3753
|
+
should_fetch = 0;
|
|
3754
|
+
reason = "paused";
|
|
3755
|
+
|
|
3756
|
+
} else if (RD_KAFKA_OFFSET_IS_LOGICAL(
|
|
3757
|
+
rktp->rktp_next_fetch_start.offset)) {
|
|
3758
|
+
should_fetch = 0;
|
|
3759
|
+
reason = "no concrete offset";
|
|
3760
|
+
} else if (rktp->rktp_ts_fetch_backoff > rd_clock()) {
|
|
3761
|
+
reason = "fetch backed off";
|
|
3762
|
+
ts_backoff = rktp->rktp_ts_fetch_backoff;
|
|
3763
|
+
should_fetch = 0;
|
|
3764
|
+
} else if (rd_kafka_q_len(rktp->rktp_fetchq) >=
|
|
3765
|
+
rkb->rkb_rk->rk_conf.queued_min_msgs) {
|
|
3766
|
+
/* Skip toppars who's local message queue is already above
|
|
3767
|
+
* the lower threshold. */
|
|
3768
|
+
reason = "queued.min.messages exceeded";
|
|
3769
|
+
ts_backoff = rd_kafka_toppar_fetch_backoff(
|
|
3770
|
+
rkb, rktp, RD_KAFKA_RESP_ERR__QUEUE_FULL);
|
|
3771
|
+
should_fetch = 0;
|
|
3772
|
+
|
|
3773
|
+
} else if ((int64_t)rd_kafka_q_size(rktp->rktp_fetchq) >=
|
|
3774
|
+
rkb->rkb_rk->rk_conf.queued_max_msg_bytes) {
|
|
3775
|
+
reason = "queued.max.messages.kbytes exceeded";
|
|
3776
|
+
ts_backoff = rd_kafka_toppar_fetch_backoff(
|
|
3777
|
+
rkb, rktp, RD_KAFKA_RESP_ERR__QUEUE_FULL);
|
|
3778
|
+
should_fetch = 0;
|
|
3779
|
+
}
|
|
3780
|
+
|
|
3781
|
+
done:
|
|
3782
|
+
/* Copy offset stats to finalized place holder. */
|
|
3783
|
+
rktp->rktp_offsets_fin = rktp->rktp_offsets;
|
|
3784
|
+
|
|
3785
|
+
if (rktp->rktp_fetch != should_fetch) {
|
|
3786
|
+
rd_rkb_dbg(
|
|
3787
|
+
rkb, FETCH, "FETCH",
|
|
3788
|
+
"Topic %s [%" PRId32
|
|
3789
|
+
"] in state %s at %s "
|
|
3790
|
+
"(%d/%d msgs, %" PRId64
|
|
3791
|
+
"/%d kb queued, "
|
|
3792
|
+
"opv %" PRId32 ") is %s%s",
|
|
3793
|
+
rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition,
|
|
3794
|
+
rd_kafka_fetch_states[rktp->rktp_fetch_state],
|
|
3795
|
+
rd_kafka_fetch_pos2str(rktp->rktp_next_fetch_start),
|
|
3796
|
+
rd_kafka_q_len(rktp->rktp_fetchq),
|
|
3797
|
+
rkb->rkb_rk->rk_conf.queued_min_msgs,
|
|
3798
|
+
rd_kafka_q_size(rktp->rktp_fetchq) / 1024,
|
|
3799
|
+
rkb->rkb_rk->rk_conf.queued_max_msg_kbytes,
|
|
3800
|
+
rktp->rktp_fetch_version,
|
|
3801
|
+
should_fetch ? "fetchable" : "not fetchable: ", reason);
|
|
3802
|
+
|
|
3803
|
+
if (should_fetch) {
|
|
3804
|
+
rd_dassert(rktp->rktp_fetch_version > 0);
|
|
3805
|
+
rd_kafka_broker_active_toppar_add(
|
|
3806
|
+
rkb, rktp, *reason ? reason : "fetchable");
|
|
3807
|
+
} else {
|
|
3808
|
+
rd_kafka_broker_active_toppar_del(rkb, rktp, reason);
|
|
3809
|
+
}
|
|
3810
|
+
}
|
|
3811
|
+
|
|
3812
|
+
rd_kafka_toppar_unlock(rktp);
|
|
3813
|
+
|
|
3814
|
+
/* Non-fetching partitions will have an
|
|
3815
|
+
* indefinate backoff, unless explicitly specified. */
|
|
3816
|
+
if (!should_fetch && !ts_backoff)
|
|
3817
|
+
ts_backoff = RD_TS_MAX;
|
|
3818
|
+
|
|
3819
|
+
return ts_backoff;
|
|
3820
|
+
}
|