@point3/node-rdkafka 3.6.0-1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.txt +20 -0
- package/README.md +636 -0
- package/binding.gyp +154 -0
- package/deps/librdkafka/.clang-format +136 -0
- package/deps/librdkafka/.clang-format-cpp +103 -0
- package/deps/librdkafka/.dir-locals.el +10 -0
- package/deps/librdkafka/.formatignore +33 -0
- package/deps/librdkafka/.gdbmacros +19 -0
- package/deps/librdkafka/.github/CODEOWNERS +1 -0
- package/deps/librdkafka/.github/ISSUE_TEMPLATE +34 -0
- package/deps/librdkafka/.semaphore/run-all-tests.yml +77 -0
- package/deps/librdkafka/.semaphore/semaphore-integration.yml +250 -0
- package/deps/librdkafka/.semaphore/semaphore.yml +378 -0
- package/deps/librdkafka/.semaphore/verify-linux-packages.yml +41 -0
- package/deps/librdkafka/CHANGELOG.md +2208 -0
- package/deps/librdkafka/CMakeLists.txt +291 -0
- package/deps/librdkafka/CODE_OF_CONDUCT.md +46 -0
- package/deps/librdkafka/CONFIGURATION.md +209 -0
- package/deps/librdkafka/CONTRIBUTING.md +431 -0
- package/deps/librdkafka/Doxyfile +2375 -0
- package/deps/librdkafka/INTRODUCTION.md +2481 -0
- package/deps/librdkafka/LICENSE +26 -0
- package/deps/librdkafka/LICENSE.cjson +22 -0
- package/deps/librdkafka/LICENSE.crc32c +28 -0
- package/deps/librdkafka/LICENSE.fnv1a +18 -0
- package/deps/librdkafka/LICENSE.hdrhistogram +27 -0
- package/deps/librdkafka/LICENSE.lz4 +26 -0
- package/deps/librdkafka/LICENSE.murmur2 +25 -0
- package/deps/librdkafka/LICENSE.nanopb +22 -0
- package/deps/librdkafka/LICENSE.opentelemetry +203 -0
- package/deps/librdkafka/LICENSE.pycrc +23 -0
- package/deps/librdkafka/LICENSE.queue +31 -0
- package/deps/librdkafka/LICENSE.regexp +5 -0
- package/deps/librdkafka/LICENSE.snappy +36 -0
- package/deps/librdkafka/LICENSE.tinycthread +26 -0
- package/deps/librdkafka/LICENSE.wingetopt +49 -0
- package/deps/librdkafka/LICENSES.txt +625 -0
- package/deps/librdkafka/Makefile +125 -0
- package/deps/librdkafka/README.md +199 -0
- package/deps/librdkafka/README.win32 +26 -0
- package/deps/librdkafka/STATISTICS.md +624 -0
- package/deps/librdkafka/configure +214 -0
- package/deps/librdkafka/configure.self +331 -0
- package/deps/librdkafka/debian/changelog +111 -0
- package/deps/librdkafka/debian/compat +1 -0
- package/deps/librdkafka/debian/control +71 -0
- package/deps/librdkafka/debian/copyright +99 -0
- package/deps/librdkafka/debian/gbp.conf +9 -0
- package/deps/librdkafka/debian/librdkafka++1.install +1 -0
- package/deps/librdkafka/debian/librdkafka-dev.examples +2 -0
- package/deps/librdkafka/debian/librdkafka-dev.install +9 -0
- package/deps/librdkafka/debian/librdkafka1.docs +5 -0
- package/deps/librdkafka/debian/librdkafka1.install +1 -0
- package/deps/librdkafka/debian/librdkafka1.symbols +135 -0
- package/deps/librdkafka/debian/rules +19 -0
- package/deps/librdkafka/debian/source/format +1 -0
- package/deps/librdkafka/debian/watch +2 -0
- package/deps/librdkafka/dev-conf.sh +123 -0
- package/deps/librdkafka/examples/CMakeLists.txt +79 -0
- package/deps/librdkafka/examples/Makefile +167 -0
- package/deps/librdkafka/examples/README.md +42 -0
- package/deps/librdkafka/examples/alter_consumer_group_offsets.c +338 -0
- package/deps/librdkafka/examples/consumer.c +271 -0
- package/deps/librdkafka/examples/delete_records.c +233 -0
- package/deps/librdkafka/examples/describe_cluster.c +322 -0
- package/deps/librdkafka/examples/describe_consumer_groups.c +455 -0
- package/deps/librdkafka/examples/describe_topics.c +427 -0
- package/deps/librdkafka/examples/elect_leaders.c +317 -0
- package/deps/librdkafka/examples/globals.json +11 -0
- package/deps/librdkafka/examples/idempotent_producer.c +344 -0
- package/deps/librdkafka/examples/incremental_alter_configs.c +347 -0
- package/deps/librdkafka/examples/kafkatest_verifiable_client.cpp +945 -0
- package/deps/librdkafka/examples/list_consumer_group_offsets.c +359 -0
- package/deps/librdkafka/examples/list_consumer_groups.c +365 -0
- package/deps/librdkafka/examples/list_offsets.c +327 -0
- package/deps/librdkafka/examples/misc.c +287 -0
- package/deps/librdkafka/examples/openssl_engine_example.cpp +248 -0
- package/deps/librdkafka/examples/producer.c +251 -0
- package/deps/librdkafka/examples/producer.cpp +228 -0
- package/deps/librdkafka/examples/rdkafka_complex_consumer_example.c +617 -0
- package/deps/librdkafka/examples/rdkafka_complex_consumer_example.cpp +467 -0
- package/deps/librdkafka/examples/rdkafka_consume_batch.cpp +264 -0
- package/deps/librdkafka/examples/rdkafka_example.c +853 -0
- package/deps/librdkafka/examples/rdkafka_example.cpp +679 -0
- package/deps/librdkafka/examples/rdkafka_performance.c +1781 -0
- package/deps/librdkafka/examples/transactions-older-broker.c +668 -0
- package/deps/librdkafka/examples/transactions.c +665 -0
- package/deps/librdkafka/examples/user_scram.c +491 -0
- package/deps/librdkafka/examples/win_ssl_cert_store.cpp +396 -0
- package/deps/librdkafka/lds-gen.py +73 -0
- package/deps/librdkafka/mainpage.doxy +40 -0
- package/deps/librdkafka/mklove/Makefile.base +329 -0
- package/deps/librdkafka/mklove/modules/configure.atomics +144 -0
- package/deps/librdkafka/mklove/modules/configure.base +2484 -0
- package/deps/librdkafka/mklove/modules/configure.builtin +70 -0
- package/deps/librdkafka/mklove/modules/configure.cc +186 -0
- package/deps/librdkafka/mklove/modules/configure.cxx +8 -0
- package/deps/librdkafka/mklove/modules/configure.fileversion +65 -0
- package/deps/librdkafka/mklove/modules/configure.gitversion +29 -0
- package/deps/librdkafka/mklove/modules/configure.good_cflags +18 -0
- package/deps/librdkafka/mklove/modules/configure.host +132 -0
- package/deps/librdkafka/mklove/modules/configure.lib +49 -0
- package/deps/librdkafka/mklove/modules/configure.libcurl +99 -0
- package/deps/librdkafka/mklove/modules/configure.libsasl2 +36 -0
- package/deps/librdkafka/mklove/modules/configure.libssl +147 -0
- package/deps/librdkafka/mklove/modules/configure.libzstd +58 -0
- package/deps/librdkafka/mklove/modules/configure.parseversion +95 -0
- package/deps/librdkafka/mklove/modules/configure.pic +16 -0
- package/deps/librdkafka/mklove/modules/configure.socket +20 -0
- package/deps/librdkafka/mklove/modules/configure.zlib +61 -0
- package/deps/librdkafka/mklove/modules/patches/README.md +8 -0
- package/deps/librdkafka/mklove/modules/patches/libcurl.0000-no-runtime-linking-check.patch +11 -0
- package/deps/librdkafka/mklove/modules/patches/libssl.0000-osx-rand-include-fix-OpenSSL-PR16409.patch +56 -0
- package/deps/librdkafka/packaging/RELEASE.md +319 -0
- package/deps/librdkafka/packaging/alpine/build-alpine.sh +38 -0
- package/deps/librdkafka/packaging/archlinux/PKGBUILD +30 -0
- package/deps/librdkafka/packaging/cmake/Config.cmake.in +37 -0
- package/deps/librdkafka/packaging/cmake/Modules/FindLZ4.cmake +38 -0
- package/deps/librdkafka/packaging/cmake/Modules/FindZSTD.cmake +27 -0
- package/deps/librdkafka/packaging/cmake/Modules/LICENSE.FindZstd +178 -0
- package/deps/librdkafka/packaging/cmake/README.md +38 -0
- package/deps/librdkafka/packaging/cmake/config.h.in +52 -0
- package/deps/librdkafka/packaging/cmake/parseversion.cmake +60 -0
- package/deps/librdkafka/packaging/cmake/rdkafka.pc.in +12 -0
- package/deps/librdkafka/packaging/cmake/try_compile/atomic_32_test.c +8 -0
- package/deps/librdkafka/packaging/cmake/try_compile/atomic_64_test.c +8 -0
- package/deps/librdkafka/packaging/cmake/try_compile/c11threads_test.c +14 -0
- package/deps/librdkafka/packaging/cmake/try_compile/crc32c_hw_test.c +27 -0
- package/deps/librdkafka/packaging/cmake/try_compile/dlopen_test.c +11 -0
- package/deps/librdkafka/packaging/cmake/try_compile/libsasl2_test.c +7 -0
- package/deps/librdkafka/packaging/cmake/try_compile/pthread_setname_darwin_test.c +6 -0
- package/deps/librdkafka/packaging/cmake/try_compile/pthread_setname_freebsd_test.c +7 -0
- package/deps/librdkafka/packaging/cmake/try_compile/pthread_setname_gnu_test.c +5 -0
- package/deps/librdkafka/packaging/cmake/try_compile/rand_r_test.c +7 -0
- package/deps/librdkafka/packaging/cmake/try_compile/rdkafka_setup.cmake +122 -0
- package/deps/librdkafka/packaging/cmake/try_compile/regex_test.c +10 -0
- package/deps/librdkafka/packaging/cmake/try_compile/strndup_test.c +5 -0
- package/deps/librdkafka/packaging/cmake/try_compile/sync_32_test.c +8 -0
- package/deps/librdkafka/packaging/cmake/try_compile/sync_64_test.c +8 -0
- package/deps/librdkafka/packaging/cp/README.md +16 -0
- package/deps/librdkafka/packaging/cp/check_features.c +72 -0
- package/deps/librdkafka/packaging/cp/verify-deb.sh +33 -0
- package/deps/librdkafka/packaging/cp/verify-packages.sh +69 -0
- package/deps/librdkafka/packaging/cp/verify-rpm.sh +32 -0
- package/deps/librdkafka/packaging/debian/changelog +66 -0
- package/deps/librdkafka/packaging/debian/compat +1 -0
- package/deps/librdkafka/packaging/debian/control +49 -0
- package/deps/librdkafka/packaging/debian/copyright +84 -0
- package/deps/librdkafka/packaging/debian/docs +5 -0
- package/deps/librdkafka/packaging/debian/gbp.conf +9 -0
- package/deps/librdkafka/packaging/debian/librdkafka-dev.dirs +2 -0
- package/deps/librdkafka/packaging/debian/librdkafka-dev.examples +2 -0
- package/deps/librdkafka/packaging/debian/librdkafka-dev.install +6 -0
- package/deps/librdkafka/packaging/debian/librdkafka-dev.substvars +1 -0
- package/deps/librdkafka/packaging/debian/librdkafka.dsc +16 -0
- package/deps/librdkafka/packaging/debian/librdkafka1-dbg.substvars +1 -0
- package/deps/librdkafka/packaging/debian/librdkafka1.dirs +1 -0
- package/deps/librdkafka/packaging/debian/librdkafka1.install +2 -0
- package/deps/librdkafka/packaging/debian/librdkafka1.postinst.debhelper +5 -0
- package/deps/librdkafka/packaging/debian/librdkafka1.postrm.debhelper +5 -0
- package/deps/librdkafka/packaging/debian/librdkafka1.symbols +64 -0
- package/deps/librdkafka/packaging/debian/rules +19 -0
- package/deps/librdkafka/packaging/debian/source/format +1 -0
- package/deps/librdkafka/packaging/debian/watch +2 -0
- package/deps/librdkafka/packaging/get_version.py +21 -0
- package/deps/librdkafka/packaging/homebrew/README.md +15 -0
- package/deps/librdkafka/packaging/homebrew/brew-update-pr.sh +31 -0
- package/deps/librdkafka/packaging/mingw-w64/configure-build-msys2-mingw-static.sh +52 -0
- package/deps/librdkafka/packaging/mingw-w64/configure-build-msys2-mingw.sh +21 -0
- package/deps/librdkafka/packaging/mingw-w64/export-variables.sh +13 -0
- package/deps/librdkafka/packaging/mingw-w64/run-tests.sh +6 -0
- package/deps/librdkafka/packaging/mingw-w64/semaphoreci-build.sh +38 -0
- package/deps/librdkafka/packaging/nuget/README.md +84 -0
- package/deps/librdkafka/packaging/nuget/artifact.py +177 -0
- package/deps/librdkafka/packaging/nuget/cleanup-s3.py +143 -0
- package/deps/librdkafka/packaging/nuget/common/p-common__plat-windows__arch-win32__bldtype-Release/msvcr120.zip +0 -0
- package/deps/librdkafka/packaging/nuget/common/p-common__plat-windows__arch-win32__bldtype-Release/msvcr140.zip +0 -0
- package/deps/librdkafka/packaging/nuget/common/p-common__plat-windows__arch-x64__bldtype-Release/msvcr120.zip +0 -0
- package/deps/librdkafka/packaging/nuget/common/p-common__plat-windows__arch-x64__bldtype-Release/msvcr140.zip +0 -0
- package/deps/librdkafka/packaging/nuget/nuget.sh +21 -0
- package/deps/librdkafka/packaging/nuget/nugetpackage.py +278 -0
- package/deps/librdkafka/packaging/nuget/packaging.py +448 -0
- package/deps/librdkafka/packaging/nuget/push-to-nuget.sh +21 -0
- package/deps/librdkafka/packaging/nuget/release.py +167 -0
- package/deps/librdkafka/packaging/nuget/requirements.txt +3 -0
- package/deps/librdkafka/packaging/nuget/staticpackage.py +178 -0
- package/deps/librdkafka/packaging/nuget/templates/librdkafka.redist.nuspec +21 -0
- package/deps/librdkafka/packaging/nuget/templates/librdkafka.redist.props +18 -0
- package/deps/librdkafka/packaging/nuget/templates/librdkafka.redist.targets +19 -0
- package/deps/librdkafka/packaging/nuget/zfile/__init__.py +0 -0
- package/deps/librdkafka/packaging/nuget/zfile/zfile.py +98 -0
- package/deps/librdkafka/packaging/rpm/Makefile +92 -0
- package/deps/librdkafka/packaging/rpm/README.md +23 -0
- package/deps/librdkafka/packaging/rpm/el7-x86_64.cfg +40 -0
- package/deps/librdkafka/packaging/rpm/librdkafka.spec +118 -0
- package/deps/librdkafka/packaging/rpm/mock-on-docker.sh +96 -0
- package/deps/librdkafka/packaging/rpm/tests/Makefile +25 -0
- package/deps/librdkafka/packaging/rpm/tests/README.md +8 -0
- package/deps/librdkafka/packaging/rpm/tests/run-test.sh +42 -0
- package/deps/librdkafka/packaging/rpm/tests/test-on-docker.sh +56 -0
- package/deps/librdkafka/packaging/rpm/tests/test.c +77 -0
- package/deps/librdkafka/packaging/rpm/tests/test.cpp +34 -0
- package/deps/librdkafka/packaging/tools/Dockerfile +31 -0
- package/deps/librdkafka/packaging/tools/build-configurations-checks.sh +12 -0
- package/deps/librdkafka/packaging/tools/build-deb-package.sh +64 -0
- package/deps/librdkafka/packaging/tools/build-debian.sh +65 -0
- package/deps/librdkafka/packaging/tools/build-manylinux.sh +68 -0
- package/deps/librdkafka/packaging/tools/build-release-artifacts.sh +139 -0
- package/deps/librdkafka/packaging/tools/distro-build.sh +38 -0
- package/deps/librdkafka/packaging/tools/gh-release-checksums.py +39 -0
- package/deps/librdkafka/packaging/tools/rdutcoverage.sh +25 -0
- package/deps/librdkafka/packaging/tools/requirements.txt +2 -0
- package/deps/librdkafka/packaging/tools/run-in-docker.sh +28 -0
- package/deps/librdkafka/packaging/tools/run-integration-tests.sh +31 -0
- package/deps/librdkafka/packaging/tools/run-style-check.sh +4 -0
- package/deps/librdkafka/packaging/tools/style-format.sh +149 -0
- package/deps/librdkafka/packaging/tools/update_rpcs_max_versions.py +100 -0
- package/deps/librdkafka/service.yml +172 -0
- package/deps/librdkafka/src/CMakeLists.txt +374 -0
- package/deps/librdkafka/src/Makefile +103 -0
- package/deps/librdkafka/src/README.lz4.md +30 -0
- package/deps/librdkafka/src/cJSON.c +2834 -0
- package/deps/librdkafka/src/cJSON.h +398 -0
- package/deps/librdkafka/src/crc32c.c +430 -0
- package/deps/librdkafka/src/crc32c.h +38 -0
- package/deps/librdkafka/src/generate_proto.sh +66 -0
- package/deps/librdkafka/src/librdkafka_cgrp_synch.png +0 -0
- package/deps/librdkafka/src/lz4.c +2727 -0
- package/deps/librdkafka/src/lz4.h +842 -0
- package/deps/librdkafka/src/lz4frame.c +2078 -0
- package/deps/librdkafka/src/lz4frame.h +692 -0
- package/deps/librdkafka/src/lz4frame_static.h +47 -0
- package/deps/librdkafka/src/lz4hc.c +1631 -0
- package/deps/librdkafka/src/lz4hc.h +413 -0
- package/deps/librdkafka/src/nanopb/pb.h +917 -0
- package/deps/librdkafka/src/nanopb/pb_common.c +388 -0
- package/deps/librdkafka/src/nanopb/pb_common.h +49 -0
- package/deps/librdkafka/src/nanopb/pb_decode.c +1727 -0
- package/deps/librdkafka/src/nanopb/pb_decode.h +193 -0
- package/deps/librdkafka/src/nanopb/pb_encode.c +1000 -0
- package/deps/librdkafka/src/nanopb/pb_encode.h +185 -0
- package/deps/librdkafka/src/opentelemetry/common.pb.c +32 -0
- package/deps/librdkafka/src/opentelemetry/common.pb.h +170 -0
- package/deps/librdkafka/src/opentelemetry/metrics.options +2 -0
- package/deps/librdkafka/src/opentelemetry/metrics.pb.c +67 -0
- package/deps/librdkafka/src/opentelemetry/metrics.pb.h +966 -0
- package/deps/librdkafka/src/opentelemetry/resource.pb.c +12 -0
- package/deps/librdkafka/src/opentelemetry/resource.pb.h +58 -0
- package/deps/librdkafka/src/queue.h +850 -0
- package/deps/librdkafka/src/rd.h +584 -0
- package/deps/librdkafka/src/rdaddr.c +255 -0
- package/deps/librdkafka/src/rdaddr.h +202 -0
- package/deps/librdkafka/src/rdatomic.h +230 -0
- package/deps/librdkafka/src/rdavg.h +260 -0
- package/deps/librdkafka/src/rdavl.c +210 -0
- package/deps/librdkafka/src/rdavl.h +250 -0
- package/deps/librdkafka/src/rdbase64.c +200 -0
- package/deps/librdkafka/src/rdbase64.h +43 -0
- package/deps/librdkafka/src/rdbuf.c +1884 -0
- package/deps/librdkafka/src/rdbuf.h +375 -0
- package/deps/librdkafka/src/rdcrc32.c +114 -0
- package/deps/librdkafka/src/rdcrc32.h +170 -0
- package/deps/librdkafka/src/rddl.c +179 -0
- package/deps/librdkafka/src/rddl.h +43 -0
- package/deps/librdkafka/src/rdendian.h +175 -0
- package/deps/librdkafka/src/rdfloat.h +67 -0
- package/deps/librdkafka/src/rdfnv1a.c +113 -0
- package/deps/librdkafka/src/rdfnv1a.h +35 -0
- package/deps/librdkafka/src/rdgz.c +120 -0
- package/deps/librdkafka/src/rdgz.h +46 -0
- package/deps/librdkafka/src/rdhdrhistogram.c +721 -0
- package/deps/librdkafka/src/rdhdrhistogram.h +87 -0
- package/deps/librdkafka/src/rdhttp.c +830 -0
- package/deps/librdkafka/src/rdhttp.h +101 -0
- package/deps/librdkafka/src/rdinterval.h +177 -0
- package/deps/librdkafka/src/rdkafka.c +5505 -0
- package/deps/librdkafka/src/rdkafka.h +10686 -0
- package/deps/librdkafka/src/rdkafka_admin.c +9794 -0
- package/deps/librdkafka/src/rdkafka_admin.h +661 -0
- package/deps/librdkafka/src/rdkafka_assignment.c +1010 -0
- package/deps/librdkafka/src/rdkafka_assignment.h +73 -0
- package/deps/librdkafka/src/rdkafka_assignor.c +1786 -0
- package/deps/librdkafka/src/rdkafka_assignor.h +402 -0
- package/deps/librdkafka/src/rdkafka_aux.c +409 -0
- package/deps/librdkafka/src/rdkafka_aux.h +174 -0
- package/deps/librdkafka/src/rdkafka_background.c +221 -0
- package/deps/librdkafka/src/rdkafka_broker.c +6337 -0
- package/deps/librdkafka/src/rdkafka_broker.h +744 -0
- package/deps/librdkafka/src/rdkafka_buf.c +543 -0
- package/deps/librdkafka/src/rdkafka_buf.h +1525 -0
- package/deps/librdkafka/src/rdkafka_cert.c +576 -0
- package/deps/librdkafka/src/rdkafka_cert.h +62 -0
- package/deps/librdkafka/src/rdkafka_cgrp.c +7587 -0
- package/deps/librdkafka/src/rdkafka_cgrp.h +477 -0
- package/deps/librdkafka/src/rdkafka_conf.c +4880 -0
- package/deps/librdkafka/src/rdkafka_conf.h +732 -0
- package/deps/librdkafka/src/rdkafka_confval.h +97 -0
- package/deps/librdkafka/src/rdkafka_coord.c +623 -0
- package/deps/librdkafka/src/rdkafka_coord.h +132 -0
- package/deps/librdkafka/src/rdkafka_error.c +228 -0
- package/deps/librdkafka/src/rdkafka_error.h +80 -0
- package/deps/librdkafka/src/rdkafka_event.c +502 -0
- package/deps/librdkafka/src/rdkafka_event.h +126 -0
- package/deps/librdkafka/src/rdkafka_feature.c +898 -0
- package/deps/librdkafka/src/rdkafka_feature.h +104 -0
- package/deps/librdkafka/src/rdkafka_fetcher.c +1422 -0
- package/deps/librdkafka/src/rdkafka_fetcher.h +44 -0
- package/deps/librdkafka/src/rdkafka_header.c +220 -0
- package/deps/librdkafka/src/rdkafka_header.h +76 -0
- package/deps/librdkafka/src/rdkafka_idempotence.c +807 -0
- package/deps/librdkafka/src/rdkafka_idempotence.h +144 -0
- package/deps/librdkafka/src/rdkafka_int.h +1260 -0
- package/deps/librdkafka/src/rdkafka_interceptor.c +819 -0
- package/deps/librdkafka/src/rdkafka_interceptor.h +104 -0
- package/deps/librdkafka/src/rdkafka_lz4.c +450 -0
- package/deps/librdkafka/src/rdkafka_lz4.h +49 -0
- package/deps/librdkafka/src/rdkafka_metadata.c +2209 -0
- package/deps/librdkafka/src/rdkafka_metadata.h +345 -0
- package/deps/librdkafka/src/rdkafka_metadata_cache.c +1183 -0
- package/deps/librdkafka/src/rdkafka_mock.c +3661 -0
- package/deps/librdkafka/src/rdkafka_mock.h +610 -0
- package/deps/librdkafka/src/rdkafka_mock_cgrp.c +1876 -0
- package/deps/librdkafka/src/rdkafka_mock_handlers.c +3113 -0
- package/deps/librdkafka/src/rdkafka_mock_int.h +710 -0
- package/deps/librdkafka/src/rdkafka_msg.c +2589 -0
- package/deps/librdkafka/src/rdkafka_msg.h +614 -0
- package/deps/librdkafka/src/rdkafka_msgbatch.h +62 -0
- package/deps/librdkafka/src/rdkafka_msgset.h +98 -0
- package/deps/librdkafka/src/rdkafka_msgset_reader.c +1806 -0
- package/deps/librdkafka/src/rdkafka_msgset_writer.c +1474 -0
- package/deps/librdkafka/src/rdkafka_offset.c +1565 -0
- package/deps/librdkafka/src/rdkafka_offset.h +150 -0
- package/deps/librdkafka/src/rdkafka_op.c +997 -0
- package/deps/librdkafka/src/rdkafka_op.h +858 -0
- package/deps/librdkafka/src/rdkafka_partition.c +4896 -0
- package/deps/librdkafka/src/rdkafka_partition.h +1182 -0
- package/deps/librdkafka/src/rdkafka_pattern.c +228 -0
- package/deps/librdkafka/src/rdkafka_pattern.h +70 -0
- package/deps/librdkafka/src/rdkafka_plugin.c +213 -0
- package/deps/librdkafka/src/rdkafka_plugin.h +41 -0
- package/deps/librdkafka/src/rdkafka_proto.h +736 -0
- package/deps/librdkafka/src/rdkafka_protocol.h +128 -0
- package/deps/librdkafka/src/rdkafka_queue.c +1230 -0
- package/deps/librdkafka/src/rdkafka_queue.h +1220 -0
- package/deps/librdkafka/src/rdkafka_range_assignor.c +1748 -0
- package/deps/librdkafka/src/rdkafka_request.c +7089 -0
- package/deps/librdkafka/src/rdkafka_request.h +732 -0
- package/deps/librdkafka/src/rdkafka_roundrobin_assignor.c +123 -0
- package/deps/librdkafka/src/rdkafka_sasl.c +530 -0
- package/deps/librdkafka/src/rdkafka_sasl.h +63 -0
- package/deps/librdkafka/src/rdkafka_sasl_cyrus.c +722 -0
- package/deps/librdkafka/src/rdkafka_sasl_int.h +89 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer.c +1833 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer.h +52 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer_oidc.c +1666 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer_oidc.h +47 -0
- package/deps/librdkafka/src/rdkafka_sasl_plain.c +142 -0
- package/deps/librdkafka/src/rdkafka_sasl_scram.c +858 -0
- package/deps/librdkafka/src/rdkafka_sasl_win32.c +550 -0
- package/deps/librdkafka/src/rdkafka_ssl.c +2129 -0
- package/deps/librdkafka/src/rdkafka_ssl.h +86 -0
- package/deps/librdkafka/src/rdkafka_sticky_assignor.c +4785 -0
- package/deps/librdkafka/src/rdkafka_subscription.c +278 -0
- package/deps/librdkafka/src/rdkafka_telemetry.c +760 -0
- package/deps/librdkafka/src/rdkafka_telemetry.h +52 -0
- package/deps/librdkafka/src/rdkafka_telemetry_decode.c +1053 -0
- package/deps/librdkafka/src/rdkafka_telemetry_decode.h +59 -0
- package/deps/librdkafka/src/rdkafka_telemetry_encode.c +997 -0
- package/deps/librdkafka/src/rdkafka_telemetry_encode.h +301 -0
- package/deps/librdkafka/src/rdkafka_timer.c +402 -0
- package/deps/librdkafka/src/rdkafka_timer.h +117 -0
- package/deps/librdkafka/src/rdkafka_topic.c +2161 -0
- package/deps/librdkafka/src/rdkafka_topic.h +334 -0
- package/deps/librdkafka/src/rdkafka_transport.c +1309 -0
- package/deps/librdkafka/src/rdkafka_transport.h +99 -0
- package/deps/librdkafka/src/rdkafka_transport_int.h +100 -0
- package/deps/librdkafka/src/rdkafka_txnmgr.c +3256 -0
- package/deps/librdkafka/src/rdkafka_txnmgr.h +171 -0
- package/deps/librdkafka/src/rdkafka_zstd.c +226 -0
- package/deps/librdkafka/src/rdkafka_zstd.h +57 -0
- package/deps/librdkafka/src/rdlist.c +576 -0
- package/deps/librdkafka/src/rdlist.h +434 -0
- package/deps/librdkafka/src/rdlog.c +89 -0
- package/deps/librdkafka/src/rdlog.h +41 -0
- package/deps/librdkafka/src/rdmap.c +508 -0
- package/deps/librdkafka/src/rdmap.h +492 -0
- package/deps/librdkafka/src/rdmurmur2.c +167 -0
- package/deps/librdkafka/src/rdmurmur2.h +35 -0
- package/deps/librdkafka/src/rdports.c +61 -0
- package/deps/librdkafka/src/rdports.h +38 -0
- package/deps/librdkafka/src/rdposix.h +250 -0
- package/deps/librdkafka/src/rdrand.c +80 -0
- package/deps/librdkafka/src/rdrand.h +43 -0
- package/deps/librdkafka/src/rdregex.c +156 -0
- package/deps/librdkafka/src/rdregex.h +43 -0
- package/deps/librdkafka/src/rdsignal.h +57 -0
- package/deps/librdkafka/src/rdstring.c +645 -0
- package/deps/librdkafka/src/rdstring.h +98 -0
- package/deps/librdkafka/src/rdsysqueue.h +404 -0
- package/deps/librdkafka/src/rdtime.h +356 -0
- package/deps/librdkafka/src/rdtypes.h +86 -0
- package/deps/librdkafka/src/rdunittest.c +549 -0
- package/deps/librdkafka/src/rdunittest.h +232 -0
- package/deps/librdkafka/src/rdvarint.c +134 -0
- package/deps/librdkafka/src/rdvarint.h +165 -0
- package/deps/librdkafka/src/rdwin32.h +382 -0
- package/deps/librdkafka/src/rdxxhash.c +1030 -0
- package/deps/librdkafka/src/rdxxhash.h +328 -0
- package/deps/librdkafka/src/regexp.c +1352 -0
- package/deps/librdkafka/src/regexp.h +41 -0
- package/deps/librdkafka/src/snappy.c +1866 -0
- package/deps/librdkafka/src/snappy.h +62 -0
- package/deps/librdkafka/src/snappy_compat.h +138 -0
- package/deps/librdkafka/src/statistics_schema.json +444 -0
- package/deps/librdkafka/src/tinycthread.c +932 -0
- package/deps/librdkafka/src/tinycthread.h +503 -0
- package/deps/librdkafka/src/tinycthread_extra.c +199 -0
- package/deps/librdkafka/src/tinycthread_extra.h +212 -0
- package/deps/librdkafka/src/win32_config.h +58 -0
- package/deps/librdkafka/src-cpp/CMakeLists.txt +90 -0
- package/deps/librdkafka/src-cpp/ConfImpl.cpp +84 -0
- package/deps/librdkafka/src-cpp/ConsumerImpl.cpp +244 -0
- package/deps/librdkafka/src-cpp/HandleImpl.cpp +436 -0
- package/deps/librdkafka/src-cpp/HeadersImpl.cpp +48 -0
- package/deps/librdkafka/src-cpp/KafkaConsumerImpl.cpp +296 -0
- package/deps/librdkafka/src-cpp/Makefile +55 -0
- package/deps/librdkafka/src-cpp/MessageImpl.cpp +38 -0
- package/deps/librdkafka/src-cpp/MetadataImpl.cpp +170 -0
- package/deps/librdkafka/src-cpp/ProducerImpl.cpp +197 -0
- package/deps/librdkafka/src-cpp/QueueImpl.cpp +70 -0
- package/deps/librdkafka/src-cpp/README.md +16 -0
- package/deps/librdkafka/src-cpp/RdKafka.cpp +59 -0
- package/deps/librdkafka/src-cpp/TopicImpl.cpp +124 -0
- package/deps/librdkafka/src-cpp/TopicPartitionImpl.cpp +57 -0
- package/deps/librdkafka/src-cpp/rdkafkacpp.h +3797 -0
- package/deps/librdkafka/src-cpp/rdkafkacpp_int.h +1641 -0
- package/deps/librdkafka/tests/0000-unittests.c +72 -0
- package/deps/librdkafka/tests/0001-multiobj.c +102 -0
- package/deps/librdkafka/tests/0002-unkpart.c +244 -0
- package/deps/librdkafka/tests/0003-msgmaxsize.c +173 -0
- package/deps/librdkafka/tests/0004-conf.c +934 -0
- package/deps/librdkafka/tests/0005-order.c +133 -0
- package/deps/librdkafka/tests/0006-symbols.c +163 -0
- package/deps/librdkafka/tests/0007-autotopic.c +136 -0
- package/deps/librdkafka/tests/0008-reqacks.c +179 -0
- package/deps/librdkafka/tests/0009-mock_cluster.c +97 -0
- package/deps/librdkafka/tests/0011-produce_batch.c +753 -0
- package/deps/librdkafka/tests/0012-produce_consume.c +537 -0
- package/deps/librdkafka/tests/0013-null-msgs.c +473 -0
- package/deps/librdkafka/tests/0014-reconsume-191.c +512 -0
- package/deps/librdkafka/tests/0015-offset_seeks.c +172 -0
- package/deps/librdkafka/tests/0016-client_swname.c +181 -0
- package/deps/librdkafka/tests/0017-compression.c +140 -0
- package/deps/librdkafka/tests/0018-cgrp_term.c +338 -0
- package/deps/librdkafka/tests/0019-list_groups.c +289 -0
- package/deps/librdkafka/tests/0020-destroy_hang.c +162 -0
- package/deps/librdkafka/tests/0021-rkt_destroy.c +72 -0
- package/deps/librdkafka/tests/0022-consume_batch.c +279 -0
- package/deps/librdkafka/tests/0025-timers.c +147 -0
- package/deps/librdkafka/tests/0026-consume_pause.c +547 -0
- package/deps/librdkafka/tests/0028-long_topicnames.c +79 -0
- package/deps/librdkafka/tests/0029-assign_offset.c +202 -0
- package/deps/librdkafka/tests/0030-offset_commit.c +589 -0
- package/deps/librdkafka/tests/0031-get_offsets.c +235 -0
- package/deps/librdkafka/tests/0033-regex_subscribe.c +536 -0
- package/deps/librdkafka/tests/0034-offset_reset.c +398 -0
- package/deps/librdkafka/tests/0035-api_version.c +73 -0
- package/deps/librdkafka/tests/0036-partial_fetch.c +87 -0
- package/deps/librdkafka/tests/0037-destroy_hang_local.c +85 -0
- package/deps/librdkafka/tests/0038-performance.c +121 -0
- package/deps/librdkafka/tests/0039-event.c +284 -0
- package/deps/librdkafka/tests/0040-io_event.c +257 -0
- package/deps/librdkafka/tests/0041-fetch_max_bytes.c +97 -0
- package/deps/librdkafka/tests/0042-many_topics.c +252 -0
- package/deps/librdkafka/tests/0043-no_connection.c +77 -0
- package/deps/librdkafka/tests/0044-partition_cnt.c +94 -0
- package/deps/librdkafka/tests/0045-subscribe_update.c +1010 -0
- package/deps/librdkafka/tests/0046-rkt_cache.c +65 -0
- package/deps/librdkafka/tests/0047-partial_buf_tmout.c +98 -0
- package/deps/librdkafka/tests/0048-partitioner.c +283 -0
- package/deps/librdkafka/tests/0049-consume_conn_close.c +162 -0
- package/deps/librdkafka/tests/0050-subscribe_adds.c +145 -0
- package/deps/librdkafka/tests/0051-assign_adds.c +126 -0
- package/deps/librdkafka/tests/0052-msg_timestamps.c +238 -0
- package/deps/librdkafka/tests/0053-stats_cb.cpp +527 -0
- package/deps/librdkafka/tests/0054-offset_time.cpp +236 -0
- package/deps/librdkafka/tests/0055-producer_latency.c +539 -0
- package/deps/librdkafka/tests/0056-balanced_group_mt.c +315 -0
- package/deps/librdkafka/tests/0057-invalid_topic.cpp +112 -0
- package/deps/librdkafka/tests/0058-log.cpp +123 -0
- package/deps/librdkafka/tests/0059-bsearch.cpp +241 -0
- package/deps/librdkafka/tests/0060-op_prio.cpp +163 -0
- package/deps/librdkafka/tests/0061-consumer_lag.cpp +295 -0
- package/deps/librdkafka/tests/0062-stats_event.c +126 -0
- package/deps/librdkafka/tests/0063-clusterid.cpp +180 -0
- package/deps/librdkafka/tests/0064-interceptors.c +481 -0
- package/deps/librdkafka/tests/0065-yield.cpp +140 -0
- package/deps/librdkafka/tests/0066-plugins.cpp +129 -0
- package/deps/librdkafka/tests/0067-empty_topic.cpp +151 -0
- package/deps/librdkafka/tests/0068-produce_timeout.c +136 -0
- package/deps/librdkafka/tests/0069-consumer_add_parts.c +119 -0
- package/deps/librdkafka/tests/0070-null_empty.cpp +197 -0
- package/deps/librdkafka/tests/0072-headers_ut.c +448 -0
- package/deps/librdkafka/tests/0073-headers.c +381 -0
- package/deps/librdkafka/tests/0074-producev.c +87 -0
- package/deps/librdkafka/tests/0075-retry.c +290 -0
- package/deps/librdkafka/tests/0076-produce_retry.c +452 -0
- package/deps/librdkafka/tests/0077-compaction.c +363 -0
- package/deps/librdkafka/tests/0078-c_from_cpp.cpp +96 -0
- package/deps/librdkafka/tests/0079-fork.c +93 -0
- package/deps/librdkafka/tests/0080-admin_ut.c +3095 -0
- package/deps/librdkafka/tests/0081-admin.c +5633 -0
- package/deps/librdkafka/tests/0082-fetch_max_bytes.cpp +137 -0
- package/deps/librdkafka/tests/0083-cb_event.c +233 -0
- package/deps/librdkafka/tests/0084-destroy_flags.c +208 -0
- package/deps/librdkafka/tests/0085-headers.cpp +392 -0
- package/deps/librdkafka/tests/0086-purge.c +368 -0
- package/deps/librdkafka/tests/0088-produce_metadata_timeout.c +162 -0
- package/deps/librdkafka/tests/0089-max_poll_interval.c +511 -0
- package/deps/librdkafka/tests/0090-idempotence.c +171 -0
- package/deps/librdkafka/tests/0091-max_poll_interval_timeout.c +295 -0
- package/deps/librdkafka/tests/0092-mixed_msgver.c +103 -0
- package/deps/librdkafka/tests/0093-holb.c +200 -0
- package/deps/librdkafka/tests/0094-idempotence_msg_timeout.c +231 -0
- package/deps/librdkafka/tests/0095-all_brokers_down.cpp +122 -0
- package/deps/librdkafka/tests/0097-ssl_verify.cpp +658 -0
- package/deps/librdkafka/tests/0098-consumer-txn.cpp +1218 -0
- package/deps/librdkafka/tests/0099-commit_metadata.c +194 -0
- package/deps/librdkafka/tests/0100-thread_interceptors.cpp +195 -0
- package/deps/librdkafka/tests/0101-fetch-from-follower.cpp +446 -0
- package/deps/librdkafka/tests/0102-static_group_rebalance.c +836 -0
- package/deps/librdkafka/tests/0103-transactions.c +1383 -0
- package/deps/librdkafka/tests/0104-fetch_from_follower_mock.c +625 -0
- package/deps/librdkafka/tests/0105-transactions_mock.c +3930 -0
- package/deps/librdkafka/tests/0106-cgrp_sess_timeout.c +318 -0
- package/deps/librdkafka/tests/0107-topic_recreate.c +259 -0
- package/deps/librdkafka/tests/0109-auto_create_topics.cpp +278 -0
- package/deps/librdkafka/tests/0110-batch_size.cpp +182 -0
- package/deps/librdkafka/tests/0111-delay_create_topics.cpp +127 -0
- package/deps/librdkafka/tests/0112-assign_unknown_part.c +87 -0
- package/deps/librdkafka/tests/0113-cooperative_rebalance.cpp +3473 -0
- package/deps/librdkafka/tests/0114-sticky_partitioning.cpp +176 -0
- package/deps/librdkafka/tests/0115-producer_auth.cpp +182 -0
- package/deps/librdkafka/tests/0116-kafkaconsumer_close.cpp +216 -0
- package/deps/librdkafka/tests/0117-mock_errors.c +331 -0
- package/deps/librdkafka/tests/0118-commit_rebalance.c +154 -0
- package/deps/librdkafka/tests/0119-consumer_auth.cpp +167 -0
- package/deps/librdkafka/tests/0120-asymmetric_subscription.c +185 -0
- package/deps/librdkafka/tests/0121-clusterid.c +115 -0
- package/deps/librdkafka/tests/0122-buffer_cleaning_after_rebalance.c +227 -0
- package/deps/librdkafka/tests/0123-connections_max_idle.c +98 -0
- package/deps/librdkafka/tests/0124-openssl_invalid_engine.c +69 -0
- package/deps/librdkafka/tests/0125-immediate_flush.c +144 -0
- package/deps/librdkafka/tests/0126-oauthbearer_oidc.c +528 -0
- package/deps/librdkafka/tests/0127-fetch_queue_backoff.cpp +165 -0
- package/deps/librdkafka/tests/0128-sasl_callback_queue.cpp +125 -0
- package/deps/librdkafka/tests/0129-fetch_aborted_msgs.c +79 -0
- package/deps/librdkafka/tests/0130-store_offsets.c +178 -0
- package/deps/librdkafka/tests/0131-connect_timeout.c +81 -0
- package/deps/librdkafka/tests/0132-strategy_ordering.c +179 -0
- package/deps/librdkafka/tests/0133-ssl_keys.c +150 -0
- package/deps/librdkafka/tests/0134-ssl_provider.c +92 -0
- package/deps/librdkafka/tests/0135-sasl_credentials.cpp +143 -0
- package/deps/librdkafka/tests/0136-resolve_cb.c +181 -0
- package/deps/librdkafka/tests/0137-barrier_batch_consume.c +619 -0
- package/deps/librdkafka/tests/0138-admin_mock.c +281 -0
- package/deps/librdkafka/tests/0139-offset_validation_mock.c +950 -0
- package/deps/librdkafka/tests/0140-commit_metadata.cpp +108 -0
- package/deps/librdkafka/tests/0142-reauthentication.c +515 -0
- package/deps/librdkafka/tests/0143-exponential_backoff_mock.c +552 -0
- package/deps/librdkafka/tests/0144-idempotence_mock.c +373 -0
- package/deps/librdkafka/tests/0145-pause_resume_mock.c +119 -0
- package/deps/librdkafka/tests/0146-metadata_mock.c +505 -0
- package/deps/librdkafka/tests/0147-consumer_group_consumer_mock.c +952 -0
- package/deps/librdkafka/tests/0148-offset_fetch_commit_error_mock.c +563 -0
- package/deps/librdkafka/tests/0149-broker-same-host-port.c +140 -0
- package/deps/librdkafka/tests/0150-telemetry_mock.c +651 -0
- package/deps/librdkafka/tests/0151-purge-brokers.c +566 -0
- package/deps/librdkafka/tests/0152-rebootstrap.c +59 -0
- package/deps/librdkafka/tests/0153-memberid.c +128 -0
- package/deps/librdkafka/tests/1000-unktopic.c +164 -0
- package/deps/librdkafka/tests/8000-idle.cpp +60 -0
- package/deps/librdkafka/tests/8001-fetch_from_follower_mock_manual.c +113 -0
- package/deps/librdkafka/tests/CMakeLists.txt +170 -0
- package/deps/librdkafka/tests/LibrdkafkaTestApp.py +291 -0
- package/deps/librdkafka/tests/Makefile +182 -0
- package/deps/librdkafka/tests/README.md +509 -0
- package/deps/librdkafka/tests/autotest.sh +33 -0
- package/deps/librdkafka/tests/backtrace.gdb +30 -0
- package/deps/librdkafka/tests/broker_version_tests.py +315 -0
- package/deps/librdkafka/tests/buildbox.sh +17 -0
- package/deps/librdkafka/tests/cleanup-checker-tests.sh +20 -0
- package/deps/librdkafka/tests/cluster_testing.py +191 -0
- package/deps/librdkafka/tests/delete-test-topics.sh +56 -0
- package/deps/librdkafka/tests/fixtures/oauthbearer/jwt_assertion_template.json +10 -0
- package/deps/librdkafka/tests/fixtures/ssl/Makefile +8 -0
- package/deps/librdkafka/tests/fixtures/ssl/README.md +13 -0
- package/deps/librdkafka/tests/fixtures/ssl/client.keystore.intermediate.p12 +0 -0
- package/deps/librdkafka/tests/fixtures/ssl/client.keystore.p12 +0 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.certificate.intermediate.pem +72 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.certificate.pem +50 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.intermediate.key +46 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.key +46 -0
- package/deps/librdkafka/tests/fixtures/ssl/create_keys.sh +168 -0
- package/deps/librdkafka/tests/fuzzers/Makefile +12 -0
- package/deps/librdkafka/tests/fuzzers/README.md +31 -0
- package/deps/librdkafka/tests/fuzzers/fuzz_regex.c +74 -0
- package/deps/librdkafka/tests/fuzzers/helpers.h +90 -0
- package/deps/librdkafka/tests/gen-ssl-certs.sh +165 -0
- package/deps/librdkafka/tests/interactive_broker_version.py +170 -0
- package/deps/librdkafka/tests/interceptor_test/CMakeLists.txt +16 -0
- package/deps/librdkafka/tests/interceptor_test/Makefile +22 -0
- package/deps/librdkafka/tests/interceptor_test/interceptor_test.c +314 -0
- package/deps/librdkafka/tests/interceptor_test/interceptor_test.h +54 -0
- package/deps/librdkafka/tests/java/IncrementalRebalanceCli.java +97 -0
- package/deps/librdkafka/tests/java/Makefile +13 -0
- package/deps/librdkafka/tests/java/Murmur2Cli.java +46 -0
- package/deps/librdkafka/tests/java/README.md +14 -0
- package/deps/librdkafka/tests/java/TransactionProducerCli.java +162 -0
- package/deps/librdkafka/tests/java/run-class.sh +11 -0
- package/deps/librdkafka/tests/librdkafka.suppressions +483 -0
- package/deps/librdkafka/tests/lz4_manual_test.sh +59 -0
- package/deps/librdkafka/tests/multi-broker-version-test.sh +50 -0
- package/deps/librdkafka/tests/parse-refcnt.sh +43 -0
- package/deps/librdkafka/tests/performance_plot.py +115 -0
- package/deps/librdkafka/tests/plugin_test/Makefile +19 -0
- package/deps/librdkafka/tests/plugin_test/plugin_test.c +58 -0
- package/deps/librdkafka/tests/requirements.txt +2 -0
- package/deps/librdkafka/tests/run-all-tests.sh +79 -0
- package/deps/librdkafka/tests/run-consumer-tests.sh +16 -0
- package/deps/librdkafka/tests/run-producer-tests.sh +16 -0
- package/deps/librdkafka/tests/run-test-batches.py +157 -0
- package/deps/librdkafka/tests/run-test.sh +140 -0
- package/deps/librdkafka/tests/rusage.c +249 -0
- package/deps/librdkafka/tests/sasl_test.py +289 -0
- package/deps/librdkafka/tests/scenarios/README.md +6 -0
- package/deps/librdkafka/tests/scenarios/ak23.json +6 -0
- package/deps/librdkafka/tests/scenarios/default.json +5 -0
- package/deps/librdkafka/tests/scenarios/noautocreate.json +5 -0
- package/deps/librdkafka/tests/sockem.c +801 -0
- package/deps/librdkafka/tests/sockem.h +85 -0
- package/deps/librdkafka/tests/sockem_ctrl.c +145 -0
- package/deps/librdkafka/tests/sockem_ctrl.h +61 -0
- package/deps/librdkafka/tests/test.c +7778 -0
- package/deps/librdkafka/tests/test.conf.example +27 -0
- package/deps/librdkafka/tests/test.h +1028 -0
- package/deps/librdkafka/tests/testcpp.cpp +131 -0
- package/deps/librdkafka/tests/testcpp.h +388 -0
- package/deps/librdkafka/tests/testshared.h +416 -0
- package/deps/librdkafka/tests/tools/README.md +4 -0
- package/deps/librdkafka/tests/tools/stats/README.md +21 -0
- package/deps/librdkafka/tests/tools/stats/filter.jq +42 -0
- package/deps/librdkafka/tests/tools/stats/graph.py +150 -0
- package/deps/librdkafka/tests/tools/stats/requirements.txt +3 -0
- package/deps/librdkafka/tests/tools/stats/to_csv.py +124 -0
- package/deps/librdkafka/tests/trivup/trivup-0.14.0.tar.gz +0 -0
- package/deps/librdkafka/tests/until-fail.sh +87 -0
- package/deps/librdkafka/tests/xxxx-assign_partition.c +122 -0
- package/deps/librdkafka/tests/xxxx-metadata.cpp +159 -0
- package/deps/librdkafka/vcpkg.json +23 -0
- package/deps/librdkafka/win32/README.md +5 -0
- package/deps/librdkafka/win32/build-package.bat +3 -0
- package/deps/librdkafka/win32/build.bat +19 -0
- package/deps/librdkafka/win32/common.vcxproj +84 -0
- package/deps/librdkafka/win32/interceptor_test/interceptor_test.vcxproj +87 -0
- package/deps/librdkafka/win32/librdkafka.autopkg.template +54 -0
- package/deps/librdkafka/win32/librdkafka.master.testing.targets +13 -0
- package/deps/librdkafka/win32/librdkafka.sln +226 -0
- package/deps/librdkafka/win32/librdkafka.vcxproj +276 -0
- package/deps/librdkafka/win32/librdkafkacpp/librdkafkacpp.vcxproj +104 -0
- package/deps/librdkafka/win32/msbuild.ps1 +15 -0
- package/deps/librdkafka/win32/openssl_engine_example/openssl_engine_example.vcxproj +132 -0
- package/deps/librdkafka/win32/package-zip.ps1 +46 -0
- package/deps/librdkafka/win32/packages/repositories.config +4 -0
- package/deps/librdkafka/win32/push-package.bat +4 -0
- package/deps/librdkafka/win32/rdkafka_complex_consumer_example_cpp/rdkafka_complex_consumer_example_cpp.vcxproj +67 -0
- package/deps/librdkafka/win32/rdkafka_example/rdkafka_example.vcxproj +97 -0
- package/deps/librdkafka/win32/rdkafka_performance/rdkafka_performance.vcxproj +97 -0
- package/deps/librdkafka/win32/setup-msys2.ps1 +47 -0
- package/deps/librdkafka/win32/setup-vcpkg.ps1 +34 -0
- package/deps/librdkafka/win32/tests/test.conf.example +25 -0
- package/deps/librdkafka/win32/tests/tests.vcxproj +253 -0
- package/deps/librdkafka/win32/win_ssl_cert_store/win_ssl_cert_store.vcxproj +132 -0
- package/deps/librdkafka/win32/wingetopt.c +564 -0
- package/deps/librdkafka/win32/wingetopt.h +101 -0
- package/deps/librdkafka/win32/wintime.h +33 -0
- package/deps/librdkafka.gyp +62 -0
- package/lib/admin.js +233 -0
- package/lib/client.js +573 -0
- package/lib/error.js +500 -0
- package/lib/index.js +34 -0
- package/lib/kafka-consumer-stream.js +397 -0
- package/lib/kafka-consumer.js +698 -0
- package/lib/producer/high-level-producer.js +323 -0
- package/lib/producer-stream.js +307 -0
- package/lib/producer.js +375 -0
- package/lib/tools/ref-counter.js +52 -0
- package/lib/topic-partition.js +88 -0
- package/lib/topic.js +42 -0
- package/lib/util.js +29 -0
- package/package.json +61 -0
- package/prebuilds/darwin-arm64/@point3+node-rdkafka.node +0 -0
- package/prebuilds/linux-x64/@point3+node-rdkafka.node +0 -0
- package/util/configure.js +30 -0
- package/util/get-env.js +6 -0
- package/util/test-compile.js +11 -0
- package/util/test-producer-delivery.js +100 -0
|
@@ -0,0 +1,1748 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* librdkafka - The Apache Kafka C/C++ library
|
|
3
|
+
*
|
|
4
|
+
* Copyright (c) 2015-2022, Magnus Edenhill
|
|
5
|
+
* 2023, Confluent Inc.
|
|
6
|
+
* All rights reserved.
|
|
7
|
+
*
|
|
8
|
+
* Redistribution and use in source and binary forms, with or without
|
|
9
|
+
* modification, are permitted provided that the following conditions are met:
|
|
10
|
+
*
|
|
11
|
+
* 1. Redistributions of source code must retain the above copyright notice,
|
|
12
|
+
* this list of conditions and the following disclaimer.
|
|
13
|
+
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
14
|
+
* this list of conditions and the following disclaimer in the documentation
|
|
15
|
+
* and/or other materials provided with the distribution.
|
|
16
|
+
*
|
|
17
|
+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
18
|
+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
19
|
+
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
20
|
+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
|
21
|
+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
22
|
+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
23
|
+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
24
|
+
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
25
|
+
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
26
|
+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
27
|
+
* POSSIBILITY OF SUCH DAMAGE.
|
|
28
|
+
*/
|
|
29
|
+
#include "rdkafka_int.h"
|
|
30
|
+
#include "rdkafka_assignor.h"
|
|
31
|
+
#include "rdunittest.h"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Source:
|
|
36
|
+
* https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/clients/consumer/RangeAssignor.java
|
|
37
|
+
*
|
|
38
|
+
* The range assignor works on a per-topic basis. For each topic, we lay out the
|
|
39
|
+
* available partitions in numeric order and the consumers in lexicographic
|
|
40
|
+
* order. We then divide the number of partitions by the total number of
|
|
41
|
+
* consumers to determine the number of partitions to assign to each consumer.
|
|
42
|
+
* If it does not evenly divide, then the first few consumers will have one
|
|
43
|
+
* extra partition.
|
|
44
|
+
*
|
|
45
|
+
* For example, suppose there are two consumers C0 and C1, two topics t0 and t1,
|
|
46
|
+
* and each topic has 3 partitions, resulting in partitions t0p0, t0p1, t0p2,
|
|
47
|
+
* t1p0, t1p1, and t1p2.
|
|
48
|
+
*
|
|
49
|
+
* The assignment will be:
|
|
50
|
+
* C0: [t0p0, t0p1, t1p0, t1p1]
|
|
51
|
+
* C1: [t0p2, t1p2]
|
|
52
|
+
*/
|
|
53
|
+
|
|
54
|
+
typedef struct {
|
|
55
|
+
rd_kafkap_str_t *member_id;
|
|
56
|
+
rd_list_t *assigned_partitions; /* Contained Type: int* */
|
|
57
|
+
} rd_kafka_member_assigned_partitions_pair_t;
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* @brief Intializes a rd_kafka_member_assigned_partitions_pair_t* with
|
|
61
|
+
* assigned_partitions = [].
|
|
62
|
+
*
|
|
63
|
+
* @param member_id
|
|
64
|
+
*
|
|
65
|
+
* The member_id isn't copied, so the returned value can be used only for the
|
|
66
|
+
* lifetime of this function's arguments.
|
|
67
|
+
* @return rd_kafka_member_assigned_partitions_pair_t*
|
|
68
|
+
*/
|
|
69
|
+
static rd_kafka_member_assigned_partitions_pair_t *
|
|
70
|
+
rd_kafka_member_assigned_partitions_pair_new(rd_kafkap_str_t *member_id) {
|
|
71
|
+
rd_kafka_member_assigned_partitions_pair_t *pair =
|
|
72
|
+
rd_calloc(1, sizeof(rd_kafka_member_assigned_partitions_pair_t));
|
|
73
|
+
|
|
74
|
+
pair->member_id = member_id;
|
|
75
|
+
pair->assigned_partitions = rd_list_new(0, NULL);
|
|
76
|
+
return pair;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
static void rd_kafka_member_assigned_partitions_pair_destroy(void *_pair) {
|
|
80
|
+
rd_kafka_member_assigned_partitions_pair_t *pair =
|
|
81
|
+
(rd_kafka_member_assigned_partitions_pair_t *)_pair;
|
|
82
|
+
|
|
83
|
+
/* Do not destroy the member_id, we don't take ownership. */
|
|
84
|
+
RD_IF_FREE(pair->assigned_partitions, rd_list_destroy);
|
|
85
|
+
RD_IF_FREE(pair, rd_free);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
static int rd_kafka_member_assigned_partitions_pair_cmp(const void *_a,
|
|
89
|
+
const void *_b) {
|
|
90
|
+
rd_kafka_member_assigned_partitions_pair_t *a =
|
|
91
|
+
(rd_kafka_member_assigned_partitions_pair_t *)_a;
|
|
92
|
+
rd_kafka_member_assigned_partitions_pair_t *b =
|
|
93
|
+
(rd_kafka_member_assigned_partitions_pair_t *)_b;
|
|
94
|
+
return rd_kafkap_str_cmp(a->member_id, b->member_id);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
static rd_kafka_member_assigned_partitions_pair_t *
|
|
98
|
+
rd_kafka_find_member_assigned_partitions_pair_by_member_id(
|
|
99
|
+
rd_kafkap_str_t *member_id,
|
|
100
|
+
rd_list_t *rd_kafka_member_assigned_partitions_pair_list) {
|
|
101
|
+
rd_kafka_member_assigned_partitions_pair_t search_pair = {member_id,
|
|
102
|
+
NULL};
|
|
103
|
+
return rd_list_find(rd_kafka_member_assigned_partitions_pair_list,
|
|
104
|
+
&search_pair,
|
|
105
|
+
rd_kafka_member_assigned_partitions_pair_cmp);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
typedef struct {
|
|
109
|
+
/* Contains topic and list of members - sorted by group instance id and
|
|
110
|
+
* member id. Also contains partitions, along with partition replicas,
|
|
111
|
+
* which will help us with the racks. The members also contain their
|
|
112
|
+
* rack id and the partitions they have already been assigned.
|
|
113
|
+
*/
|
|
114
|
+
rd_kafka_assignor_topic_t *topic;
|
|
115
|
+
/* unassigned_partitions[i] is true if the ith partition of this topic
|
|
116
|
+
* is not assigned. We prefer using an array rather than using an
|
|
117
|
+
* rd_list and removing elements, because that involves a memmove on
|
|
118
|
+
* each remove. */
|
|
119
|
+
rd_bool_t *unassigned_partitions;
|
|
120
|
+
/* Number of partitions still to be assigned.*/
|
|
121
|
+
size_t unassigned_partitions_left;
|
|
122
|
+
/* An array of char** arrays. The ith element of this array is a sorted
|
|
123
|
+
* char** array, denoting the racks for the ith partition of this topic.
|
|
124
|
+
* The size of this array is equal to the partition_cnt. */
|
|
125
|
+
char ***partition_racks;
|
|
126
|
+
/* The ith element of this array is the size of partition_racks[i]. */
|
|
127
|
+
size_t *racks_cnt;
|
|
128
|
+
/* Contains a pair denoting the partitions assigned to every subscribed
|
|
129
|
+
* consumer (member, [rd_list_t* of int*]). Sorted by member_id.
|
|
130
|
+
* Contained Type: rd_kafka_member_assigned_partitions_pair_t* */
|
|
131
|
+
rd_list_t *member_to_assigned_partitions;
|
|
132
|
+
/* Contains the number of partitions that should be ideally assigned to
|
|
133
|
+
* every subscribing consumer. */
|
|
134
|
+
int num_partitions_per_consumer;
|
|
135
|
+
/* Contains the number of consumers with extra partitions in case number
|
|
136
|
+
* of partitions isn't perfectly divisible by number of consumers. */
|
|
137
|
+
int remaining_consumers_with_extra_partition;
|
|
138
|
+
/* True if we need to perform rack aware assignment. */
|
|
139
|
+
rd_bool_t needs_rack_aware_assignment;
|
|
140
|
+
} rd_kafka_topic_assignment_state_t;
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* @brief Initialize an rd_kafka_topic_assignment_state_t.
|
|
145
|
+
*
|
|
146
|
+
* @param topic
|
|
147
|
+
* @param broker_rack_pair
|
|
148
|
+
* @param broker_rack_pair_cnt
|
|
149
|
+
*
|
|
150
|
+
* The struct rd_kafka_topic_assignment_state_t is mostly for convenience and
|
|
151
|
+
* easy grouping, so we avoid copying values as much as possible. Hence, the
|
|
152
|
+
* returned rd_kafka_topic_assignment_state_t does not own all its values, and
|
|
153
|
+
* should not be used beyond the lifetime of this function's arguments. This
|
|
154
|
+
* function also computes the value of needsRackAwareAssignment given the other
|
|
155
|
+
* information.
|
|
156
|
+
*
|
|
157
|
+
* @return rd_kafka_topic_assignment_state_t*
|
|
158
|
+
*/
|
|
159
|
+
|
|
160
|
+
static rd_kafka_topic_assignment_state_t *
|
|
161
|
+
rd_kafka_topic_assignment_state_new(rd_kafka_assignor_topic_t *topic,
|
|
162
|
+
const rd_kafka_metadata_internal_t *mdi) {
|
|
163
|
+
int i;
|
|
164
|
+
rd_kafka_group_member_t *member;
|
|
165
|
+
rd_kafka_topic_assignment_state_t *rktas;
|
|
166
|
+
const int partition_cnt = topic->metadata->partition_cnt;
|
|
167
|
+
|
|
168
|
+
rktas = rd_calloc(1, sizeof(rd_kafka_topic_assignment_state_t));
|
|
169
|
+
rktas->topic = topic; /* don't copy. */
|
|
170
|
+
|
|
171
|
+
rktas->unassigned_partitions =
|
|
172
|
+
rd_malloc(sizeof(rd_bool_t) * partition_cnt);
|
|
173
|
+
rktas->unassigned_partitions_left = partition_cnt;
|
|
174
|
+
for (i = 0; i < partition_cnt; i++) {
|
|
175
|
+
rktas->unassigned_partitions[i] = rd_true;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
rktas->num_partitions_per_consumer = 0;
|
|
179
|
+
rktas->remaining_consumers_with_extra_partition = 0;
|
|
180
|
+
if (rd_list_cnt(&topic->members)) {
|
|
181
|
+
rktas->num_partitions_per_consumer =
|
|
182
|
+
partition_cnt / rd_list_cnt(&topic->members);
|
|
183
|
+
rktas->remaining_consumers_with_extra_partition =
|
|
184
|
+
partition_cnt % rd_list_cnt(&topic->members);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
rktas->member_to_assigned_partitions =
|
|
188
|
+
rd_list_new(0, rd_kafka_member_assigned_partitions_pair_destroy);
|
|
189
|
+
|
|
190
|
+
RD_LIST_FOREACH(member, &topic->members, i) {
|
|
191
|
+
rd_list_add(rktas->member_to_assigned_partitions,
|
|
192
|
+
rd_kafka_member_assigned_partitions_pair_new(
|
|
193
|
+
member->rkgm_member_id));
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
rd_list_sort(rktas->member_to_assigned_partitions,
|
|
197
|
+
rd_kafka_member_assigned_partitions_pair_cmp);
|
|
198
|
+
|
|
199
|
+
rktas->partition_racks = rd_calloc(partition_cnt, sizeof(char **));
|
|
200
|
+
rktas->racks_cnt = rd_calloc(partition_cnt, sizeof(size_t));
|
|
201
|
+
for (i = 0; topic->metadata_internal->partitions && i < partition_cnt;
|
|
202
|
+
i++) {
|
|
203
|
+
rktas->racks_cnt[i] =
|
|
204
|
+
topic->metadata_internal->partitions[i].racks_cnt;
|
|
205
|
+
rktas->partition_racks[i] =
|
|
206
|
+
topic->metadata_internal->partitions[i].racks;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
rktas->needs_rack_aware_assignment =
|
|
210
|
+
rd_kafka_use_rack_aware_assignment(&topic, 1, mdi);
|
|
211
|
+
|
|
212
|
+
return rktas;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/* Destroy a rd_kafka_topic_assignment_state_t. */
|
|
216
|
+
static void rd_kafka_topic_assignment_state_destroy(void *_rktas) {
|
|
217
|
+
rd_kafka_topic_assignment_state_t *rktas =
|
|
218
|
+
(rd_kafka_topic_assignment_state_t *)_rktas;
|
|
219
|
+
|
|
220
|
+
rd_free(rktas->unassigned_partitions);
|
|
221
|
+
rd_list_destroy(rktas->member_to_assigned_partitions);
|
|
222
|
+
rd_free(rktas->partition_racks);
|
|
223
|
+
rd_free(rktas->racks_cnt);
|
|
224
|
+
rd_free(rktas);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Compare two topic_assignment_states, first on the sorted list of consumers
|
|
229
|
+
* (each consumer from the list of consumers is matched till the first point of
|
|
230
|
+
* difference), and if that's equal, compare on the number of partitions.
|
|
231
|
+
*
|
|
232
|
+
* A list sorted with this comparator will group the topic_assignment_states
|
|
233
|
+
* having the same consumers and the same number of partitions together - this
|
|
234
|
+
* is the criteria of co-partitioned topics.
|
|
235
|
+
*/
|
|
236
|
+
static int rd_kafka_topic_assignment_state_cmp(const void *_a, const void *_b) {
|
|
237
|
+
int i;
|
|
238
|
+
rd_kafka_topic_assignment_state_t *a =
|
|
239
|
+
(rd_kafka_topic_assignment_state_t *)_a;
|
|
240
|
+
rd_kafka_topic_assignment_state_t *b =
|
|
241
|
+
(rd_kafka_topic_assignment_state_t *)_b;
|
|
242
|
+
|
|
243
|
+
/* This guarantee comes from rd_kafka_range_assignor_assign_cb. */
|
|
244
|
+
rd_assert(a->topic->members.rl_flags & RD_LIST_F_SORTED);
|
|
245
|
+
rd_assert(b->topic->members.rl_flags & RD_LIST_F_SORTED);
|
|
246
|
+
|
|
247
|
+
/* Based on consumers */
|
|
248
|
+
for (i = 0; i < rd_list_cnt(&a->topic->members) &&
|
|
249
|
+
i < rd_list_cnt(&b->topic->members);
|
|
250
|
+
i++) {
|
|
251
|
+
rd_kafka_group_member_t *am =
|
|
252
|
+
rd_list_elem(&a->topic->members, i);
|
|
253
|
+
rd_kafka_group_member_t *bm =
|
|
254
|
+
rd_list_elem(&b->topic->members, i);
|
|
255
|
+
int cmp_res =
|
|
256
|
+
rd_kafkap_str_cmp(am->rkgm_member_id, bm->rkgm_member_id);
|
|
257
|
+
if (cmp_res != 0)
|
|
258
|
+
return cmp_res;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
if (rd_list_cnt(&a->topic->members) !=
|
|
262
|
+
rd_list_cnt(&b->topic->members)) {
|
|
263
|
+
return RD_CMP(rd_list_cnt(&a->topic->members),
|
|
264
|
+
rd_list_cnt(&b->topic->members));
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/* Based on number of partitions */
|
|
268
|
+
return RD_CMP(a->topic->metadata->partition_cnt,
|
|
269
|
+
b->topic->metadata->partition_cnt);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
/* Helper function to wrap a bsearch on the partition's racks. */
|
|
274
|
+
static char *rd_kafka_topic_assignment_state_rack_search(
|
|
275
|
+
rd_kafka_topic_assignment_state_t *rktas,
|
|
276
|
+
int partition,
|
|
277
|
+
const char *rack) {
|
|
278
|
+
char **partition_racks = rktas->partition_racks[partition];
|
|
279
|
+
size_t cnt = rktas->racks_cnt[partition];
|
|
280
|
+
void *res = NULL;
|
|
281
|
+
|
|
282
|
+
if (!partition_racks)
|
|
283
|
+
return NULL;
|
|
284
|
+
|
|
285
|
+
res = bsearch(&rack, partition_racks, cnt, sizeof(char *), rd_strcmp3);
|
|
286
|
+
if (!res)
|
|
287
|
+
return NULL;
|
|
288
|
+
|
|
289
|
+
return *(char **)res;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/*
|
|
293
|
+
* Assigns a partition to a member, and updates fields in rktas for accounting.
|
|
294
|
+
* It's assumed that the partitions assigned to this member don't exceed the
|
|
295
|
+
* allowed number.
|
|
296
|
+
*/
|
|
297
|
+
static void rd_kafka_assign_partition(rd_kafka_group_member_t *member,
|
|
298
|
+
rd_kafka_topic_assignment_state_t *rktas,
|
|
299
|
+
int32_t partition) {
|
|
300
|
+
rd_kafka_member_assigned_partitions_pair_t *member_assignment =
|
|
301
|
+
rd_kafka_find_member_assigned_partitions_pair_by_member_id(
|
|
302
|
+
member->rkgm_member_id, rktas->member_to_assigned_partitions);
|
|
303
|
+
rd_assert(member_assignment);
|
|
304
|
+
|
|
305
|
+
/* We can't use &partition, since that's a copy on the stack. */
|
|
306
|
+
rd_list_add(member_assignment->assigned_partitions,
|
|
307
|
+
(void *)&rktas->topic->metadata->partitions[partition].id);
|
|
308
|
+
rd_kafka_topic_partition_list_add_range(member->rkgm_assignment,
|
|
309
|
+
rktas->topic->metadata->topic,
|
|
310
|
+
partition, partition);
|
|
311
|
+
|
|
312
|
+
rd_assert(rktas->unassigned_partitions[partition]);
|
|
313
|
+
rktas->unassigned_partitions[partition] = rd_false;
|
|
314
|
+
rktas->unassigned_partitions_left--;
|
|
315
|
+
|
|
316
|
+
if (rd_list_cnt(member_assignment->assigned_partitions) >
|
|
317
|
+
rktas->num_partitions_per_consumer) {
|
|
318
|
+
rktas->remaining_consumers_with_extra_partition -= 1;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
/* Implementation of may_assign for rd_kafka_assign_ranges. True if the consumer
|
|
324
|
+
* rack is empty, or if is exists within the partition racks. */
|
|
325
|
+
static rd_bool_t rd_kafka_racks_match(rd_kafka_group_member_t *member,
|
|
326
|
+
rd_kafka_topic_assignment_state_t *rktas,
|
|
327
|
+
int32_t partition) {
|
|
328
|
+
rd_kafkap_str_t *consumer_rack = member->rkgm_rack_id;
|
|
329
|
+
|
|
330
|
+
if (!consumer_rack || RD_KAFKAP_STR_LEN(consumer_rack) == 0) {
|
|
331
|
+
return rd_true;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
return rd_kafka_topic_assignment_state_rack_search(
|
|
335
|
+
rktas, partition, consumer_rack->str) != NULL;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
/* Implementation of may_assign for rd_kafka_assign_ranges. Always true, used to
|
|
340
|
+
* assign remaining partitions after rack-aware assignment is complete. */
|
|
341
|
+
static rd_bool_t rd_kafka_always(rd_kafka_group_member_t *member,
|
|
342
|
+
rd_kafka_topic_assignment_state_t *rktas,
|
|
343
|
+
int32_t partition) {
|
|
344
|
+
return rd_true;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/* Assigns as many partitions as possible for a topic to subscribing members,
|
|
348
|
+
* such that no subscribing member exceeds their limit of allowed partitions,
|
|
349
|
+
* and may_assign(member, rktas, partition) is true for each member and
|
|
350
|
+
* partition.
|
|
351
|
+
*/
|
|
352
|
+
static void rd_kafka_assign_ranges(
|
|
353
|
+
rd_kafka_topic_assignment_state_t *rktas,
|
|
354
|
+
rd_bool_t (*may_assign)(rd_kafka_group_member_t *member,
|
|
355
|
+
rd_kafka_topic_assignment_state_t *rktas,
|
|
356
|
+
int32_t partition)) {
|
|
357
|
+
int i;
|
|
358
|
+
rd_kafka_group_member_t *member;
|
|
359
|
+
int32_t *partitions_to_assign =
|
|
360
|
+
rd_alloca(rktas->unassigned_partitions_left * sizeof(int32_t));
|
|
361
|
+
|
|
362
|
+
RD_LIST_FOREACH(member, &rktas->topic->members, i) {
|
|
363
|
+
int j;
|
|
364
|
+
rd_kafka_member_assigned_partitions_pair_t *member_assignment;
|
|
365
|
+
int maximum_assignable_to_consumer;
|
|
366
|
+
int partitions_to_assign_cnt;
|
|
367
|
+
|
|
368
|
+
if (rktas->unassigned_partitions_left == 0)
|
|
369
|
+
break;
|
|
370
|
+
|
|
371
|
+
member_assignment =
|
|
372
|
+
rd_kafka_find_member_assigned_partitions_pair_by_member_id(
|
|
373
|
+
member->rkgm_member_id,
|
|
374
|
+
rktas->member_to_assigned_partitions);
|
|
375
|
+
|
|
376
|
+
maximum_assignable_to_consumer =
|
|
377
|
+
rktas->num_partitions_per_consumer +
|
|
378
|
+
(rktas->remaining_consumers_with_extra_partition > 0) -
|
|
379
|
+
rd_list_cnt(member_assignment->assigned_partitions);
|
|
380
|
+
|
|
381
|
+
if (maximum_assignable_to_consumer <= 0)
|
|
382
|
+
continue;
|
|
383
|
+
|
|
384
|
+
partitions_to_assign_cnt = 0;
|
|
385
|
+
for (j = 0; j < rktas->topic->metadata->partition_cnt; j++) {
|
|
386
|
+
if (!rktas->unassigned_partitions[j]) {
|
|
387
|
+
continue;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
if (maximum_assignable_to_consumer <= 0)
|
|
391
|
+
break;
|
|
392
|
+
|
|
393
|
+
if (!may_assign(member, rktas, j))
|
|
394
|
+
continue;
|
|
395
|
+
|
|
396
|
+
partitions_to_assign[partitions_to_assign_cnt] = j;
|
|
397
|
+
partitions_to_assign_cnt++;
|
|
398
|
+
maximum_assignable_to_consumer--;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
for (j = 0; j < partitions_to_assign_cnt; j++)
|
|
402
|
+
rd_kafka_assign_partition(member, rktas,
|
|
403
|
+
partitions_to_assign[j]);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/*
|
|
408
|
+
* Assigns partitions for co-partitioned topics in a rack-aware manner on a best
|
|
409
|
+
* effort basis. All partitions may not be assigned to consumers in case a rack
|
|
410
|
+
* aware assignment does not exist.
|
|
411
|
+
*/
|
|
412
|
+
static void rd_kafka_assign_co_partitioned(
|
|
413
|
+
rd_list_t *
|
|
414
|
+
rktas_bucket /* Contained Type: rd_kafka_topic_assignment_state_t* */) {
|
|
415
|
+
rd_kafka_topic_assignment_state_t *first_rktas =
|
|
416
|
+
rd_list_elem(rktas_bucket, 0);
|
|
417
|
+
rd_kafka_topic_assignment_state_t *rktas;
|
|
418
|
+
rd_kafka_group_member_t *member;
|
|
419
|
+
int i;
|
|
420
|
+
|
|
421
|
+
/* Since a "bucket" is a group of topic_assignment_states with the same
|
|
422
|
+
* consumers and number of partitions, we can just fetch them from the
|
|
423
|
+
* first member of the bucket. */
|
|
424
|
+
const int partition_cnt = first_rktas->topic->metadata->partition_cnt;
|
|
425
|
+
const rd_list_t *consumers = &first_rktas->topic->members;
|
|
426
|
+
|
|
427
|
+
for (i = 0; i < partition_cnt; i++) {
|
|
428
|
+
/*
|
|
429
|
+
* To assign the ith partition of all the co partitioned topics,
|
|
430
|
+
* we need to find a consumerX that fulfils the criteria:
|
|
431
|
+
* for all topic_assignment_states in the bucket:
|
|
432
|
+
* 1. rack(consumerX) is contained inside racks(partition i)
|
|
433
|
+
* 2. partitions assigned to consumerX does not exceed limits.
|
|
434
|
+
*/
|
|
435
|
+
int j;
|
|
436
|
+
RD_LIST_FOREACH(member, consumers, j) {
|
|
437
|
+
int m;
|
|
438
|
+
RD_LIST_FOREACH(rktas, rktas_bucket, m) {
|
|
439
|
+
int maximum_assignable;
|
|
440
|
+
rd_kafka_member_assigned_partitions_pair_t
|
|
441
|
+
*member_assignment;
|
|
442
|
+
|
|
443
|
+
/* Check (1.) */
|
|
444
|
+
if (!member->rkgm_rack_id ||
|
|
445
|
+
RD_KAFKAP_STR_LEN(member->rkgm_rack_id) ==
|
|
446
|
+
0 ||
|
|
447
|
+
rd_kafka_topic_assignment_state_rack_search(
|
|
448
|
+
rktas, i, member->rkgm_rack_id->str) ==
|
|
449
|
+
NULL) {
|
|
450
|
+
break;
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
/* Check (2.) */
|
|
454
|
+
member_assignment =
|
|
455
|
+
rd_kafka_find_member_assigned_partitions_pair_by_member_id(
|
|
456
|
+
member->rkgm_member_id,
|
|
457
|
+
rktas->member_to_assigned_partitions);
|
|
458
|
+
maximum_assignable =
|
|
459
|
+
rktas->num_partitions_per_consumer +
|
|
460
|
+
(rktas
|
|
461
|
+
->remaining_consumers_with_extra_partition >
|
|
462
|
+
0) -
|
|
463
|
+
rd_list_cnt(
|
|
464
|
+
member_assignment->assigned_partitions);
|
|
465
|
+
|
|
466
|
+
if (maximum_assignable <= 0) {
|
|
467
|
+
break;
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
if (m == rd_list_cnt(rktas_bucket)) {
|
|
471
|
+
/* Break early - this consumer can be assigned
|
|
472
|
+
* this partition. */
|
|
473
|
+
break;
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
if (j == rd_list_cnt(&first_rktas->topic->members)) {
|
|
477
|
+
continue; /* We didn't find a suitable consumer. */
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
rd_assert(member);
|
|
481
|
+
|
|
482
|
+
RD_LIST_FOREACH(rktas, rktas_bucket, j) {
|
|
483
|
+
rd_kafka_assign_partition(member, rktas, i);
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
/* FIXME: A possible optimization: early break here if no
|
|
487
|
+
* consumer remains with maximum_assignable_to_consumer > 0
|
|
488
|
+
* across all topics. */
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
rd_kafka_resp_err_t
|
|
494
|
+
rd_kafka_range_assignor_assign_cb(rd_kafka_t *rk,
|
|
495
|
+
const rd_kafka_assignor_t *rkas,
|
|
496
|
+
const char *member_id,
|
|
497
|
+
const rd_kafka_metadata_t *metadata,
|
|
498
|
+
rd_kafka_group_member_t *members,
|
|
499
|
+
size_t member_cnt,
|
|
500
|
+
rd_kafka_assignor_topic_t **eligible_topics,
|
|
501
|
+
size_t eligible_topic_cnt,
|
|
502
|
+
char *errstr,
|
|
503
|
+
size_t errstr_size,
|
|
504
|
+
void *opaque) {
|
|
505
|
+
unsigned int ti;
|
|
506
|
+
int i;
|
|
507
|
+
rd_list_t *rktas_list = rd_list_new(
|
|
508
|
+
eligible_topic_cnt, rd_kafka_topic_assignment_state_destroy);
|
|
509
|
+
rd_list_t *rktas_buckets = rd_list_new(0, rd_list_destroy_free);
|
|
510
|
+
rd_list_t
|
|
511
|
+
*rktas_current_bucket; /* Contained Type:
|
|
512
|
+
rd_kafka_topic_assignment_state_t* */
|
|
513
|
+
rd_kafka_topic_assignment_state_t *rktas;
|
|
514
|
+
rd_kafka_topic_assignment_state_t *prev_rktas;
|
|
515
|
+
const rd_kafka_metadata_internal_t *mdi =
|
|
516
|
+
rd_kafka_metadata_get_internal(metadata);
|
|
517
|
+
|
|
518
|
+
/* The range assignor works on a per-topic basis. */
|
|
519
|
+
for (ti = 0; ti < eligible_topic_cnt; ti++) {
|
|
520
|
+
rd_kafka_assignor_topic_t *eligible_topic = eligible_topics[ti];
|
|
521
|
+
|
|
522
|
+
/* For each topic, we sort the consumers in lexicographic order,
|
|
523
|
+
* and create a topic_assignment_state. */
|
|
524
|
+
rd_list_sort(&eligible_topic->members,
|
|
525
|
+
rd_kafka_group_member_cmp);
|
|
526
|
+
rd_list_add(rktas_list, rd_kafka_topic_assignment_state_new(
|
|
527
|
+
eligible_topic, mdi));
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
/* Sort the topic_assignment_states to group the topics which need to be
|
|
531
|
+
* co-partitioned. */
|
|
532
|
+
rd_list_sort(rktas_list, rd_kafka_topic_assignment_state_cmp);
|
|
533
|
+
|
|
534
|
+
/* Use the sorted list of topic_assignment_states and separate them into
|
|
535
|
+
* "buckets". Each bucket contains topics which can be co-partitioned,
|
|
536
|
+
* ie with the same consumers and number of partitions. */
|
|
537
|
+
prev_rktas = NULL;
|
|
538
|
+
rktas_current_bucket = NULL;
|
|
539
|
+
RD_LIST_FOREACH(rktas, rktas_list, i) {
|
|
540
|
+
if (prev_rktas && rd_kafka_topic_assignment_state_cmp(
|
|
541
|
+
rktas, prev_rktas) == 0) {
|
|
542
|
+
rd_list_add(rktas_current_bucket, rktas);
|
|
543
|
+
continue;
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
/* The free function is set to NULL, as we don't copy any of the
|
|
547
|
+
* topic_assignment_states. */
|
|
548
|
+
rktas_current_bucket = rd_list_new(0, NULL);
|
|
549
|
+
rd_list_add(rktas_buckets, rktas_current_bucket);
|
|
550
|
+
prev_rktas = rktas;
|
|
551
|
+
rd_list_add(rktas_current_bucket, rktas);
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
/* Iterate through each bucket. In case there's more than one element in
|
|
555
|
+
* the bucket, we prefer co-partitioning over rack awareness. Otherwise,
|
|
556
|
+
* assign with rack-awareness. */
|
|
557
|
+
rktas = NULL;
|
|
558
|
+
rktas_current_bucket = NULL;
|
|
559
|
+
RD_LIST_FOREACH(rktas_current_bucket, rktas_buckets, i) {
|
|
560
|
+
rd_assert(rd_list_cnt(rktas_current_bucket) > 0);
|
|
561
|
+
|
|
562
|
+
if (rd_list_cnt(rktas_current_bucket) == 1) {
|
|
563
|
+
rktas = rd_list_elem(rktas_current_bucket, 0);
|
|
564
|
+
if (!rktas->needs_rack_aware_assignment)
|
|
565
|
+
continue;
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
rd_kafka_dbg(rk, CGRP, "ASSIGN",
|
|
569
|
+
"range: Topic %s with %d partition(s) and "
|
|
570
|
+
"%d subscribing member(s), single-topic "
|
|
571
|
+
"rack-aware assignment",
|
|
572
|
+
rktas->topic->metadata->topic,
|
|
573
|
+
rktas->topic->metadata->partition_cnt,
|
|
574
|
+
rd_list_cnt(&rktas->topic->members));
|
|
575
|
+
|
|
576
|
+
rd_kafka_assign_ranges(rktas, rd_kafka_racks_match);
|
|
577
|
+
} else {
|
|
578
|
+
rktas = rd_list_elem(rktas_current_bucket, 0);
|
|
579
|
+
rd_kafka_dbg(
|
|
580
|
+
rk, CGRP, "ASSIGN",
|
|
581
|
+
"range: %d topics with %d partition(s) and "
|
|
582
|
+
"%d subscribing member(s), co-partitioned "
|
|
583
|
+
"rack-aware assignment",
|
|
584
|
+
rd_list_cnt(rktas_current_bucket),
|
|
585
|
+
rktas->topic->metadata->partition_cnt,
|
|
586
|
+
rd_list_cnt(&rktas->topic->members));
|
|
587
|
+
|
|
588
|
+
rd_kafka_assign_co_partitioned(rktas_current_bucket);
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
/* Iterate through each rktas, doing normal assignment for any
|
|
593
|
+
* partitions that might not have gotten a rack-aware assignment.*/
|
|
594
|
+
RD_LIST_FOREACH(rktas, rktas_list, i) {
|
|
595
|
+
rd_kafka_dbg(rk, CGRP, "ASSIGN",
|
|
596
|
+
"range: Topic %s with %d partition(s) and "
|
|
597
|
+
"%d subscribing member(s), single-topic "
|
|
598
|
+
"non-rack-aware assignment for %" PRIusz
|
|
599
|
+
" leftover partitions",
|
|
600
|
+
rktas->topic->metadata->topic,
|
|
601
|
+
rktas->topic->metadata->partition_cnt,
|
|
602
|
+
rd_list_cnt(&rktas->topic->members),
|
|
603
|
+
rktas->unassigned_partitions_left);
|
|
604
|
+
rd_kafka_assign_ranges(rktas, rd_kafka_always);
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
rd_list_destroy(rktas_list);
|
|
608
|
+
rd_list_destroy(rktas_buckets);
|
|
609
|
+
|
|
610
|
+
return 0;
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
/**
|
|
615
|
+
* @name Sticky assignor unit tests
|
|
616
|
+
*
|
|
617
|
+
*
|
|
618
|
+
* These are based on RangeAssignorTest.java
|
|
619
|
+
*
|
|
620
|
+
*
|
|
621
|
+
*
|
|
622
|
+
*/
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
/* All possible racks used in tests, as well as several common rack configs used
|
|
626
|
+
* by consumers */
|
|
627
|
+
static rd_kafkap_str_t
|
|
628
|
+
*ALL_RACKS[7]; /* initialized before starting the unit tests. */
|
|
629
|
+
static int RACKS_INITIAL[] = {0, 1, 2};
|
|
630
|
+
static int RACKS_NULL[] = {6, 6, 6};
|
|
631
|
+
static int RACKS_FINAL[] = {4, 5, 6};
|
|
632
|
+
static int RACKS_ONE_NULL[] = {6, 4, 5};
|
|
633
|
+
|
|
634
|
+
static int
|
|
635
|
+
ut_testOneConsumerNoTopic(rd_kafka_t *rk,
|
|
636
|
+
const rd_kafka_assignor_t *rkas,
|
|
637
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) {
|
|
638
|
+
rd_kafka_resp_err_t err;
|
|
639
|
+
char errstr[512];
|
|
640
|
+
rd_kafka_metadata_t *metadata;
|
|
641
|
+
rd_kafka_group_member_t members[1];
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
if (parametrization == RD_KAFKA_RANGE_ASSIGNOR_UT_NO_BROKER_RACK) {
|
|
645
|
+
RD_UT_PASS();
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
|
|
649
|
+
RD_ARRAYSIZE(ALL_RACKS), parametrization,
|
|
650
|
+
0);
|
|
651
|
+
|
|
652
|
+
ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
|
|
653
|
+
parametrization, "t1", NULL);
|
|
654
|
+
|
|
655
|
+
err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
|
|
656
|
+
RD_ARRAYSIZE(members), errstr,
|
|
657
|
+
sizeof(errstr));
|
|
658
|
+
RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
|
|
659
|
+
|
|
660
|
+
verifyAssignment(&members[0], NULL);
|
|
661
|
+
|
|
662
|
+
rd_kafka_group_member_clear(&members[0]);
|
|
663
|
+
ut_destroy_metadata(metadata);
|
|
664
|
+
|
|
665
|
+
RD_UT_PASS();
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
static int ut_testOneConsumerNonexistentTopic(
|
|
669
|
+
rd_kafka_t *rk,
|
|
670
|
+
const rd_kafka_assignor_t *rkas,
|
|
671
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) {
|
|
672
|
+
rd_kafka_resp_err_t err;
|
|
673
|
+
char errstr[512];
|
|
674
|
+
rd_kafka_metadata_t *metadata;
|
|
675
|
+
rd_kafka_group_member_t members[1];
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
if (parametrization == RD_KAFKA_RANGE_ASSIGNOR_UT_NO_BROKER_RACK) {
|
|
679
|
+
RD_UT_PASS();
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
|
|
683
|
+
RD_ARRAYSIZE(ALL_RACKS), parametrization,
|
|
684
|
+
1, "t1", 0);
|
|
685
|
+
|
|
686
|
+
ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
|
|
687
|
+
parametrization, "t1", NULL);
|
|
688
|
+
|
|
689
|
+
err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
|
|
690
|
+
RD_ARRAYSIZE(members), errstr,
|
|
691
|
+
sizeof(errstr));
|
|
692
|
+
RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
|
|
693
|
+
|
|
694
|
+
verifyAssignment(&members[0], NULL);
|
|
695
|
+
|
|
696
|
+
rd_kafka_group_member_clear(&members[0]);
|
|
697
|
+
ut_destroy_metadata(metadata);
|
|
698
|
+
|
|
699
|
+
RD_UT_PASS();
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
|
|
703
|
+
static int
|
|
704
|
+
ut_testOneConsumerOneTopic(rd_kafka_t *rk,
|
|
705
|
+
const rd_kafka_assignor_t *rkas,
|
|
706
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) {
|
|
707
|
+
rd_kafka_resp_err_t err;
|
|
708
|
+
char errstr[512];
|
|
709
|
+
rd_kafka_metadata_t *metadata;
|
|
710
|
+
rd_kafka_group_member_t members[1];
|
|
711
|
+
|
|
712
|
+
ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
|
|
713
|
+
RD_ARRAYSIZE(ALL_RACKS), parametrization,
|
|
714
|
+
1, "t1", 3);
|
|
715
|
+
|
|
716
|
+
ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
|
|
717
|
+
parametrization, "t1", NULL);
|
|
718
|
+
|
|
719
|
+
err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
|
|
720
|
+
RD_ARRAYSIZE(members), errstr,
|
|
721
|
+
sizeof(errstr));
|
|
722
|
+
RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
|
|
723
|
+
RD_UT_ASSERT(members[0].rkgm_assignment->cnt == 3,
|
|
724
|
+
"expected assignment of 3 partitions, got %d partition(s)",
|
|
725
|
+
members[0].rkgm_assignment->cnt);
|
|
726
|
+
|
|
727
|
+
verifyAssignment(&members[0], "t1", 0, "t1", 1, "t1", 2, NULL);
|
|
728
|
+
|
|
729
|
+
rd_kafka_group_member_clear(&members[0]);
|
|
730
|
+
ut_destroy_metadata(metadata);
|
|
731
|
+
|
|
732
|
+
RD_UT_PASS();
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
static int ut_testOnlyAssignsPartitionsFromSubscribedTopics(
|
|
737
|
+
rd_kafka_t *rk,
|
|
738
|
+
const rd_kafka_assignor_t *rkas,
|
|
739
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) {
|
|
740
|
+
rd_kafka_resp_err_t err;
|
|
741
|
+
char errstr[512];
|
|
742
|
+
rd_kafka_metadata_t *metadata;
|
|
743
|
+
rd_kafka_group_member_t members[1];
|
|
744
|
+
|
|
745
|
+
ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
|
|
746
|
+
RD_ARRAYSIZE(ALL_RACKS), parametrization,
|
|
747
|
+
2, "t1", 3, "t2", 3);
|
|
748
|
+
|
|
749
|
+
ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
|
|
750
|
+
parametrization, "t1", NULL);
|
|
751
|
+
|
|
752
|
+
err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
|
|
753
|
+
RD_ARRAYSIZE(members), errstr,
|
|
754
|
+
sizeof(errstr));
|
|
755
|
+
RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
|
|
756
|
+
|
|
757
|
+
verifyAssignment(&members[0], "t1", 0, "t1", 1, "t1", 2, NULL);
|
|
758
|
+
|
|
759
|
+
rd_kafka_group_member_clear(&members[0]);
|
|
760
|
+
ut_destroy_metadata(metadata);
|
|
761
|
+
|
|
762
|
+
RD_UT_PASS();
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
static int ut_testOneConsumerMultipleTopics(
|
|
766
|
+
rd_kafka_t *rk,
|
|
767
|
+
const rd_kafka_assignor_t *rkas,
|
|
768
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) {
|
|
769
|
+
rd_kafka_resp_err_t err;
|
|
770
|
+
char errstr[512];
|
|
771
|
+
rd_kafka_metadata_t *metadata;
|
|
772
|
+
rd_kafka_group_member_t members[1];
|
|
773
|
+
|
|
774
|
+
ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
|
|
775
|
+
RD_ARRAYSIZE(ALL_RACKS), parametrization,
|
|
776
|
+
2, "t1", 1, "t2", 2);
|
|
777
|
+
|
|
778
|
+
ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
|
|
779
|
+
parametrization, "t1", "t2", NULL);
|
|
780
|
+
|
|
781
|
+
err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
|
|
782
|
+
RD_ARRAYSIZE(members), errstr,
|
|
783
|
+
sizeof(errstr));
|
|
784
|
+
RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
|
|
785
|
+
|
|
786
|
+
verifyAssignment(&members[0], "t1", 0, "t2", 0, "t2", 1, NULL);
|
|
787
|
+
|
|
788
|
+
rd_kafka_group_member_clear(&members[0]);
|
|
789
|
+
ut_destroy_metadata(metadata);
|
|
790
|
+
|
|
791
|
+
RD_UT_PASS();
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
static int ut_testTwoConsumersOneTopicOnePartition(
|
|
795
|
+
rd_kafka_t *rk,
|
|
796
|
+
const rd_kafka_assignor_t *rkas,
|
|
797
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) {
|
|
798
|
+
rd_kafka_resp_err_t err;
|
|
799
|
+
char errstr[512];
|
|
800
|
+
rd_kafka_metadata_t *metadata;
|
|
801
|
+
rd_kafka_group_member_t members[2];
|
|
802
|
+
|
|
803
|
+
ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
|
|
804
|
+
RD_ARRAYSIZE(ALL_RACKS), parametrization,
|
|
805
|
+
1, "t1", 1);
|
|
806
|
+
|
|
807
|
+
ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
|
|
808
|
+
parametrization, "t1", NULL);
|
|
809
|
+
ut_initMemberConditionalRack(&members[1], "consumer2", ALL_RACKS[1],
|
|
810
|
+
parametrization, "t1", NULL);
|
|
811
|
+
|
|
812
|
+
err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
|
|
813
|
+
RD_ARRAYSIZE(members), errstr,
|
|
814
|
+
sizeof(errstr));
|
|
815
|
+
RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
|
|
816
|
+
|
|
817
|
+
verifyAssignment(&members[0], "t1", 0, NULL);
|
|
818
|
+
verifyAssignment(&members[1], NULL);
|
|
819
|
+
|
|
820
|
+
rd_kafka_group_member_clear(&members[0]);
|
|
821
|
+
rd_kafka_group_member_clear(&members[1]);
|
|
822
|
+
ut_destroy_metadata(metadata);
|
|
823
|
+
|
|
824
|
+
RD_UT_PASS();
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
static int ut_testTwoConsumersOneTopicTwoPartitions(
|
|
828
|
+
rd_kafka_t *rk,
|
|
829
|
+
const rd_kafka_assignor_t *rkas,
|
|
830
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) {
|
|
831
|
+
rd_kafka_resp_err_t err;
|
|
832
|
+
char errstr[512];
|
|
833
|
+
rd_kafka_metadata_t *metadata;
|
|
834
|
+
rd_kafka_group_member_t members[2];
|
|
835
|
+
|
|
836
|
+
ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
|
|
837
|
+
RD_ARRAYSIZE(ALL_RACKS), parametrization,
|
|
838
|
+
1, "t1", 2);
|
|
839
|
+
|
|
840
|
+
ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
|
|
841
|
+
parametrization, "t1", NULL);
|
|
842
|
+
ut_initMemberConditionalRack(&members[1], "consumer2", ALL_RACKS[1],
|
|
843
|
+
parametrization, "t1", NULL);
|
|
844
|
+
|
|
845
|
+
err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
|
|
846
|
+
RD_ARRAYSIZE(members), errstr,
|
|
847
|
+
sizeof(errstr));
|
|
848
|
+
RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
|
|
849
|
+
|
|
850
|
+
verifyAssignment(&members[0], "t1", 0, NULL);
|
|
851
|
+
verifyAssignment(&members[1], "t1", 1, NULL);
|
|
852
|
+
|
|
853
|
+
rd_kafka_group_member_clear(&members[0]);
|
|
854
|
+
rd_kafka_group_member_clear(&members[1]);
|
|
855
|
+
ut_destroy_metadata(metadata);
|
|
856
|
+
|
|
857
|
+
RD_UT_PASS();
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
static int ut_testMultipleConsumersMixedTopicSubscriptions(
|
|
861
|
+
rd_kafka_t *rk,
|
|
862
|
+
const rd_kafka_assignor_t *rkas,
|
|
863
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) {
|
|
864
|
+
rd_kafka_resp_err_t err;
|
|
865
|
+
char errstr[512];
|
|
866
|
+
rd_kafka_metadata_t *metadata;
|
|
867
|
+
rd_kafka_group_member_t members[3];
|
|
868
|
+
|
|
869
|
+
ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
|
|
870
|
+
RD_ARRAYSIZE(ALL_RACKS), parametrization,
|
|
871
|
+
2, "t1", 3, "t2", 2);
|
|
872
|
+
|
|
873
|
+
ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
|
|
874
|
+
parametrization, "t1", NULL);
|
|
875
|
+
ut_initMemberConditionalRack(&members[1], "consumer2", ALL_RACKS[1],
|
|
876
|
+
parametrization, "t1", "t2", NULL);
|
|
877
|
+
ut_initMemberConditionalRack(&members[2], "consumer3", ALL_RACKS[2],
|
|
878
|
+
parametrization, "t1", NULL);
|
|
879
|
+
|
|
880
|
+
err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
|
|
881
|
+
RD_ARRAYSIZE(members), errstr,
|
|
882
|
+
sizeof(errstr));
|
|
883
|
+
RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
|
|
884
|
+
|
|
885
|
+
verifyAssignment(&members[0], "t1", 0, NULL);
|
|
886
|
+
verifyAssignment(&members[1], "t1", 1, "t2", 0, "t2", 1, NULL);
|
|
887
|
+
verifyAssignment(&members[2], "t1", 2, NULL);
|
|
888
|
+
|
|
889
|
+
rd_kafka_group_member_clear(&members[0]);
|
|
890
|
+
rd_kafka_group_member_clear(&members[1]);
|
|
891
|
+
rd_kafka_group_member_clear(&members[2]);
|
|
892
|
+
ut_destroy_metadata(metadata);
|
|
893
|
+
|
|
894
|
+
RD_UT_PASS();
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
static int ut_testTwoConsumersTwoTopicsSixPartitions(
|
|
898
|
+
rd_kafka_t *rk,
|
|
899
|
+
const rd_kafka_assignor_t *rkas,
|
|
900
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) {
|
|
901
|
+
rd_kafka_resp_err_t err;
|
|
902
|
+
char errstr[512];
|
|
903
|
+
rd_kafka_metadata_t *metadata;
|
|
904
|
+
rd_kafka_group_member_t members[2];
|
|
905
|
+
|
|
906
|
+
ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
|
|
907
|
+
RD_ARRAYSIZE(ALL_RACKS), parametrization,
|
|
908
|
+
2, "t1", 3, "t2", 3);
|
|
909
|
+
|
|
910
|
+
ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
|
|
911
|
+
parametrization, "t1", "t2", NULL);
|
|
912
|
+
ut_initMemberConditionalRack(&members[1], "consumer2", ALL_RACKS[1],
|
|
913
|
+
parametrization, "t1", "t2", NULL);
|
|
914
|
+
|
|
915
|
+
err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
|
|
916
|
+
RD_ARRAYSIZE(members), errstr,
|
|
917
|
+
sizeof(errstr));
|
|
918
|
+
RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
|
|
919
|
+
|
|
920
|
+
verifyAssignment(&members[0], "t1", 0, "t1", 1, "t2", 0, "t2", 1, NULL);
|
|
921
|
+
verifyAssignment(&members[1], "t1", 2, "t2", 2, NULL);
|
|
922
|
+
|
|
923
|
+
rd_kafka_group_member_clear(&members[0]);
|
|
924
|
+
rd_kafka_group_member_clear(&members[1]);
|
|
925
|
+
ut_destroy_metadata(metadata);
|
|
926
|
+
|
|
927
|
+
RD_UT_PASS();
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
|
|
931
|
+
/* Helper for setting up metadata and members, and running the assignor. Does
|
|
932
|
+
* not check the results of the assignment. */
|
|
933
|
+
static int setupRackAwareAssignment0(rd_kafka_t *rk,
|
|
934
|
+
const rd_kafka_assignor_t *rkas,
|
|
935
|
+
rd_kafka_group_member_t *members,
|
|
936
|
+
size_t member_cnt,
|
|
937
|
+
int replication_factor,
|
|
938
|
+
int num_broker_racks,
|
|
939
|
+
size_t topic_cnt,
|
|
940
|
+
char *topics[],
|
|
941
|
+
int *partitions,
|
|
942
|
+
int *subscriptions_count,
|
|
943
|
+
char **subscriptions[],
|
|
944
|
+
int *consumer_racks,
|
|
945
|
+
rd_kafka_metadata_t **metadata) {
|
|
946
|
+
rd_kafka_resp_err_t err;
|
|
947
|
+
char errstr[512];
|
|
948
|
+
rd_kafka_metadata_t *metadata_local = NULL;
|
|
949
|
+
if (!metadata)
|
|
950
|
+
metadata = &metadata_local;
|
|
951
|
+
|
|
952
|
+
size_t i = 0;
|
|
953
|
+
const int num_brokers = num_broker_racks > 0
|
|
954
|
+
? replication_factor * num_broker_racks
|
|
955
|
+
: replication_factor;
|
|
956
|
+
|
|
957
|
+
/* The member naming for tests is consumerN where N is a single
|
|
958
|
+
* character. */
|
|
959
|
+
rd_assert(member_cnt <= 9);
|
|
960
|
+
|
|
961
|
+
*metadata = rd_kafka_metadata_new_topic_with_partition_replicas_mock(
|
|
962
|
+
replication_factor, num_brokers, topics, partitions, topic_cnt);
|
|
963
|
+
ut_populate_internal_broker_metadata(
|
|
964
|
+
rd_kafka_metadata_get_internal(*metadata), num_broker_racks,
|
|
965
|
+
ALL_RACKS, RD_ARRAYSIZE(ALL_RACKS));
|
|
966
|
+
ut_populate_internal_topic_metadata(
|
|
967
|
+
rd_kafka_metadata_get_internal(*metadata));
|
|
968
|
+
|
|
969
|
+
for (i = 0; i < member_cnt; i++) {
|
|
970
|
+
char member_id[10];
|
|
971
|
+
snprintf(member_id, 10, "consumer%d", (int)(i + 1));
|
|
972
|
+
ut_init_member_with_rack(
|
|
973
|
+
&members[i], member_id, ALL_RACKS[consumer_racks[i]],
|
|
974
|
+
subscriptions[i], subscriptions_count[i]);
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, *metadata, members,
|
|
978
|
+
member_cnt, errstr, sizeof(errstr));
|
|
979
|
+
RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
|
|
980
|
+
|
|
981
|
+
if (metadata_local)
|
|
982
|
+
ut_destroy_metadata(metadata_local);
|
|
983
|
+
return 0;
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
static int setupRackAwareAssignment(rd_kafka_t *rk,
|
|
987
|
+
const rd_kafka_assignor_t *rkas,
|
|
988
|
+
rd_kafka_group_member_t *members,
|
|
989
|
+
size_t member_cnt,
|
|
990
|
+
int replication_factor,
|
|
991
|
+
int num_broker_racks,
|
|
992
|
+
size_t topic_cnt,
|
|
993
|
+
char *topics[],
|
|
994
|
+
int *partitions,
|
|
995
|
+
int *subscriptions_count,
|
|
996
|
+
char **subscriptions[],
|
|
997
|
+
int *consumer_racks) {
|
|
998
|
+
return setupRackAwareAssignment0(
|
|
999
|
+
rk, rkas, members, member_cnt, replication_factor, num_broker_racks,
|
|
1000
|
+
topic_cnt, topics, partitions, subscriptions_count, subscriptions,
|
|
1001
|
+
consumer_racks, NULL);
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
/* Helper for testing cases where rack-aware assignment should not be triggered,
|
|
1005
|
+
* and assignment should be the same as the pre-rack-aware assignor. */
|
|
1006
|
+
#define verifyNonRackAwareAssignment(rk, rkas, members, member_cnt, topic_cnt, \
|
|
1007
|
+
topics, partitions, subscriptions_count, \
|
|
1008
|
+
subscriptions, ...) \
|
|
1009
|
+
do { \
|
|
1010
|
+
size_t idx = 0; \
|
|
1011
|
+
rd_kafka_metadata_t *metadata = NULL; \
|
|
1012
|
+
\
|
|
1013
|
+
/* num_broker_racks = 0, implies that brokers have no \
|
|
1014
|
+
* configured racks. */ \
|
|
1015
|
+
setupRackAwareAssignment(rk, rkas, members, member_cnt, 3, 0, \
|
|
1016
|
+
topic_cnt, topics, partitions, \
|
|
1017
|
+
subscriptions_count, subscriptions, \
|
|
1018
|
+
RACKS_INITIAL); \
|
|
1019
|
+
verifyMultipleAssignment(members, member_cnt, __VA_ARGS__); \
|
|
1020
|
+
for (idx = 0; idx < member_cnt; idx++) \
|
|
1021
|
+
rd_kafka_group_member_clear(&members[idx]); \
|
|
1022
|
+
/* consumer_racks = RACKS_NULL implies that consumers have no \
|
|
1023
|
+
* racks. */ \
|
|
1024
|
+
setupRackAwareAssignment(rk, rkas, members, member_cnt, 3, 3, \
|
|
1025
|
+
topic_cnt, topics, partitions, \
|
|
1026
|
+
subscriptions_count, subscriptions, \
|
|
1027
|
+
RACKS_NULL); \
|
|
1028
|
+
verifyMultipleAssignment(members, member_cnt, __VA_ARGS__); \
|
|
1029
|
+
for (idx = 0; idx < member_cnt; idx++) \
|
|
1030
|
+
rd_kafka_group_member_clear(&members[idx]); \
|
|
1031
|
+
/* replication_factor = 3 and num_broker_racks = 3 means that \
|
|
1032
|
+
* all partitions are replicated on all racks.*/ \
|
|
1033
|
+
setupRackAwareAssignment0(rk, rkas, members, member_cnt, 3, 3, \
|
|
1034
|
+
topic_cnt, topics, partitions, \
|
|
1035
|
+
subscriptions_count, subscriptions, \
|
|
1036
|
+
RACKS_INITIAL, &metadata); \
|
|
1037
|
+
verifyMultipleAssignment(members, member_cnt, __VA_ARGS__); \
|
|
1038
|
+
verifyNumPartitionsWithRackMismatch(metadata, members, \
|
|
1039
|
+
RD_ARRAYSIZE(members), 0); \
|
|
1040
|
+
\
|
|
1041
|
+
for (idx = 0; idx < member_cnt; idx++) \
|
|
1042
|
+
rd_kafka_group_member_clear(&members[idx]); \
|
|
1043
|
+
ut_destroy_metadata(metadata); \
|
|
1044
|
+
/* replication_factor = 4 and num_broker_racks = 4 means that \
|
|
1045
|
+
* all partitions are replicated on all racks. */ \
|
|
1046
|
+
setupRackAwareAssignment0(rk, rkas, members, member_cnt, 4, 4, \
|
|
1047
|
+
topic_cnt, topics, partitions, \
|
|
1048
|
+
subscriptions_count, subscriptions, \
|
|
1049
|
+
RACKS_INITIAL, &metadata); \
|
|
1050
|
+
verifyMultipleAssignment(members, member_cnt, __VA_ARGS__); \
|
|
1051
|
+
verifyNumPartitionsWithRackMismatch(metadata, members, \
|
|
1052
|
+
RD_ARRAYSIZE(members), 0); \
|
|
1053
|
+
\
|
|
1054
|
+
for (idx = 0; idx < member_cnt; idx++) \
|
|
1055
|
+
rd_kafka_group_member_clear(&members[idx]); \
|
|
1056
|
+
ut_destroy_metadata(metadata); \
|
|
1057
|
+
/* There's no overap between broker racks and consumer racks, \
|
|
1058
|
+
* since num_broker_racks = 3, they'll be picked from a,b,c \
|
|
1059
|
+
* and consumer racks are d,e,f. */ \
|
|
1060
|
+
setupRackAwareAssignment(rk, rkas, members, member_cnt, 3, 3, \
|
|
1061
|
+
topic_cnt, topics, partitions, \
|
|
1062
|
+
subscriptions_count, subscriptions, \
|
|
1063
|
+
RACKS_FINAL); \
|
|
1064
|
+
verifyMultipleAssignment(members, member_cnt, __VA_ARGS__); \
|
|
1065
|
+
for (idx = 0; idx < member_cnt; idx++) \
|
|
1066
|
+
rd_kafka_group_member_clear(&members[idx]); \
|
|
1067
|
+
/* There's no overap between broker racks and consumer racks, \
|
|
1068
|
+
* since num_broker_racks = 3, they'll be picked from a,b,c \
|
|
1069
|
+
* and consumer racks are d,e,NULL. */ \
|
|
1070
|
+
setupRackAwareAssignment(rk, rkas, members, member_cnt, 3, 3, \
|
|
1071
|
+
topic_cnt, topics, partitions, \
|
|
1072
|
+
subscriptions_count, subscriptions, \
|
|
1073
|
+
RACKS_ONE_NULL); \
|
|
1074
|
+
verifyMultipleAssignment(members, member_cnt, __VA_ARGS__); \
|
|
1075
|
+
for (idx = 0; idx < member_cnt; idx++) \
|
|
1076
|
+
rd_kafka_group_member_clear(&members[idx]); \
|
|
1077
|
+
} while (0)
|
|
1078
|
+
|
|
1079
|
+
static int ut_testRackAwareAssignmentWithUniformSubscription(
|
|
1080
|
+
rd_kafka_t *rk,
|
|
1081
|
+
const rd_kafka_assignor_t *rkas,
|
|
1082
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) {
|
|
1083
|
+
char *topics[] = {"t1", "t2", "t3"};
|
|
1084
|
+
int partitions[] = {6, 7, 2};
|
|
1085
|
+
rd_kafka_metadata_t *metadata;
|
|
1086
|
+
rd_kafka_group_member_t members[3];
|
|
1087
|
+
size_t i = 0;
|
|
1088
|
+
int subscriptions_count[] = {3, 3, 3};
|
|
1089
|
+
char **subscriptions[] = {topics, topics, topics};
|
|
1090
|
+
|
|
1091
|
+
if (parametrization !=
|
|
1092
|
+
RD_KAFKA_RANGE_ASSIGNOR_UT_BROKER_AND_CONSUMER_RACK) {
|
|
1093
|
+
RD_UT_PASS();
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
verifyNonRackAwareAssignment(
|
|
1097
|
+
rk, rkas, members, RD_ARRAYSIZE(members), RD_ARRAYSIZE(topics),
|
|
1098
|
+
topics, partitions, subscriptions_count, subscriptions,
|
|
1099
|
+
/* consumer1*/
|
|
1100
|
+
"t1", 0, "t1", 1, "t2", 0, "t2", 1, "t2", 2, "t3", 0, NULL,
|
|
1101
|
+
/* consumer2 */
|
|
1102
|
+
"t1", 2, "t1", 3, "t2", 3, "t2", 4, "t3", 1, NULL,
|
|
1103
|
+
/* consumer3 */
|
|
1104
|
+
"t1", 4, "t1", 5, "t2", 5, "t2", 6, NULL);
|
|
1105
|
+
|
|
1106
|
+
/* Verify best-effort rack-aware assignment for lower replication factor
|
|
1107
|
+
* where racks have a subset of partitions.*/
|
|
1108
|
+
setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 1,
|
|
1109
|
+
3, RD_ARRAYSIZE(topics), topics, partitions,
|
|
1110
|
+
subscriptions_count, subscriptions,
|
|
1111
|
+
RACKS_INITIAL, &metadata);
|
|
1112
|
+
verifyMultipleAssignment(
|
|
1113
|
+
members, RD_ARRAYSIZE(members),
|
|
1114
|
+
/* consumer1 */
|
|
1115
|
+
"t1", 0, "t1", 3, "t2", 0, "t2", 3, "t2", 6, NULL,
|
|
1116
|
+
/* consumer2 */
|
|
1117
|
+
"t1", 1, "t1", 4, "t2", 1, "t2", 4, "t3", 0, NULL,
|
|
1118
|
+
/* consumer3 */
|
|
1119
|
+
"t1", 2, "t1", 5, "t2", 2, "t2", 5, "t3", 1, NULL);
|
|
1120
|
+
verifyNumPartitionsWithRackMismatch(metadata, members,
|
|
1121
|
+
RD_ARRAYSIZE(members), 0);
|
|
1122
|
+
|
|
1123
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1124
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1125
|
+
ut_destroy_metadata(metadata);
|
|
1126
|
+
|
|
1127
|
+
setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 2,
|
|
1128
|
+
3, RD_ARRAYSIZE(topics), topics, partitions,
|
|
1129
|
+
subscriptions_count, subscriptions,
|
|
1130
|
+
RACKS_INITIAL, &metadata);
|
|
1131
|
+
verifyMultipleAssignment(
|
|
1132
|
+
members, RD_ARRAYSIZE(members),
|
|
1133
|
+
/*consumer1*/
|
|
1134
|
+
"t1", 0, "t1", 2, "t2", 0, "t2", 2, "t2", 3, "t3", 1, NULL,
|
|
1135
|
+
/* consumer2 */
|
|
1136
|
+
"t1", 1, "t1", 3, "t2", 1, "t2", 4, "t3", 0, NULL,
|
|
1137
|
+
/* consumer 3*/
|
|
1138
|
+
"t1", 4, "t1", 5, "t2", 5, "t2", 6, NULL);
|
|
1139
|
+
verifyNumPartitionsWithRackMismatch(metadata, members,
|
|
1140
|
+
RD_ARRAYSIZE(members), 1);
|
|
1141
|
+
|
|
1142
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1143
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1144
|
+
ut_destroy_metadata(metadata);
|
|
1145
|
+
|
|
1146
|
+
|
|
1147
|
+
/* One consumer on a rack with no partitions. */
|
|
1148
|
+
setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 3,
|
|
1149
|
+
2, RD_ARRAYSIZE(topics), topics, partitions,
|
|
1150
|
+
subscriptions_count, subscriptions,
|
|
1151
|
+
RACKS_INITIAL, &metadata);
|
|
1152
|
+
verifyMultipleAssignment(members, RD_ARRAYSIZE(members),
|
|
1153
|
+
/* consumer1 */ "t1", 0, "t1", 1, "t2", 0,
|
|
1154
|
+
"t2", 1, "t2", 2, "t3", 0, NULL,
|
|
1155
|
+
/* consumer2 */
|
|
1156
|
+
"t1", 2, "t1", 3, "t2", 3, "t2", 4, "t3", 1,
|
|
1157
|
+
NULL,
|
|
1158
|
+
/* consumer3 */
|
|
1159
|
+
"t1", 4, "t1", 5, "t2", 5, "t2", 6, NULL);
|
|
1160
|
+
verifyNumPartitionsWithRackMismatch(metadata, members,
|
|
1161
|
+
RD_ARRAYSIZE(members), 4);
|
|
1162
|
+
|
|
1163
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1164
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1165
|
+
ut_destroy_metadata(metadata);
|
|
1166
|
+
|
|
1167
|
+
RD_UT_PASS();
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
static int ut_testRackAwareAssignmentWithNonEqualSubscription(
|
|
1171
|
+
rd_kafka_t *rk,
|
|
1172
|
+
const rd_kafka_assignor_t *rkas,
|
|
1173
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) {
|
|
1174
|
+
rd_kafka_metadata_t *metadata;
|
|
1175
|
+
char *topics[] = {"t1", "t2", "t3"};
|
|
1176
|
+
int partitions[] = {6, 7, 2};
|
|
1177
|
+
rd_kafka_group_member_t members[3];
|
|
1178
|
+
size_t i = 0;
|
|
1179
|
+
int subscriptions_count[] = {3, 3, 2};
|
|
1180
|
+
char *subscription13[] = {"t1", "t3"};
|
|
1181
|
+
char **subscriptions[] = {topics, topics, subscription13};
|
|
1182
|
+
|
|
1183
|
+
if (parametrization !=
|
|
1184
|
+
RD_KAFKA_RANGE_ASSIGNOR_UT_BROKER_AND_CONSUMER_RACK) {
|
|
1185
|
+
RD_UT_PASS();
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
verifyNonRackAwareAssignment(
|
|
1189
|
+
rk, rkas, members, RD_ARRAYSIZE(members), RD_ARRAYSIZE(topics),
|
|
1190
|
+
topics, partitions, subscriptions_count, subscriptions,
|
|
1191
|
+
/* consumer1*/
|
|
1192
|
+
"t1", 0, "t1", 1, "t2", 0, "t2", 1, "t2", 2, "t2", 3, "t3", 0, NULL,
|
|
1193
|
+
/* consumer2 */
|
|
1194
|
+
"t1", 2, "t1", 3, "t2", 4, "t2", 5, "t2", 6, "t3", 1, NULL,
|
|
1195
|
+
/* consumer3 */
|
|
1196
|
+
"t1", 4, "t1", 5, NULL);
|
|
1197
|
+
|
|
1198
|
+
/* Verify best-effort rack-aware assignment for lower replication factor
|
|
1199
|
+
* where racks have a subset of partitions. */
|
|
1200
|
+
setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 1,
|
|
1201
|
+
3, RD_ARRAYSIZE(topics), topics, partitions,
|
|
1202
|
+
subscriptions_count, subscriptions,
|
|
1203
|
+
RACKS_INITIAL, &metadata);
|
|
1204
|
+
verifyMultipleAssignment(
|
|
1205
|
+
members, RD_ARRAYSIZE(members),
|
|
1206
|
+
/* consumer1 */
|
|
1207
|
+
"t1", 0, "t1", 3, "t2", 0, "t2", 2, "t2", 3, "t2", 6, NULL,
|
|
1208
|
+
/* consumer2 */
|
|
1209
|
+
"t1", 1, "t1", 4, "t2", 1, "t2", 4, "t2", 5, "t3", 0, NULL,
|
|
1210
|
+
/* consumer3 */
|
|
1211
|
+
"t1", 2, "t1", 5, "t3", 1, NULL);
|
|
1212
|
+
verifyNumPartitionsWithRackMismatch(metadata, members,
|
|
1213
|
+
RD_ARRAYSIZE(members), 2);
|
|
1214
|
+
|
|
1215
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1216
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1217
|
+
ut_destroy_metadata(metadata);
|
|
1218
|
+
|
|
1219
|
+
setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 2,
|
|
1220
|
+
3, RD_ARRAYSIZE(topics), topics, partitions,
|
|
1221
|
+
subscriptions_count, subscriptions,
|
|
1222
|
+
RACKS_INITIAL, &metadata);
|
|
1223
|
+
verifyMultipleAssignment(
|
|
1224
|
+
members, RD_ARRAYSIZE(members),
|
|
1225
|
+
/* consumer1 */
|
|
1226
|
+
"t1", 0, "t1", 2, "t2", 0, "t2", 2, "t2", 3, "t2", 5, "t3", 1, NULL,
|
|
1227
|
+
/* consumer2 */
|
|
1228
|
+
"t1", 1, "t1", 3, "t2", 1, "t2", 4, "t2", 6, "t3", 0, NULL,
|
|
1229
|
+
/* consumer3 */
|
|
1230
|
+
"t1", 4, "t1", 5, NULL);
|
|
1231
|
+
verifyNumPartitionsWithRackMismatch(metadata, members,
|
|
1232
|
+
RD_ARRAYSIZE(members), 0);
|
|
1233
|
+
|
|
1234
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1235
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1236
|
+
ut_destroy_metadata(metadata);
|
|
1237
|
+
|
|
1238
|
+
/* One consumer on a rack with no partitions */
|
|
1239
|
+
setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 3,
|
|
1240
|
+
2, RD_ARRAYSIZE(topics), topics, partitions,
|
|
1241
|
+
subscriptions_count, subscriptions,
|
|
1242
|
+
RACKS_INITIAL, &metadata);
|
|
1243
|
+
verifyMultipleAssignment(
|
|
1244
|
+
members, RD_ARRAYSIZE(members),
|
|
1245
|
+
/* consumer1 */
|
|
1246
|
+
"t1", 0, "t1", 1, "t2", 0, "t2", 1, "t2", 2, "t2", 3, "t3", 0, NULL,
|
|
1247
|
+
/* consumer2 */
|
|
1248
|
+
"t1", 2, "t1", 3, "t2", 4, "t2", 5, "t2", 6, "t3", 1, NULL,
|
|
1249
|
+
/* consumer3 */
|
|
1250
|
+
"t1", 4, "t1", 5, NULL);
|
|
1251
|
+
verifyNumPartitionsWithRackMismatch(metadata, members,
|
|
1252
|
+
RD_ARRAYSIZE(members), 2);
|
|
1253
|
+
|
|
1254
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1255
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1256
|
+
ut_destroy_metadata(metadata);
|
|
1257
|
+
|
|
1258
|
+
RD_UT_PASS();
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
static int ut_testRackAwareAssignmentWithUniformPartitions(
|
|
1262
|
+
rd_kafka_t *rk,
|
|
1263
|
+
const rd_kafka_assignor_t *rkas,
|
|
1264
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) {
|
|
1265
|
+
char *topics[] = {"t1", "t2", "t3"};
|
|
1266
|
+
int partitions[] = {5, 5, 5};
|
|
1267
|
+
int partitions_mismatch[] = {10, 5, 3};
|
|
1268
|
+
rd_kafka_group_member_t members[3];
|
|
1269
|
+
size_t i = 0;
|
|
1270
|
+
int replication_factor = 0;
|
|
1271
|
+
int subscriptions_count[] = {3, 3, 3};
|
|
1272
|
+
char **subscriptions[] = {topics, topics, topics};
|
|
1273
|
+
|
|
1274
|
+
if (parametrization !=
|
|
1275
|
+
RD_KAFKA_RANGE_ASSIGNOR_UT_BROKER_AND_CONSUMER_RACK) {
|
|
1276
|
+
RD_UT_PASS();
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
/* Verify combinations where rack-aware logic is not used. */
|
|
1280
|
+
verifyNonRackAwareAssignment(
|
|
1281
|
+
rk, rkas, members, RD_ARRAYSIZE(members), RD_ARRAYSIZE(topics),
|
|
1282
|
+
topics, partitions, subscriptions_count, subscriptions,
|
|
1283
|
+
/* consumer1*/
|
|
1284
|
+
"t1", 0, "t1", 1, "t2", 0, "t2", 1, "t3", 0, "t3", 1, NULL,
|
|
1285
|
+
/* consumer2 */
|
|
1286
|
+
"t1", 2, "t1", 3, "t2", 2, "t2", 3, "t3", 2, "t3", 3, NULL,
|
|
1287
|
+
/* consumer3 */
|
|
1288
|
+
"t1", 4, "t2", 4, "t3", 4, NULL);
|
|
1289
|
+
|
|
1290
|
+
/* Verify that co-partitioning is prioritized over rack-alignment for
|
|
1291
|
+
* topics with equal subscriptions */
|
|
1292
|
+
for (replication_factor = 1; replication_factor <= 3;
|
|
1293
|
+
replication_factor++) {
|
|
1294
|
+
rd_kafka_metadata_t *metadata = NULL;
|
|
1295
|
+
setupRackAwareAssignment0(
|
|
1296
|
+
rk, rkas, members, RD_ARRAYSIZE(members),
|
|
1297
|
+
replication_factor, replication_factor < 3 ? 3 : 2,
|
|
1298
|
+
RD_ARRAYSIZE(topics), topics, partitions,
|
|
1299
|
+
subscriptions_count, subscriptions, RACKS_INITIAL,
|
|
1300
|
+
&metadata);
|
|
1301
|
+
verifyMultipleAssignment(
|
|
1302
|
+
members, RD_ARRAYSIZE(members),
|
|
1303
|
+
/* consumer1*/
|
|
1304
|
+
"t1", 0, "t1", 1, "t2", 0, "t2", 1, "t3", 0, "t3", 1, NULL,
|
|
1305
|
+
/* consumer2 */
|
|
1306
|
+
"t1", 2, "t1", 3, "t2", 2, "t2", 3, "t3", 2, "t3", 3, NULL,
|
|
1307
|
+
/* consumer3 */
|
|
1308
|
+
"t1", 4, "t2", 4, "t3", 4, NULL);
|
|
1309
|
+
verifyNumPartitionsWithRackMismatch(
|
|
1310
|
+
metadata, members, RD_ARRAYSIZE(members),
|
|
1311
|
+
partitions_mismatch[replication_factor - 1]);
|
|
1312
|
+
|
|
1313
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1314
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1315
|
+
ut_destroy_metadata(metadata);
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1318
|
+
RD_UT_PASS();
|
|
1319
|
+
}
|
|
1320
|
+
|
|
1321
|
+
static int ut_testRackAwareAssignmentWithUniformPartitionsNonEqualSubscription(
|
|
1322
|
+
rd_kafka_t *rk,
|
|
1323
|
+
const rd_kafka_assignor_t *rkas,
|
|
1324
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) {
|
|
1325
|
+
rd_kafka_metadata_t *metadata = NULL;
|
|
1326
|
+
char *topics[] = {"t1", "t2", "t3"};
|
|
1327
|
+
int partitions[] = {5, 5, 5};
|
|
1328
|
+
rd_kafka_group_member_t members[3];
|
|
1329
|
+
size_t i = 0;
|
|
1330
|
+
int subscriptions_count[] = {3, 3, 2};
|
|
1331
|
+
char *subscription13[] = {"t1", "t3"};
|
|
1332
|
+
char **subscriptions[] = {topics, topics, subscription13};
|
|
1333
|
+
|
|
1334
|
+
if (parametrization !=
|
|
1335
|
+
RD_KAFKA_RANGE_ASSIGNOR_UT_BROKER_AND_CONSUMER_RACK) {
|
|
1336
|
+
RD_UT_PASS();
|
|
1337
|
+
}
|
|
1338
|
+
|
|
1339
|
+
/* Verify combinations where rack-aware logic is not used. */
|
|
1340
|
+
verifyNonRackAwareAssignment(
|
|
1341
|
+
rk, rkas, members, RD_ARRAYSIZE(members), RD_ARRAYSIZE(topics),
|
|
1342
|
+
topics, partitions, subscriptions_count, subscriptions,
|
|
1343
|
+
/* consumer1*/
|
|
1344
|
+
"t1", 0, "t1", 1, "t2", 0, "t2", 1, "t2", 2, "t3", 0, "t3", 1, NULL,
|
|
1345
|
+
/* consumer2 */
|
|
1346
|
+
"t1", 2, "t1", 3, "t2", 3, "t2", 4, "t3", 2, "t3", 3, NULL,
|
|
1347
|
+
/* consumer3 */
|
|
1348
|
+
"t1", 4, "t3", 4, NULL);
|
|
1349
|
+
|
|
1350
|
+
/* Verify that co-partitioning is prioritized over rack-alignment for
|
|
1351
|
+
* topics with equal subscriptions */
|
|
1352
|
+
setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 1,
|
|
1353
|
+
3, RD_ARRAYSIZE(topics), topics, partitions,
|
|
1354
|
+
subscriptions_count, subscriptions,
|
|
1355
|
+
RACKS_INITIAL, &metadata);
|
|
1356
|
+
verifyMultipleAssignment(
|
|
1357
|
+
members, RD_ARRAYSIZE(members),
|
|
1358
|
+
/* consumer1 */
|
|
1359
|
+
"t1", 0, "t1", 1, "t2", 0, "t2", 1, "t2", 4, "t3", 0, "t3", 1, NULL,
|
|
1360
|
+
/* consumer2 */
|
|
1361
|
+
"t1", 2, "t1", 3, "t2", 2, "t2", 3, "t3", 2, "t3", 3, NULL,
|
|
1362
|
+
/* consumer3 */
|
|
1363
|
+
"t1", 4, "t3", 4, NULL);
|
|
1364
|
+
verifyNumPartitionsWithRackMismatch(metadata, members,
|
|
1365
|
+
RD_ARRAYSIZE(members), 9);
|
|
1366
|
+
|
|
1367
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1368
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1369
|
+
ut_destroy_metadata(metadata);
|
|
1370
|
+
|
|
1371
|
+
|
|
1372
|
+
setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 2,
|
|
1373
|
+
3, RD_ARRAYSIZE(topics), topics, partitions,
|
|
1374
|
+
subscriptions_count, subscriptions,
|
|
1375
|
+
RACKS_INITIAL, &metadata);
|
|
1376
|
+
verifyMultipleAssignment(
|
|
1377
|
+
members, RD_ARRAYSIZE(members),
|
|
1378
|
+
/* consumer1 */
|
|
1379
|
+
"t1", 2, "t2", 0, "t2", 1, "t2", 3, "t3", 2, NULL,
|
|
1380
|
+
/* consumer2 */
|
|
1381
|
+
"t1", 0, "t1", 3, "t2", 2, "t2", 4, "t3", 0, "t3", 3, NULL,
|
|
1382
|
+
/* consumer3 */
|
|
1383
|
+
"t1", 1, "t1", 4, "t3", 1, "t3", 4, NULL);
|
|
1384
|
+
verifyNumPartitionsWithRackMismatch(metadata, members,
|
|
1385
|
+
RD_ARRAYSIZE(members), 0);
|
|
1386
|
+
|
|
1387
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1388
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1389
|
+
ut_destroy_metadata(metadata);
|
|
1390
|
+
|
|
1391
|
+
/* One consumer on a rack with no partitions */
|
|
1392
|
+
setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 3,
|
|
1393
|
+
2, RD_ARRAYSIZE(topics), topics, partitions,
|
|
1394
|
+
subscriptions_count, subscriptions,
|
|
1395
|
+
RACKS_INITIAL, &metadata);
|
|
1396
|
+
verifyMultipleAssignment(
|
|
1397
|
+
members, RD_ARRAYSIZE(members),
|
|
1398
|
+
/* consumer1 */
|
|
1399
|
+
"t1", 0, "t1", 1, "t2", 0, "t2", 1, "t2", 2, "t3", 0, "t3", 1, NULL,
|
|
1400
|
+
/* consumer2 */
|
|
1401
|
+
"t1", 2, "t1", 3, "t2", 3, "t2", 4, "t3", 2, "t3", 3, NULL,
|
|
1402
|
+
/* consumer3 */
|
|
1403
|
+
"t1", 4, "t3", 4, NULL);
|
|
1404
|
+
verifyNumPartitionsWithRackMismatch(metadata, members,
|
|
1405
|
+
RD_ARRAYSIZE(members), 2);
|
|
1406
|
+
|
|
1407
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1408
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1409
|
+
ut_destroy_metadata(metadata);
|
|
1410
|
+
|
|
1411
|
+
RD_UT_PASS();
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1414
|
+
static int ut_testRackAwareAssignmentWithCoPartitioning0(
|
|
1415
|
+
rd_kafka_t *rk,
|
|
1416
|
+
const rd_kafka_assignor_t *rkas,
|
|
1417
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) {
|
|
1418
|
+
rd_kafka_metadata_t *metadata = NULL;
|
|
1419
|
+
char *topics[] = {"t1", "t2", "t3", "t4"};
|
|
1420
|
+
int partitions[] = {6, 6, 2, 2};
|
|
1421
|
+
rd_kafka_group_member_t members[4];
|
|
1422
|
+
size_t i = 0;
|
|
1423
|
+
int subscriptions_count[] = {2, 2, 2, 2};
|
|
1424
|
+
char *subscription12[] = {"t1", "t2"};
|
|
1425
|
+
char *subscription34[] = {"t3", "t4"};
|
|
1426
|
+
char **subscriptions[] = {subscription12, subscription12,
|
|
1427
|
+
subscription34, subscription34};
|
|
1428
|
+
int racks[] = {0, 1, 1, 0};
|
|
1429
|
+
|
|
1430
|
+
if (parametrization !=
|
|
1431
|
+
RD_KAFKA_RANGE_ASSIGNOR_UT_BROKER_AND_CONSUMER_RACK) {
|
|
1432
|
+
RD_UT_PASS();
|
|
1433
|
+
}
|
|
1434
|
+
|
|
1435
|
+
setupRackAwareAssignment(rk, rkas, members, RD_ARRAYSIZE(members), 3, 2,
|
|
1436
|
+
RD_ARRAYSIZE(topics), topics, partitions,
|
|
1437
|
+
subscriptions_count, subscriptions, racks);
|
|
1438
|
+
verifyMultipleAssignment(
|
|
1439
|
+
members, RD_ARRAYSIZE(members),
|
|
1440
|
+
/* consumer1 */
|
|
1441
|
+
"t1", 0, "t1", 1, "t1", 2, "t2", 0, "t2", 1, "t2", 2, NULL,
|
|
1442
|
+
/* consumer2 */
|
|
1443
|
+
"t1", 3, "t1", 4, "t1", 5, "t2", 3, "t2", 4, "t2", 5, NULL,
|
|
1444
|
+
/* consumer3 */
|
|
1445
|
+
"t3", 0, "t4", 0, NULL,
|
|
1446
|
+
/* consumer4 */
|
|
1447
|
+
"t3", 1, "t4", 1, NULL);
|
|
1448
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1449
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1450
|
+
|
|
1451
|
+
setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 2,
|
|
1452
|
+
2, RD_ARRAYSIZE(topics), topics, partitions,
|
|
1453
|
+
subscriptions_count, subscriptions, racks,
|
|
1454
|
+
&metadata);
|
|
1455
|
+
verifyMultipleAssignment(
|
|
1456
|
+
members, RD_ARRAYSIZE(members),
|
|
1457
|
+
/* consumer1 */
|
|
1458
|
+
"t1", 0, "t1", 1, "t1", 2, "t2", 0, "t2", 1, "t2", 2, NULL,
|
|
1459
|
+
/* consumer2 */
|
|
1460
|
+
"t1", 3, "t1", 4, "t1", 5, "t2", 3, "t2", 4, "t2", 5, NULL,
|
|
1461
|
+
/* consumer3 */
|
|
1462
|
+
"t3", 0, "t4", 0, NULL,
|
|
1463
|
+
/* consumer4 */
|
|
1464
|
+
"t3", 1, "t4", 1, NULL);
|
|
1465
|
+
verifyNumPartitionsWithRackMismatch(metadata, members,
|
|
1466
|
+
RD_ARRAYSIZE(members), 0);
|
|
1467
|
+
|
|
1468
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1469
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1470
|
+
ut_destroy_metadata(metadata);
|
|
1471
|
+
|
|
1472
|
+
setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 1,
|
|
1473
|
+
2, RD_ARRAYSIZE(topics), topics, partitions,
|
|
1474
|
+
subscriptions_count, subscriptions, racks,
|
|
1475
|
+
&metadata);
|
|
1476
|
+
verifyMultipleAssignment(
|
|
1477
|
+
members, RD_ARRAYSIZE(members),
|
|
1478
|
+
/* consumer1 */
|
|
1479
|
+
"t1", 0, "t1", 2, "t1", 4, "t2", 0, "t2", 2, "t2", 4, NULL,
|
|
1480
|
+
/* consumer2 */
|
|
1481
|
+
"t1", 1, "t1", 3, "t1", 5, "t2", 1, "t2", 3, "t2", 5, NULL,
|
|
1482
|
+
/* consumer3 */
|
|
1483
|
+
"t3", 1, "t4", 1, NULL,
|
|
1484
|
+
/* consumer4 */
|
|
1485
|
+
"t3", 0, "t4", 0, NULL);
|
|
1486
|
+
verifyNumPartitionsWithRackMismatch(metadata, members,
|
|
1487
|
+
RD_ARRAYSIZE(members), 0);
|
|
1488
|
+
|
|
1489
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1490
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1491
|
+
ut_destroy_metadata(metadata);
|
|
1492
|
+
|
|
1493
|
+
RD_UT_PASS();
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
static int ut_testRackAwareAssignmentWithCoPartitioning1(
|
|
1497
|
+
rd_kafka_t *rk,
|
|
1498
|
+
const rd_kafka_assignor_t *rkas,
|
|
1499
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) {
|
|
1500
|
+
rd_kafka_metadata_t *metadata = NULL;
|
|
1501
|
+
char *topics[] = {"t1", "t2", "t3", "t4"};
|
|
1502
|
+
int partitions[] = {6, 6, 2, 2};
|
|
1503
|
+
rd_kafka_group_member_t members[4];
|
|
1504
|
+
size_t i = 0;
|
|
1505
|
+
int subscriptions_count[] = {4, 4, 4, 4};
|
|
1506
|
+
char **subscriptions[] = {topics, topics, topics, topics};
|
|
1507
|
+
int racks[] = {0, 1, 1, 0};
|
|
1508
|
+
|
|
1509
|
+
if (parametrization !=
|
|
1510
|
+
RD_KAFKA_RANGE_ASSIGNOR_UT_BROKER_AND_CONSUMER_RACK) {
|
|
1511
|
+
RD_UT_PASS();
|
|
1512
|
+
}
|
|
1513
|
+
|
|
1514
|
+
setupRackAwareAssignment(rk, rkas, members, RD_ARRAYSIZE(members), 3, 2,
|
|
1515
|
+
RD_ARRAYSIZE(topics), topics, partitions,
|
|
1516
|
+
subscriptions_count, subscriptions, racks);
|
|
1517
|
+
verifyMultipleAssignment(
|
|
1518
|
+
members, RD_ARRAYSIZE(members),
|
|
1519
|
+
/* consumer1 */
|
|
1520
|
+
"t1", 0, "t1", 1, "t2", 0, "t2", 1, "t3", 0, "t4", 0, NULL,
|
|
1521
|
+
/* consumer2 */
|
|
1522
|
+
"t1", 2, "t1", 3, "t2", 2, "t2", 3, "t3", 1, "t4", 1, NULL,
|
|
1523
|
+
/* consumer3 */
|
|
1524
|
+
"t1", 4, "t2", 4, NULL,
|
|
1525
|
+
/* consumer4 */
|
|
1526
|
+
"t1", 5, "t2", 5, NULL);
|
|
1527
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1528
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1529
|
+
|
|
1530
|
+
setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 2,
|
|
1531
|
+
2, RD_ARRAYSIZE(topics), topics, partitions,
|
|
1532
|
+
subscriptions_count, subscriptions, racks,
|
|
1533
|
+
&metadata);
|
|
1534
|
+
verifyMultipleAssignment(
|
|
1535
|
+
members, RD_ARRAYSIZE(members),
|
|
1536
|
+
/* consumer1 */
|
|
1537
|
+
"t1", 0, "t1", 1, "t2", 0, "t2", 1, "t3", 0, "t4", 0, NULL,
|
|
1538
|
+
/* consumer2 */
|
|
1539
|
+
"t1", 2, "t1", 3, "t2", 2, "t2", 3, "t3", 1, "t4", 1, NULL,
|
|
1540
|
+
/* consumer3 */
|
|
1541
|
+
"t1", 4, "t2", 4, NULL,
|
|
1542
|
+
/* consumer4 */
|
|
1543
|
+
"t1", 5, "t2", 5, NULL);
|
|
1544
|
+
verifyNumPartitionsWithRackMismatch(metadata, members,
|
|
1545
|
+
RD_ARRAYSIZE(members), 0);
|
|
1546
|
+
|
|
1547
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1548
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1549
|
+
ut_destroy_metadata(metadata);
|
|
1550
|
+
|
|
1551
|
+
|
|
1552
|
+
setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 1,
|
|
1553
|
+
2, RD_ARRAYSIZE(topics), topics, partitions,
|
|
1554
|
+
subscriptions_count, subscriptions, racks,
|
|
1555
|
+
&metadata);
|
|
1556
|
+
verifyMultipleAssignment(
|
|
1557
|
+
members, RD_ARRAYSIZE(members),
|
|
1558
|
+
/* consumer1 */
|
|
1559
|
+
"t1", 0, "t1", 2, "t2", 0, "t2", 2, "t3", 0, "t4", 0, NULL,
|
|
1560
|
+
/* consumer2 */
|
|
1561
|
+
"t1", 1, "t1", 3, "t2", 1, "t2", 3, "t3", 1, "t4", 1, NULL,
|
|
1562
|
+
/* consumer3 */
|
|
1563
|
+
"t1", 5, "t2", 5, NULL,
|
|
1564
|
+
/* consumer4 */
|
|
1565
|
+
"t1", 4, "t2", 4, NULL);
|
|
1566
|
+
verifyNumPartitionsWithRackMismatch(metadata, members,
|
|
1567
|
+
RD_ARRAYSIZE(members), 0);
|
|
1568
|
+
|
|
1569
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1570
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1571
|
+
ut_destroy_metadata(metadata);
|
|
1572
|
+
|
|
1573
|
+
|
|
1574
|
+
setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 1,
|
|
1575
|
+
3, RD_ARRAYSIZE(topics), topics, partitions,
|
|
1576
|
+
subscriptions_count, subscriptions, racks,
|
|
1577
|
+
&metadata);
|
|
1578
|
+
verifyMultipleAssignment(
|
|
1579
|
+
members, RD_ARRAYSIZE(members),
|
|
1580
|
+
/* consumer1 */
|
|
1581
|
+
"t1", 0, "t1", 3, "t2", 0, "t2", 3, "t3", 0, "t4", 0, NULL,
|
|
1582
|
+
/* consumer2 */
|
|
1583
|
+
"t1", 1, "t1", 4, "t2", 1, "t2", 4, "t3", 1, "t4", 1, NULL,
|
|
1584
|
+
/* consumer3 */
|
|
1585
|
+
"t1", 2, "t2", 2, NULL,
|
|
1586
|
+
/* consumer4 */
|
|
1587
|
+
"t1", 5, "t2", 5, NULL);
|
|
1588
|
+
verifyNumPartitionsWithRackMismatch(metadata, members,
|
|
1589
|
+
RD_ARRAYSIZE(members), 6);
|
|
1590
|
+
|
|
1591
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1592
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1593
|
+
ut_destroy_metadata(metadata);
|
|
1594
|
+
|
|
1595
|
+
RD_UT_PASS();
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
static int ut_testCoPartitionedAssignmentWithSameSubscription(
|
|
1599
|
+
rd_kafka_t *rk,
|
|
1600
|
+
const rd_kafka_assignor_t *rkas,
|
|
1601
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) {
|
|
1602
|
+
rd_kafka_metadata_t *metadata = NULL;
|
|
1603
|
+
char *topics[] = {"t1", "t2", "t3", "t4", "t5", "t6"};
|
|
1604
|
+
int partitions[] = {6, 6, 2, 2, 4, 4};
|
|
1605
|
+
rd_kafka_group_member_t members[3];
|
|
1606
|
+
size_t i = 0;
|
|
1607
|
+
int subscriptions_count[] = {6, 6, 6};
|
|
1608
|
+
char **subscriptions[] = {topics, topics, topics};
|
|
1609
|
+
|
|
1610
|
+
if (parametrization !=
|
|
1611
|
+
RD_KAFKA_RANGE_ASSIGNOR_UT_BROKER_AND_CONSUMER_RACK) {
|
|
1612
|
+
RD_UT_PASS();
|
|
1613
|
+
}
|
|
1614
|
+
|
|
1615
|
+
setupRackAwareAssignment(rk, rkas, members, RD_ARRAYSIZE(members), 3, 0,
|
|
1616
|
+
RD_ARRAYSIZE(topics), topics, partitions,
|
|
1617
|
+
subscriptions_count, subscriptions,
|
|
1618
|
+
RACKS_INITIAL);
|
|
1619
|
+
verifyMultipleAssignment(
|
|
1620
|
+
members, RD_ARRAYSIZE(members),
|
|
1621
|
+
/* consumer1 */
|
|
1622
|
+
"t1", 0, "t1", 1, "t2", 0, "t2", 1, "t3", 0, "t4", 0, "t5", 0, "t5",
|
|
1623
|
+
1, "t6", 0, "t6", 1, NULL,
|
|
1624
|
+
/* consumer2 */
|
|
1625
|
+
"t1", 2, "t1", 3, "t2", 2, "t2", 3, "t3", 1, "t4", 1, "t5", 2, "t6",
|
|
1626
|
+
2, NULL,
|
|
1627
|
+
/* consumer3 */
|
|
1628
|
+
"t1", 4, "t1", 5, "t2", 4, "t2", 5, "t5", 3, "t6", 3, NULL);
|
|
1629
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1630
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1631
|
+
|
|
1632
|
+
setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 3,
|
|
1633
|
+
3, RD_ARRAYSIZE(topics), topics, partitions,
|
|
1634
|
+
subscriptions_count, subscriptions,
|
|
1635
|
+
RACKS_INITIAL, &metadata);
|
|
1636
|
+
verifyMultipleAssignment(
|
|
1637
|
+
members, RD_ARRAYSIZE(members),
|
|
1638
|
+
/* consumer1 */
|
|
1639
|
+
"t1", 0, "t1", 1, "t2", 0, "t2", 1, "t3", 0, "t4", 0, "t5", 0, "t5",
|
|
1640
|
+
1, "t6", 0, "t6", 1, NULL,
|
|
1641
|
+
/* consumer2 */
|
|
1642
|
+
"t1", 2, "t1", 3, "t2", 2, "t2", 3, "t3", 1, "t4", 1, "t5", 2, "t6",
|
|
1643
|
+
2, NULL,
|
|
1644
|
+
/* consumer3 */
|
|
1645
|
+
"t1", 4, "t1", 5, "t2", 4, "t2", 5, "t5", 3, "t6", 3, NULL);
|
|
1646
|
+
verifyNumPartitionsWithRackMismatch(metadata, members,
|
|
1647
|
+
RD_ARRAYSIZE(members), 0);
|
|
1648
|
+
for (i = 0; i < RD_ARRAYSIZE(members); i++)
|
|
1649
|
+
rd_kafka_group_member_clear(&members[i]);
|
|
1650
|
+
ut_destroy_metadata(metadata);
|
|
1651
|
+
|
|
1652
|
+
RD_UT_PASS();
|
|
1653
|
+
}
|
|
1654
|
+
|
|
1655
|
+
|
|
1656
|
+
static int rd_kafka_range_assignor_unittest(void) {
|
|
1657
|
+
rd_kafka_conf_t *conf;
|
|
1658
|
+
rd_kafka_t *rk;
|
|
1659
|
+
int fails = 0;
|
|
1660
|
+
char errstr[256];
|
|
1661
|
+
rd_kafka_assignor_t *rkas;
|
|
1662
|
+
size_t i;
|
|
1663
|
+
|
|
1664
|
+
conf = rd_kafka_conf_new();
|
|
1665
|
+
if (rd_kafka_conf_set(conf, "group.id", "test", errstr,
|
|
1666
|
+
sizeof(errstr)) ||
|
|
1667
|
+
rd_kafka_conf_set(conf, "partition.assignment.strategy", "range",
|
|
1668
|
+
errstr, sizeof(errstr)))
|
|
1669
|
+
RD_UT_FAIL("range assignor conf failed: %s", errstr);
|
|
1670
|
+
|
|
1671
|
+
rd_kafka_conf_set(conf, "debug", rd_getenv("TEST_DEBUG", NULL), NULL,
|
|
1672
|
+
0);
|
|
1673
|
+
|
|
1674
|
+
rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr));
|
|
1675
|
+
RD_UT_ASSERT(rk, "range assignor client instantiation failed: %s",
|
|
1676
|
+
errstr);
|
|
1677
|
+
rkas = rd_kafka_assignor_find(rk, "range");
|
|
1678
|
+
RD_UT_ASSERT(rkas, "range assignor not found");
|
|
1679
|
+
|
|
1680
|
+
for (i = 0; i < RD_ARRAY_SIZE(ALL_RACKS) - 1; i++) {
|
|
1681
|
+
char c = 'a' + i;
|
|
1682
|
+
ALL_RACKS[i] = rd_kafkap_str_new(&c, 1);
|
|
1683
|
+
}
|
|
1684
|
+
ALL_RACKS[i] = NULL;
|
|
1685
|
+
|
|
1686
|
+
static int (*tests[])(
|
|
1687
|
+
rd_kafka_t *, const rd_kafka_assignor_t *,
|
|
1688
|
+
rd_kafka_assignor_ut_rack_config_t parametrization) = {
|
|
1689
|
+
ut_testOneConsumerNoTopic,
|
|
1690
|
+
ut_testOneConsumerNonexistentTopic,
|
|
1691
|
+
ut_testOneConsumerOneTopic,
|
|
1692
|
+
ut_testOnlyAssignsPartitionsFromSubscribedTopics,
|
|
1693
|
+
ut_testOneConsumerMultipleTopics,
|
|
1694
|
+
ut_testTwoConsumersOneTopicOnePartition,
|
|
1695
|
+
ut_testTwoConsumersOneTopicTwoPartitions,
|
|
1696
|
+
ut_testMultipleConsumersMixedTopicSubscriptions,
|
|
1697
|
+
ut_testTwoConsumersTwoTopicsSixPartitions,
|
|
1698
|
+
ut_testRackAwareAssignmentWithUniformSubscription,
|
|
1699
|
+
ut_testRackAwareAssignmentWithNonEqualSubscription,
|
|
1700
|
+
ut_testRackAwareAssignmentWithUniformPartitions,
|
|
1701
|
+
ut_testRackAwareAssignmentWithUniformPartitionsNonEqualSubscription,
|
|
1702
|
+
ut_testRackAwareAssignmentWithCoPartitioning0,
|
|
1703
|
+
ut_testRackAwareAssignmentWithCoPartitioning1,
|
|
1704
|
+
ut_testCoPartitionedAssignmentWithSameSubscription,
|
|
1705
|
+
NULL,
|
|
1706
|
+
};
|
|
1707
|
+
|
|
1708
|
+
for (i = 0; tests[i]; i++) {
|
|
1709
|
+
rd_ts_t ts = rd_clock();
|
|
1710
|
+
int r = 0;
|
|
1711
|
+
rd_kafka_assignor_ut_rack_config_t j;
|
|
1712
|
+
|
|
1713
|
+
for (j = RD_KAFKA_RANGE_ASSIGNOR_UT_NO_BROKER_RACK;
|
|
1714
|
+
j != RD_KAFKA_RANGE_ASSIGNOR_UT_CONFIG_CNT; j++) {
|
|
1715
|
+
RD_UT_SAY("[ Test #%" PRIusz ", RackConfig = %d ]", i,
|
|
1716
|
+
j);
|
|
1717
|
+
r += tests[i](rk, rkas, j);
|
|
1718
|
+
}
|
|
1719
|
+
RD_UT_SAY("[ Test #%" PRIusz " ran for %.3fms ]", i,
|
|
1720
|
+
(double)(rd_clock() - ts) / 1000.0);
|
|
1721
|
+
|
|
1722
|
+
RD_UT_ASSERT(!r, "^ failed");
|
|
1723
|
+
|
|
1724
|
+
fails += r;
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1727
|
+
for (i = 0; i < RD_ARRAY_SIZE(ALL_RACKS) - 1; i++) {
|
|
1728
|
+
rd_kafkap_str_destroy(ALL_RACKS[i]);
|
|
1729
|
+
}
|
|
1730
|
+
|
|
1731
|
+
rd_kafka_destroy(rk);
|
|
1732
|
+
|
|
1733
|
+
return fails;
|
|
1734
|
+
}
|
|
1735
|
+
|
|
1736
|
+
|
|
1737
|
+
|
|
1738
|
+
/**
|
|
1739
|
+
* @brief Initialzie and add range assignor.
|
|
1740
|
+
*/
|
|
1741
|
+
rd_kafka_resp_err_t rd_kafka_range_assignor_init(rd_kafka_t *rk) {
|
|
1742
|
+
return rd_kafka_assignor_add(
|
|
1743
|
+
rk, "consumer", "range", RD_KAFKA_REBALANCE_PROTOCOL_EAGER,
|
|
1744
|
+
rd_kafka_range_assignor_assign_cb,
|
|
1745
|
+
rd_kafka_assignor_get_metadata_with_empty_userdata,
|
|
1746
|
+
NULL /* on_assignment_cb */, NULL /* destroy_state_cb */,
|
|
1747
|
+
rd_kafka_range_assignor_unittest, NULL);
|
|
1748
|
+
}
|