@point3/node-rdkafka 3.6.0-1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.txt +20 -0
- package/README.md +636 -0
- package/binding.gyp +154 -0
- package/deps/librdkafka/.clang-format +136 -0
- package/deps/librdkafka/.clang-format-cpp +103 -0
- package/deps/librdkafka/.dir-locals.el +10 -0
- package/deps/librdkafka/.formatignore +33 -0
- package/deps/librdkafka/.gdbmacros +19 -0
- package/deps/librdkafka/.github/CODEOWNERS +1 -0
- package/deps/librdkafka/.github/ISSUE_TEMPLATE +34 -0
- package/deps/librdkafka/.semaphore/run-all-tests.yml +77 -0
- package/deps/librdkafka/.semaphore/semaphore-integration.yml +250 -0
- package/deps/librdkafka/.semaphore/semaphore.yml +378 -0
- package/deps/librdkafka/.semaphore/verify-linux-packages.yml +41 -0
- package/deps/librdkafka/CHANGELOG.md +2208 -0
- package/deps/librdkafka/CMakeLists.txt +291 -0
- package/deps/librdkafka/CODE_OF_CONDUCT.md +46 -0
- package/deps/librdkafka/CONFIGURATION.md +209 -0
- package/deps/librdkafka/CONTRIBUTING.md +431 -0
- package/deps/librdkafka/Doxyfile +2375 -0
- package/deps/librdkafka/INTRODUCTION.md +2481 -0
- package/deps/librdkafka/LICENSE +26 -0
- package/deps/librdkafka/LICENSE.cjson +22 -0
- package/deps/librdkafka/LICENSE.crc32c +28 -0
- package/deps/librdkafka/LICENSE.fnv1a +18 -0
- package/deps/librdkafka/LICENSE.hdrhistogram +27 -0
- package/deps/librdkafka/LICENSE.lz4 +26 -0
- package/deps/librdkafka/LICENSE.murmur2 +25 -0
- package/deps/librdkafka/LICENSE.nanopb +22 -0
- package/deps/librdkafka/LICENSE.opentelemetry +203 -0
- package/deps/librdkafka/LICENSE.pycrc +23 -0
- package/deps/librdkafka/LICENSE.queue +31 -0
- package/deps/librdkafka/LICENSE.regexp +5 -0
- package/deps/librdkafka/LICENSE.snappy +36 -0
- package/deps/librdkafka/LICENSE.tinycthread +26 -0
- package/deps/librdkafka/LICENSE.wingetopt +49 -0
- package/deps/librdkafka/LICENSES.txt +625 -0
- package/deps/librdkafka/Makefile +125 -0
- package/deps/librdkafka/README.md +199 -0
- package/deps/librdkafka/README.win32 +26 -0
- package/deps/librdkafka/STATISTICS.md +624 -0
- package/deps/librdkafka/configure +214 -0
- package/deps/librdkafka/configure.self +331 -0
- package/deps/librdkafka/debian/changelog +111 -0
- package/deps/librdkafka/debian/compat +1 -0
- package/deps/librdkafka/debian/control +71 -0
- package/deps/librdkafka/debian/copyright +99 -0
- package/deps/librdkafka/debian/gbp.conf +9 -0
- package/deps/librdkafka/debian/librdkafka++1.install +1 -0
- package/deps/librdkafka/debian/librdkafka-dev.examples +2 -0
- package/deps/librdkafka/debian/librdkafka-dev.install +9 -0
- package/deps/librdkafka/debian/librdkafka1.docs +5 -0
- package/deps/librdkafka/debian/librdkafka1.install +1 -0
- package/deps/librdkafka/debian/librdkafka1.symbols +135 -0
- package/deps/librdkafka/debian/rules +19 -0
- package/deps/librdkafka/debian/source/format +1 -0
- package/deps/librdkafka/debian/watch +2 -0
- package/deps/librdkafka/dev-conf.sh +123 -0
- package/deps/librdkafka/examples/CMakeLists.txt +79 -0
- package/deps/librdkafka/examples/Makefile +167 -0
- package/deps/librdkafka/examples/README.md +42 -0
- package/deps/librdkafka/examples/alter_consumer_group_offsets.c +338 -0
- package/deps/librdkafka/examples/consumer.c +271 -0
- package/deps/librdkafka/examples/delete_records.c +233 -0
- package/deps/librdkafka/examples/describe_cluster.c +322 -0
- package/deps/librdkafka/examples/describe_consumer_groups.c +455 -0
- package/deps/librdkafka/examples/describe_topics.c +427 -0
- package/deps/librdkafka/examples/elect_leaders.c +317 -0
- package/deps/librdkafka/examples/globals.json +11 -0
- package/deps/librdkafka/examples/idempotent_producer.c +344 -0
- package/deps/librdkafka/examples/incremental_alter_configs.c +347 -0
- package/deps/librdkafka/examples/kafkatest_verifiable_client.cpp +945 -0
- package/deps/librdkafka/examples/list_consumer_group_offsets.c +359 -0
- package/deps/librdkafka/examples/list_consumer_groups.c +365 -0
- package/deps/librdkafka/examples/list_offsets.c +327 -0
- package/deps/librdkafka/examples/misc.c +287 -0
- package/deps/librdkafka/examples/openssl_engine_example.cpp +248 -0
- package/deps/librdkafka/examples/producer.c +251 -0
- package/deps/librdkafka/examples/producer.cpp +228 -0
- package/deps/librdkafka/examples/rdkafka_complex_consumer_example.c +617 -0
- package/deps/librdkafka/examples/rdkafka_complex_consumer_example.cpp +467 -0
- package/deps/librdkafka/examples/rdkafka_consume_batch.cpp +264 -0
- package/deps/librdkafka/examples/rdkafka_example.c +853 -0
- package/deps/librdkafka/examples/rdkafka_example.cpp +679 -0
- package/deps/librdkafka/examples/rdkafka_performance.c +1781 -0
- package/deps/librdkafka/examples/transactions-older-broker.c +668 -0
- package/deps/librdkafka/examples/transactions.c +665 -0
- package/deps/librdkafka/examples/user_scram.c +491 -0
- package/deps/librdkafka/examples/win_ssl_cert_store.cpp +396 -0
- package/deps/librdkafka/lds-gen.py +73 -0
- package/deps/librdkafka/mainpage.doxy +40 -0
- package/deps/librdkafka/mklove/Makefile.base +329 -0
- package/deps/librdkafka/mklove/modules/configure.atomics +144 -0
- package/deps/librdkafka/mklove/modules/configure.base +2484 -0
- package/deps/librdkafka/mklove/modules/configure.builtin +70 -0
- package/deps/librdkafka/mklove/modules/configure.cc +186 -0
- package/deps/librdkafka/mklove/modules/configure.cxx +8 -0
- package/deps/librdkafka/mklove/modules/configure.fileversion +65 -0
- package/deps/librdkafka/mklove/modules/configure.gitversion +29 -0
- package/deps/librdkafka/mklove/modules/configure.good_cflags +18 -0
- package/deps/librdkafka/mklove/modules/configure.host +132 -0
- package/deps/librdkafka/mklove/modules/configure.lib +49 -0
- package/deps/librdkafka/mklove/modules/configure.libcurl +99 -0
- package/deps/librdkafka/mklove/modules/configure.libsasl2 +36 -0
- package/deps/librdkafka/mklove/modules/configure.libssl +147 -0
- package/deps/librdkafka/mklove/modules/configure.libzstd +58 -0
- package/deps/librdkafka/mklove/modules/configure.parseversion +95 -0
- package/deps/librdkafka/mklove/modules/configure.pic +16 -0
- package/deps/librdkafka/mklove/modules/configure.socket +20 -0
- package/deps/librdkafka/mklove/modules/configure.zlib +61 -0
- package/deps/librdkafka/mklove/modules/patches/README.md +8 -0
- package/deps/librdkafka/mklove/modules/patches/libcurl.0000-no-runtime-linking-check.patch +11 -0
- package/deps/librdkafka/mklove/modules/patches/libssl.0000-osx-rand-include-fix-OpenSSL-PR16409.patch +56 -0
- package/deps/librdkafka/packaging/RELEASE.md +319 -0
- package/deps/librdkafka/packaging/alpine/build-alpine.sh +38 -0
- package/deps/librdkafka/packaging/archlinux/PKGBUILD +30 -0
- package/deps/librdkafka/packaging/cmake/Config.cmake.in +37 -0
- package/deps/librdkafka/packaging/cmake/Modules/FindLZ4.cmake +38 -0
- package/deps/librdkafka/packaging/cmake/Modules/FindZSTD.cmake +27 -0
- package/deps/librdkafka/packaging/cmake/Modules/LICENSE.FindZstd +178 -0
- package/deps/librdkafka/packaging/cmake/README.md +38 -0
- package/deps/librdkafka/packaging/cmake/config.h.in +52 -0
- package/deps/librdkafka/packaging/cmake/parseversion.cmake +60 -0
- package/deps/librdkafka/packaging/cmake/rdkafka.pc.in +12 -0
- package/deps/librdkafka/packaging/cmake/try_compile/atomic_32_test.c +8 -0
- package/deps/librdkafka/packaging/cmake/try_compile/atomic_64_test.c +8 -0
- package/deps/librdkafka/packaging/cmake/try_compile/c11threads_test.c +14 -0
- package/deps/librdkafka/packaging/cmake/try_compile/crc32c_hw_test.c +27 -0
- package/deps/librdkafka/packaging/cmake/try_compile/dlopen_test.c +11 -0
- package/deps/librdkafka/packaging/cmake/try_compile/libsasl2_test.c +7 -0
- package/deps/librdkafka/packaging/cmake/try_compile/pthread_setname_darwin_test.c +6 -0
- package/deps/librdkafka/packaging/cmake/try_compile/pthread_setname_freebsd_test.c +7 -0
- package/deps/librdkafka/packaging/cmake/try_compile/pthread_setname_gnu_test.c +5 -0
- package/deps/librdkafka/packaging/cmake/try_compile/rand_r_test.c +7 -0
- package/deps/librdkafka/packaging/cmake/try_compile/rdkafka_setup.cmake +122 -0
- package/deps/librdkafka/packaging/cmake/try_compile/regex_test.c +10 -0
- package/deps/librdkafka/packaging/cmake/try_compile/strndup_test.c +5 -0
- package/deps/librdkafka/packaging/cmake/try_compile/sync_32_test.c +8 -0
- package/deps/librdkafka/packaging/cmake/try_compile/sync_64_test.c +8 -0
- package/deps/librdkafka/packaging/cp/README.md +16 -0
- package/deps/librdkafka/packaging/cp/check_features.c +72 -0
- package/deps/librdkafka/packaging/cp/verify-deb.sh +33 -0
- package/deps/librdkafka/packaging/cp/verify-packages.sh +69 -0
- package/deps/librdkafka/packaging/cp/verify-rpm.sh +32 -0
- package/deps/librdkafka/packaging/debian/changelog +66 -0
- package/deps/librdkafka/packaging/debian/compat +1 -0
- package/deps/librdkafka/packaging/debian/control +49 -0
- package/deps/librdkafka/packaging/debian/copyright +84 -0
- package/deps/librdkafka/packaging/debian/docs +5 -0
- package/deps/librdkafka/packaging/debian/gbp.conf +9 -0
- package/deps/librdkafka/packaging/debian/librdkafka-dev.dirs +2 -0
- package/deps/librdkafka/packaging/debian/librdkafka-dev.examples +2 -0
- package/deps/librdkafka/packaging/debian/librdkafka-dev.install +6 -0
- package/deps/librdkafka/packaging/debian/librdkafka-dev.substvars +1 -0
- package/deps/librdkafka/packaging/debian/librdkafka.dsc +16 -0
- package/deps/librdkafka/packaging/debian/librdkafka1-dbg.substvars +1 -0
- package/deps/librdkafka/packaging/debian/librdkafka1.dirs +1 -0
- package/deps/librdkafka/packaging/debian/librdkafka1.install +2 -0
- package/deps/librdkafka/packaging/debian/librdkafka1.postinst.debhelper +5 -0
- package/deps/librdkafka/packaging/debian/librdkafka1.postrm.debhelper +5 -0
- package/deps/librdkafka/packaging/debian/librdkafka1.symbols +64 -0
- package/deps/librdkafka/packaging/debian/rules +19 -0
- package/deps/librdkafka/packaging/debian/source/format +1 -0
- package/deps/librdkafka/packaging/debian/watch +2 -0
- package/deps/librdkafka/packaging/get_version.py +21 -0
- package/deps/librdkafka/packaging/homebrew/README.md +15 -0
- package/deps/librdkafka/packaging/homebrew/brew-update-pr.sh +31 -0
- package/deps/librdkafka/packaging/mingw-w64/configure-build-msys2-mingw-static.sh +52 -0
- package/deps/librdkafka/packaging/mingw-w64/configure-build-msys2-mingw.sh +21 -0
- package/deps/librdkafka/packaging/mingw-w64/export-variables.sh +13 -0
- package/deps/librdkafka/packaging/mingw-w64/run-tests.sh +6 -0
- package/deps/librdkafka/packaging/mingw-w64/semaphoreci-build.sh +38 -0
- package/deps/librdkafka/packaging/nuget/README.md +84 -0
- package/deps/librdkafka/packaging/nuget/artifact.py +177 -0
- package/deps/librdkafka/packaging/nuget/cleanup-s3.py +143 -0
- package/deps/librdkafka/packaging/nuget/common/p-common__plat-windows__arch-win32__bldtype-Release/msvcr120.zip +0 -0
- package/deps/librdkafka/packaging/nuget/common/p-common__plat-windows__arch-win32__bldtype-Release/msvcr140.zip +0 -0
- package/deps/librdkafka/packaging/nuget/common/p-common__plat-windows__arch-x64__bldtype-Release/msvcr120.zip +0 -0
- package/deps/librdkafka/packaging/nuget/common/p-common__plat-windows__arch-x64__bldtype-Release/msvcr140.zip +0 -0
- package/deps/librdkafka/packaging/nuget/nuget.sh +21 -0
- package/deps/librdkafka/packaging/nuget/nugetpackage.py +278 -0
- package/deps/librdkafka/packaging/nuget/packaging.py +448 -0
- package/deps/librdkafka/packaging/nuget/push-to-nuget.sh +21 -0
- package/deps/librdkafka/packaging/nuget/release.py +167 -0
- package/deps/librdkafka/packaging/nuget/requirements.txt +3 -0
- package/deps/librdkafka/packaging/nuget/staticpackage.py +178 -0
- package/deps/librdkafka/packaging/nuget/templates/librdkafka.redist.nuspec +21 -0
- package/deps/librdkafka/packaging/nuget/templates/librdkafka.redist.props +18 -0
- package/deps/librdkafka/packaging/nuget/templates/librdkafka.redist.targets +19 -0
- package/deps/librdkafka/packaging/nuget/zfile/__init__.py +0 -0
- package/deps/librdkafka/packaging/nuget/zfile/zfile.py +98 -0
- package/deps/librdkafka/packaging/rpm/Makefile +92 -0
- package/deps/librdkafka/packaging/rpm/README.md +23 -0
- package/deps/librdkafka/packaging/rpm/el7-x86_64.cfg +40 -0
- package/deps/librdkafka/packaging/rpm/librdkafka.spec +118 -0
- package/deps/librdkafka/packaging/rpm/mock-on-docker.sh +96 -0
- package/deps/librdkafka/packaging/rpm/tests/Makefile +25 -0
- package/deps/librdkafka/packaging/rpm/tests/README.md +8 -0
- package/deps/librdkafka/packaging/rpm/tests/run-test.sh +42 -0
- package/deps/librdkafka/packaging/rpm/tests/test-on-docker.sh +56 -0
- package/deps/librdkafka/packaging/rpm/tests/test.c +77 -0
- package/deps/librdkafka/packaging/rpm/tests/test.cpp +34 -0
- package/deps/librdkafka/packaging/tools/Dockerfile +31 -0
- package/deps/librdkafka/packaging/tools/build-configurations-checks.sh +12 -0
- package/deps/librdkafka/packaging/tools/build-deb-package.sh +64 -0
- package/deps/librdkafka/packaging/tools/build-debian.sh +65 -0
- package/deps/librdkafka/packaging/tools/build-manylinux.sh +68 -0
- package/deps/librdkafka/packaging/tools/build-release-artifacts.sh +139 -0
- package/deps/librdkafka/packaging/tools/distro-build.sh +38 -0
- package/deps/librdkafka/packaging/tools/gh-release-checksums.py +39 -0
- package/deps/librdkafka/packaging/tools/rdutcoverage.sh +25 -0
- package/deps/librdkafka/packaging/tools/requirements.txt +2 -0
- package/deps/librdkafka/packaging/tools/run-in-docker.sh +28 -0
- package/deps/librdkafka/packaging/tools/run-integration-tests.sh +31 -0
- package/deps/librdkafka/packaging/tools/run-style-check.sh +4 -0
- package/deps/librdkafka/packaging/tools/style-format.sh +149 -0
- package/deps/librdkafka/packaging/tools/update_rpcs_max_versions.py +100 -0
- package/deps/librdkafka/service.yml +172 -0
- package/deps/librdkafka/src/CMakeLists.txt +374 -0
- package/deps/librdkafka/src/Makefile +103 -0
- package/deps/librdkafka/src/README.lz4.md +30 -0
- package/deps/librdkafka/src/cJSON.c +2834 -0
- package/deps/librdkafka/src/cJSON.h +398 -0
- package/deps/librdkafka/src/crc32c.c +430 -0
- package/deps/librdkafka/src/crc32c.h +38 -0
- package/deps/librdkafka/src/generate_proto.sh +66 -0
- package/deps/librdkafka/src/librdkafka_cgrp_synch.png +0 -0
- package/deps/librdkafka/src/lz4.c +2727 -0
- package/deps/librdkafka/src/lz4.h +842 -0
- package/deps/librdkafka/src/lz4frame.c +2078 -0
- package/deps/librdkafka/src/lz4frame.h +692 -0
- package/deps/librdkafka/src/lz4frame_static.h +47 -0
- package/deps/librdkafka/src/lz4hc.c +1631 -0
- package/deps/librdkafka/src/lz4hc.h +413 -0
- package/deps/librdkafka/src/nanopb/pb.h +917 -0
- package/deps/librdkafka/src/nanopb/pb_common.c +388 -0
- package/deps/librdkafka/src/nanopb/pb_common.h +49 -0
- package/deps/librdkafka/src/nanopb/pb_decode.c +1727 -0
- package/deps/librdkafka/src/nanopb/pb_decode.h +193 -0
- package/deps/librdkafka/src/nanopb/pb_encode.c +1000 -0
- package/deps/librdkafka/src/nanopb/pb_encode.h +185 -0
- package/deps/librdkafka/src/opentelemetry/common.pb.c +32 -0
- package/deps/librdkafka/src/opentelemetry/common.pb.h +170 -0
- package/deps/librdkafka/src/opentelemetry/metrics.options +2 -0
- package/deps/librdkafka/src/opentelemetry/metrics.pb.c +67 -0
- package/deps/librdkafka/src/opentelemetry/metrics.pb.h +966 -0
- package/deps/librdkafka/src/opentelemetry/resource.pb.c +12 -0
- package/deps/librdkafka/src/opentelemetry/resource.pb.h +58 -0
- package/deps/librdkafka/src/queue.h +850 -0
- package/deps/librdkafka/src/rd.h +584 -0
- package/deps/librdkafka/src/rdaddr.c +255 -0
- package/deps/librdkafka/src/rdaddr.h +202 -0
- package/deps/librdkafka/src/rdatomic.h +230 -0
- package/deps/librdkafka/src/rdavg.h +260 -0
- package/deps/librdkafka/src/rdavl.c +210 -0
- package/deps/librdkafka/src/rdavl.h +250 -0
- package/deps/librdkafka/src/rdbase64.c +200 -0
- package/deps/librdkafka/src/rdbase64.h +43 -0
- package/deps/librdkafka/src/rdbuf.c +1884 -0
- package/deps/librdkafka/src/rdbuf.h +375 -0
- package/deps/librdkafka/src/rdcrc32.c +114 -0
- package/deps/librdkafka/src/rdcrc32.h +170 -0
- package/deps/librdkafka/src/rddl.c +179 -0
- package/deps/librdkafka/src/rddl.h +43 -0
- package/deps/librdkafka/src/rdendian.h +175 -0
- package/deps/librdkafka/src/rdfloat.h +67 -0
- package/deps/librdkafka/src/rdfnv1a.c +113 -0
- package/deps/librdkafka/src/rdfnv1a.h +35 -0
- package/deps/librdkafka/src/rdgz.c +120 -0
- package/deps/librdkafka/src/rdgz.h +46 -0
- package/deps/librdkafka/src/rdhdrhistogram.c +721 -0
- package/deps/librdkafka/src/rdhdrhistogram.h +87 -0
- package/deps/librdkafka/src/rdhttp.c +830 -0
- package/deps/librdkafka/src/rdhttp.h +101 -0
- package/deps/librdkafka/src/rdinterval.h +177 -0
- package/deps/librdkafka/src/rdkafka.c +5505 -0
- package/deps/librdkafka/src/rdkafka.h +10686 -0
- package/deps/librdkafka/src/rdkafka_admin.c +9794 -0
- package/deps/librdkafka/src/rdkafka_admin.h +661 -0
- package/deps/librdkafka/src/rdkafka_assignment.c +1010 -0
- package/deps/librdkafka/src/rdkafka_assignment.h +73 -0
- package/deps/librdkafka/src/rdkafka_assignor.c +1786 -0
- package/deps/librdkafka/src/rdkafka_assignor.h +402 -0
- package/deps/librdkafka/src/rdkafka_aux.c +409 -0
- package/deps/librdkafka/src/rdkafka_aux.h +174 -0
- package/deps/librdkafka/src/rdkafka_background.c +221 -0
- package/deps/librdkafka/src/rdkafka_broker.c +6337 -0
- package/deps/librdkafka/src/rdkafka_broker.h +744 -0
- package/deps/librdkafka/src/rdkafka_buf.c +543 -0
- package/deps/librdkafka/src/rdkafka_buf.h +1525 -0
- package/deps/librdkafka/src/rdkafka_cert.c +576 -0
- package/deps/librdkafka/src/rdkafka_cert.h +62 -0
- package/deps/librdkafka/src/rdkafka_cgrp.c +7587 -0
- package/deps/librdkafka/src/rdkafka_cgrp.h +477 -0
- package/deps/librdkafka/src/rdkafka_conf.c +4880 -0
- package/deps/librdkafka/src/rdkafka_conf.h +732 -0
- package/deps/librdkafka/src/rdkafka_confval.h +97 -0
- package/deps/librdkafka/src/rdkafka_coord.c +623 -0
- package/deps/librdkafka/src/rdkafka_coord.h +132 -0
- package/deps/librdkafka/src/rdkafka_error.c +228 -0
- package/deps/librdkafka/src/rdkafka_error.h +80 -0
- package/deps/librdkafka/src/rdkafka_event.c +502 -0
- package/deps/librdkafka/src/rdkafka_event.h +126 -0
- package/deps/librdkafka/src/rdkafka_feature.c +898 -0
- package/deps/librdkafka/src/rdkafka_feature.h +104 -0
- package/deps/librdkafka/src/rdkafka_fetcher.c +1422 -0
- package/deps/librdkafka/src/rdkafka_fetcher.h +44 -0
- package/deps/librdkafka/src/rdkafka_header.c +220 -0
- package/deps/librdkafka/src/rdkafka_header.h +76 -0
- package/deps/librdkafka/src/rdkafka_idempotence.c +807 -0
- package/deps/librdkafka/src/rdkafka_idempotence.h +144 -0
- package/deps/librdkafka/src/rdkafka_int.h +1260 -0
- package/deps/librdkafka/src/rdkafka_interceptor.c +819 -0
- package/deps/librdkafka/src/rdkafka_interceptor.h +104 -0
- package/deps/librdkafka/src/rdkafka_lz4.c +450 -0
- package/deps/librdkafka/src/rdkafka_lz4.h +49 -0
- package/deps/librdkafka/src/rdkafka_metadata.c +2209 -0
- package/deps/librdkafka/src/rdkafka_metadata.h +345 -0
- package/deps/librdkafka/src/rdkafka_metadata_cache.c +1183 -0
- package/deps/librdkafka/src/rdkafka_mock.c +3661 -0
- package/deps/librdkafka/src/rdkafka_mock.h +610 -0
- package/deps/librdkafka/src/rdkafka_mock_cgrp.c +1876 -0
- package/deps/librdkafka/src/rdkafka_mock_handlers.c +3113 -0
- package/deps/librdkafka/src/rdkafka_mock_int.h +710 -0
- package/deps/librdkafka/src/rdkafka_msg.c +2589 -0
- package/deps/librdkafka/src/rdkafka_msg.h +614 -0
- package/deps/librdkafka/src/rdkafka_msgbatch.h +62 -0
- package/deps/librdkafka/src/rdkafka_msgset.h +98 -0
- package/deps/librdkafka/src/rdkafka_msgset_reader.c +1806 -0
- package/deps/librdkafka/src/rdkafka_msgset_writer.c +1474 -0
- package/deps/librdkafka/src/rdkafka_offset.c +1565 -0
- package/deps/librdkafka/src/rdkafka_offset.h +150 -0
- package/deps/librdkafka/src/rdkafka_op.c +997 -0
- package/deps/librdkafka/src/rdkafka_op.h +858 -0
- package/deps/librdkafka/src/rdkafka_partition.c +4896 -0
- package/deps/librdkafka/src/rdkafka_partition.h +1182 -0
- package/deps/librdkafka/src/rdkafka_pattern.c +228 -0
- package/deps/librdkafka/src/rdkafka_pattern.h +70 -0
- package/deps/librdkafka/src/rdkafka_plugin.c +213 -0
- package/deps/librdkafka/src/rdkafka_plugin.h +41 -0
- package/deps/librdkafka/src/rdkafka_proto.h +736 -0
- package/deps/librdkafka/src/rdkafka_protocol.h +128 -0
- package/deps/librdkafka/src/rdkafka_queue.c +1230 -0
- package/deps/librdkafka/src/rdkafka_queue.h +1220 -0
- package/deps/librdkafka/src/rdkafka_range_assignor.c +1748 -0
- package/deps/librdkafka/src/rdkafka_request.c +7089 -0
- package/deps/librdkafka/src/rdkafka_request.h +732 -0
- package/deps/librdkafka/src/rdkafka_roundrobin_assignor.c +123 -0
- package/deps/librdkafka/src/rdkafka_sasl.c +530 -0
- package/deps/librdkafka/src/rdkafka_sasl.h +63 -0
- package/deps/librdkafka/src/rdkafka_sasl_cyrus.c +722 -0
- package/deps/librdkafka/src/rdkafka_sasl_int.h +89 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer.c +1833 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer.h +52 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer_oidc.c +1666 -0
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer_oidc.h +47 -0
- package/deps/librdkafka/src/rdkafka_sasl_plain.c +142 -0
- package/deps/librdkafka/src/rdkafka_sasl_scram.c +858 -0
- package/deps/librdkafka/src/rdkafka_sasl_win32.c +550 -0
- package/deps/librdkafka/src/rdkafka_ssl.c +2129 -0
- package/deps/librdkafka/src/rdkafka_ssl.h +86 -0
- package/deps/librdkafka/src/rdkafka_sticky_assignor.c +4785 -0
- package/deps/librdkafka/src/rdkafka_subscription.c +278 -0
- package/deps/librdkafka/src/rdkafka_telemetry.c +760 -0
- package/deps/librdkafka/src/rdkafka_telemetry.h +52 -0
- package/deps/librdkafka/src/rdkafka_telemetry_decode.c +1053 -0
- package/deps/librdkafka/src/rdkafka_telemetry_decode.h +59 -0
- package/deps/librdkafka/src/rdkafka_telemetry_encode.c +997 -0
- package/deps/librdkafka/src/rdkafka_telemetry_encode.h +301 -0
- package/deps/librdkafka/src/rdkafka_timer.c +402 -0
- package/deps/librdkafka/src/rdkafka_timer.h +117 -0
- package/deps/librdkafka/src/rdkafka_topic.c +2161 -0
- package/deps/librdkafka/src/rdkafka_topic.h +334 -0
- package/deps/librdkafka/src/rdkafka_transport.c +1309 -0
- package/deps/librdkafka/src/rdkafka_transport.h +99 -0
- package/deps/librdkafka/src/rdkafka_transport_int.h +100 -0
- package/deps/librdkafka/src/rdkafka_txnmgr.c +3256 -0
- package/deps/librdkafka/src/rdkafka_txnmgr.h +171 -0
- package/deps/librdkafka/src/rdkafka_zstd.c +226 -0
- package/deps/librdkafka/src/rdkafka_zstd.h +57 -0
- package/deps/librdkafka/src/rdlist.c +576 -0
- package/deps/librdkafka/src/rdlist.h +434 -0
- package/deps/librdkafka/src/rdlog.c +89 -0
- package/deps/librdkafka/src/rdlog.h +41 -0
- package/deps/librdkafka/src/rdmap.c +508 -0
- package/deps/librdkafka/src/rdmap.h +492 -0
- package/deps/librdkafka/src/rdmurmur2.c +167 -0
- package/deps/librdkafka/src/rdmurmur2.h +35 -0
- package/deps/librdkafka/src/rdports.c +61 -0
- package/deps/librdkafka/src/rdports.h +38 -0
- package/deps/librdkafka/src/rdposix.h +250 -0
- package/deps/librdkafka/src/rdrand.c +80 -0
- package/deps/librdkafka/src/rdrand.h +43 -0
- package/deps/librdkafka/src/rdregex.c +156 -0
- package/deps/librdkafka/src/rdregex.h +43 -0
- package/deps/librdkafka/src/rdsignal.h +57 -0
- package/deps/librdkafka/src/rdstring.c +645 -0
- package/deps/librdkafka/src/rdstring.h +98 -0
- package/deps/librdkafka/src/rdsysqueue.h +404 -0
- package/deps/librdkafka/src/rdtime.h +356 -0
- package/deps/librdkafka/src/rdtypes.h +86 -0
- package/deps/librdkafka/src/rdunittest.c +549 -0
- package/deps/librdkafka/src/rdunittest.h +232 -0
- package/deps/librdkafka/src/rdvarint.c +134 -0
- package/deps/librdkafka/src/rdvarint.h +165 -0
- package/deps/librdkafka/src/rdwin32.h +382 -0
- package/deps/librdkafka/src/rdxxhash.c +1030 -0
- package/deps/librdkafka/src/rdxxhash.h +328 -0
- package/deps/librdkafka/src/regexp.c +1352 -0
- package/deps/librdkafka/src/regexp.h +41 -0
- package/deps/librdkafka/src/snappy.c +1866 -0
- package/deps/librdkafka/src/snappy.h +62 -0
- package/deps/librdkafka/src/snappy_compat.h +138 -0
- package/deps/librdkafka/src/statistics_schema.json +444 -0
- package/deps/librdkafka/src/tinycthread.c +932 -0
- package/deps/librdkafka/src/tinycthread.h +503 -0
- package/deps/librdkafka/src/tinycthread_extra.c +199 -0
- package/deps/librdkafka/src/tinycthread_extra.h +212 -0
- package/deps/librdkafka/src/win32_config.h +58 -0
- package/deps/librdkafka/src-cpp/CMakeLists.txt +90 -0
- package/deps/librdkafka/src-cpp/ConfImpl.cpp +84 -0
- package/deps/librdkafka/src-cpp/ConsumerImpl.cpp +244 -0
- package/deps/librdkafka/src-cpp/HandleImpl.cpp +436 -0
- package/deps/librdkafka/src-cpp/HeadersImpl.cpp +48 -0
- package/deps/librdkafka/src-cpp/KafkaConsumerImpl.cpp +296 -0
- package/deps/librdkafka/src-cpp/Makefile +55 -0
- package/deps/librdkafka/src-cpp/MessageImpl.cpp +38 -0
- package/deps/librdkafka/src-cpp/MetadataImpl.cpp +170 -0
- package/deps/librdkafka/src-cpp/ProducerImpl.cpp +197 -0
- package/deps/librdkafka/src-cpp/QueueImpl.cpp +70 -0
- package/deps/librdkafka/src-cpp/README.md +16 -0
- package/deps/librdkafka/src-cpp/RdKafka.cpp +59 -0
- package/deps/librdkafka/src-cpp/TopicImpl.cpp +124 -0
- package/deps/librdkafka/src-cpp/TopicPartitionImpl.cpp +57 -0
- package/deps/librdkafka/src-cpp/rdkafkacpp.h +3797 -0
- package/deps/librdkafka/src-cpp/rdkafkacpp_int.h +1641 -0
- package/deps/librdkafka/tests/0000-unittests.c +72 -0
- package/deps/librdkafka/tests/0001-multiobj.c +102 -0
- package/deps/librdkafka/tests/0002-unkpart.c +244 -0
- package/deps/librdkafka/tests/0003-msgmaxsize.c +173 -0
- package/deps/librdkafka/tests/0004-conf.c +934 -0
- package/deps/librdkafka/tests/0005-order.c +133 -0
- package/deps/librdkafka/tests/0006-symbols.c +163 -0
- package/deps/librdkafka/tests/0007-autotopic.c +136 -0
- package/deps/librdkafka/tests/0008-reqacks.c +179 -0
- package/deps/librdkafka/tests/0009-mock_cluster.c +97 -0
- package/deps/librdkafka/tests/0011-produce_batch.c +753 -0
- package/deps/librdkafka/tests/0012-produce_consume.c +537 -0
- package/deps/librdkafka/tests/0013-null-msgs.c +473 -0
- package/deps/librdkafka/tests/0014-reconsume-191.c +512 -0
- package/deps/librdkafka/tests/0015-offset_seeks.c +172 -0
- package/deps/librdkafka/tests/0016-client_swname.c +181 -0
- package/deps/librdkafka/tests/0017-compression.c +140 -0
- package/deps/librdkafka/tests/0018-cgrp_term.c +338 -0
- package/deps/librdkafka/tests/0019-list_groups.c +289 -0
- package/deps/librdkafka/tests/0020-destroy_hang.c +162 -0
- package/deps/librdkafka/tests/0021-rkt_destroy.c +72 -0
- package/deps/librdkafka/tests/0022-consume_batch.c +279 -0
- package/deps/librdkafka/tests/0025-timers.c +147 -0
- package/deps/librdkafka/tests/0026-consume_pause.c +547 -0
- package/deps/librdkafka/tests/0028-long_topicnames.c +79 -0
- package/deps/librdkafka/tests/0029-assign_offset.c +202 -0
- package/deps/librdkafka/tests/0030-offset_commit.c +589 -0
- package/deps/librdkafka/tests/0031-get_offsets.c +235 -0
- package/deps/librdkafka/tests/0033-regex_subscribe.c +536 -0
- package/deps/librdkafka/tests/0034-offset_reset.c +398 -0
- package/deps/librdkafka/tests/0035-api_version.c +73 -0
- package/deps/librdkafka/tests/0036-partial_fetch.c +87 -0
- package/deps/librdkafka/tests/0037-destroy_hang_local.c +85 -0
- package/deps/librdkafka/tests/0038-performance.c +121 -0
- package/deps/librdkafka/tests/0039-event.c +284 -0
- package/deps/librdkafka/tests/0040-io_event.c +257 -0
- package/deps/librdkafka/tests/0041-fetch_max_bytes.c +97 -0
- package/deps/librdkafka/tests/0042-many_topics.c +252 -0
- package/deps/librdkafka/tests/0043-no_connection.c +77 -0
- package/deps/librdkafka/tests/0044-partition_cnt.c +94 -0
- package/deps/librdkafka/tests/0045-subscribe_update.c +1010 -0
- package/deps/librdkafka/tests/0046-rkt_cache.c +65 -0
- package/deps/librdkafka/tests/0047-partial_buf_tmout.c +98 -0
- package/deps/librdkafka/tests/0048-partitioner.c +283 -0
- package/deps/librdkafka/tests/0049-consume_conn_close.c +162 -0
- package/deps/librdkafka/tests/0050-subscribe_adds.c +145 -0
- package/deps/librdkafka/tests/0051-assign_adds.c +126 -0
- package/deps/librdkafka/tests/0052-msg_timestamps.c +238 -0
- package/deps/librdkafka/tests/0053-stats_cb.cpp +527 -0
- package/deps/librdkafka/tests/0054-offset_time.cpp +236 -0
- package/deps/librdkafka/tests/0055-producer_latency.c +539 -0
- package/deps/librdkafka/tests/0056-balanced_group_mt.c +315 -0
- package/deps/librdkafka/tests/0057-invalid_topic.cpp +112 -0
- package/deps/librdkafka/tests/0058-log.cpp +123 -0
- package/deps/librdkafka/tests/0059-bsearch.cpp +241 -0
- package/deps/librdkafka/tests/0060-op_prio.cpp +163 -0
- package/deps/librdkafka/tests/0061-consumer_lag.cpp +295 -0
- package/deps/librdkafka/tests/0062-stats_event.c +126 -0
- package/deps/librdkafka/tests/0063-clusterid.cpp +180 -0
- package/deps/librdkafka/tests/0064-interceptors.c +481 -0
- package/deps/librdkafka/tests/0065-yield.cpp +140 -0
- package/deps/librdkafka/tests/0066-plugins.cpp +129 -0
- package/deps/librdkafka/tests/0067-empty_topic.cpp +151 -0
- package/deps/librdkafka/tests/0068-produce_timeout.c +136 -0
- package/deps/librdkafka/tests/0069-consumer_add_parts.c +119 -0
- package/deps/librdkafka/tests/0070-null_empty.cpp +197 -0
- package/deps/librdkafka/tests/0072-headers_ut.c +448 -0
- package/deps/librdkafka/tests/0073-headers.c +381 -0
- package/deps/librdkafka/tests/0074-producev.c +87 -0
- package/deps/librdkafka/tests/0075-retry.c +290 -0
- package/deps/librdkafka/tests/0076-produce_retry.c +452 -0
- package/deps/librdkafka/tests/0077-compaction.c +363 -0
- package/deps/librdkafka/tests/0078-c_from_cpp.cpp +96 -0
- package/deps/librdkafka/tests/0079-fork.c +93 -0
- package/deps/librdkafka/tests/0080-admin_ut.c +3095 -0
- package/deps/librdkafka/tests/0081-admin.c +5633 -0
- package/deps/librdkafka/tests/0082-fetch_max_bytes.cpp +137 -0
- package/deps/librdkafka/tests/0083-cb_event.c +233 -0
- package/deps/librdkafka/tests/0084-destroy_flags.c +208 -0
- package/deps/librdkafka/tests/0085-headers.cpp +392 -0
- package/deps/librdkafka/tests/0086-purge.c +368 -0
- package/deps/librdkafka/tests/0088-produce_metadata_timeout.c +162 -0
- package/deps/librdkafka/tests/0089-max_poll_interval.c +511 -0
- package/deps/librdkafka/tests/0090-idempotence.c +171 -0
- package/deps/librdkafka/tests/0091-max_poll_interval_timeout.c +295 -0
- package/deps/librdkafka/tests/0092-mixed_msgver.c +103 -0
- package/deps/librdkafka/tests/0093-holb.c +200 -0
- package/deps/librdkafka/tests/0094-idempotence_msg_timeout.c +231 -0
- package/deps/librdkafka/tests/0095-all_brokers_down.cpp +122 -0
- package/deps/librdkafka/tests/0097-ssl_verify.cpp +658 -0
- package/deps/librdkafka/tests/0098-consumer-txn.cpp +1218 -0
- package/deps/librdkafka/tests/0099-commit_metadata.c +194 -0
- package/deps/librdkafka/tests/0100-thread_interceptors.cpp +195 -0
- package/deps/librdkafka/tests/0101-fetch-from-follower.cpp +446 -0
- package/deps/librdkafka/tests/0102-static_group_rebalance.c +836 -0
- package/deps/librdkafka/tests/0103-transactions.c +1383 -0
- package/deps/librdkafka/tests/0104-fetch_from_follower_mock.c +625 -0
- package/deps/librdkafka/tests/0105-transactions_mock.c +3930 -0
- package/deps/librdkafka/tests/0106-cgrp_sess_timeout.c +318 -0
- package/deps/librdkafka/tests/0107-topic_recreate.c +259 -0
- package/deps/librdkafka/tests/0109-auto_create_topics.cpp +278 -0
- package/deps/librdkafka/tests/0110-batch_size.cpp +182 -0
- package/deps/librdkafka/tests/0111-delay_create_topics.cpp +127 -0
- package/deps/librdkafka/tests/0112-assign_unknown_part.c +87 -0
- package/deps/librdkafka/tests/0113-cooperative_rebalance.cpp +3473 -0
- package/deps/librdkafka/tests/0114-sticky_partitioning.cpp +176 -0
- package/deps/librdkafka/tests/0115-producer_auth.cpp +182 -0
- package/deps/librdkafka/tests/0116-kafkaconsumer_close.cpp +216 -0
- package/deps/librdkafka/tests/0117-mock_errors.c +331 -0
- package/deps/librdkafka/tests/0118-commit_rebalance.c +154 -0
- package/deps/librdkafka/tests/0119-consumer_auth.cpp +167 -0
- package/deps/librdkafka/tests/0120-asymmetric_subscription.c +185 -0
- package/deps/librdkafka/tests/0121-clusterid.c +115 -0
- package/deps/librdkafka/tests/0122-buffer_cleaning_after_rebalance.c +227 -0
- package/deps/librdkafka/tests/0123-connections_max_idle.c +98 -0
- package/deps/librdkafka/tests/0124-openssl_invalid_engine.c +69 -0
- package/deps/librdkafka/tests/0125-immediate_flush.c +144 -0
- package/deps/librdkafka/tests/0126-oauthbearer_oidc.c +528 -0
- package/deps/librdkafka/tests/0127-fetch_queue_backoff.cpp +165 -0
- package/deps/librdkafka/tests/0128-sasl_callback_queue.cpp +125 -0
- package/deps/librdkafka/tests/0129-fetch_aborted_msgs.c +79 -0
- package/deps/librdkafka/tests/0130-store_offsets.c +178 -0
- package/deps/librdkafka/tests/0131-connect_timeout.c +81 -0
- package/deps/librdkafka/tests/0132-strategy_ordering.c +179 -0
- package/deps/librdkafka/tests/0133-ssl_keys.c +150 -0
- package/deps/librdkafka/tests/0134-ssl_provider.c +92 -0
- package/deps/librdkafka/tests/0135-sasl_credentials.cpp +143 -0
- package/deps/librdkafka/tests/0136-resolve_cb.c +181 -0
- package/deps/librdkafka/tests/0137-barrier_batch_consume.c +619 -0
- package/deps/librdkafka/tests/0138-admin_mock.c +281 -0
- package/deps/librdkafka/tests/0139-offset_validation_mock.c +950 -0
- package/deps/librdkafka/tests/0140-commit_metadata.cpp +108 -0
- package/deps/librdkafka/tests/0142-reauthentication.c +515 -0
- package/deps/librdkafka/tests/0143-exponential_backoff_mock.c +552 -0
- package/deps/librdkafka/tests/0144-idempotence_mock.c +373 -0
- package/deps/librdkafka/tests/0145-pause_resume_mock.c +119 -0
- package/deps/librdkafka/tests/0146-metadata_mock.c +505 -0
- package/deps/librdkafka/tests/0147-consumer_group_consumer_mock.c +952 -0
- package/deps/librdkafka/tests/0148-offset_fetch_commit_error_mock.c +563 -0
- package/deps/librdkafka/tests/0149-broker-same-host-port.c +140 -0
- package/deps/librdkafka/tests/0150-telemetry_mock.c +651 -0
- package/deps/librdkafka/tests/0151-purge-brokers.c +566 -0
- package/deps/librdkafka/tests/0152-rebootstrap.c +59 -0
- package/deps/librdkafka/tests/0153-memberid.c +128 -0
- package/deps/librdkafka/tests/1000-unktopic.c +164 -0
- package/deps/librdkafka/tests/8000-idle.cpp +60 -0
- package/deps/librdkafka/tests/8001-fetch_from_follower_mock_manual.c +113 -0
- package/deps/librdkafka/tests/CMakeLists.txt +170 -0
- package/deps/librdkafka/tests/LibrdkafkaTestApp.py +291 -0
- package/deps/librdkafka/tests/Makefile +182 -0
- package/deps/librdkafka/tests/README.md +509 -0
- package/deps/librdkafka/tests/autotest.sh +33 -0
- package/deps/librdkafka/tests/backtrace.gdb +30 -0
- package/deps/librdkafka/tests/broker_version_tests.py +315 -0
- package/deps/librdkafka/tests/buildbox.sh +17 -0
- package/deps/librdkafka/tests/cleanup-checker-tests.sh +20 -0
- package/deps/librdkafka/tests/cluster_testing.py +191 -0
- package/deps/librdkafka/tests/delete-test-topics.sh +56 -0
- package/deps/librdkafka/tests/fixtures/oauthbearer/jwt_assertion_template.json +10 -0
- package/deps/librdkafka/tests/fixtures/ssl/Makefile +8 -0
- package/deps/librdkafka/tests/fixtures/ssl/README.md +13 -0
- package/deps/librdkafka/tests/fixtures/ssl/client.keystore.intermediate.p12 +0 -0
- package/deps/librdkafka/tests/fixtures/ssl/client.keystore.p12 +0 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.certificate.intermediate.pem +72 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.certificate.pem +50 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.intermediate.key +46 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.key +46 -0
- package/deps/librdkafka/tests/fixtures/ssl/create_keys.sh +168 -0
- package/deps/librdkafka/tests/fuzzers/Makefile +12 -0
- package/deps/librdkafka/tests/fuzzers/README.md +31 -0
- package/deps/librdkafka/tests/fuzzers/fuzz_regex.c +74 -0
- package/deps/librdkafka/tests/fuzzers/helpers.h +90 -0
- package/deps/librdkafka/tests/gen-ssl-certs.sh +165 -0
- package/deps/librdkafka/tests/interactive_broker_version.py +170 -0
- package/deps/librdkafka/tests/interceptor_test/CMakeLists.txt +16 -0
- package/deps/librdkafka/tests/interceptor_test/Makefile +22 -0
- package/deps/librdkafka/tests/interceptor_test/interceptor_test.c +314 -0
- package/deps/librdkafka/tests/interceptor_test/interceptor_test.h +54 -0
- package/deps/librdkafka/tests/java/IncrementalRebalanceCli.java +97 -0
- package/deps/librdkafka/tests/java/Makefile +13 -0
- package/deps/librdkafka/tests/java/Murmur2Cli.java +46 -0
- package/deps/librdkafka/tests/java/README.md +14 -0
- package/deps/librdkafka/tests/java/TransactionProducerCli.java +162 -0
- package/deps/librdkafka/tests/java/run-class.sh +11 -0
- package/deps/librdkafka/tests/librdkafka.suppressions +483 -0
- package/deps/librdkafka/tests/lz4_manual_test.sh +59 -0
- package/deps/librdkafka/tests/multi-broker-version-test.sh +50 -0
- package/deps/librdkafka/tests/parse-refcnt.sh +43 -0
- package/deps/librdkafka/tests/performance_plot.py +115 -0
- package/deps/librdkafka/tests/plugin_test/Makefile +19 -0
- package/deps/librdkafka/tests/plugin_test/plugin_test.c +58 -0
- package/deps/librdkafka/tests/requirements.txt +2 -0
- package/deps/librdkafka/tests/run-all-tests.sh +79 -0
- package/deps/librdkafka/tests/run-consumer-tests.sh +16 -0
- package/deps/librdkafka/tests/run-producer-tests.sh +16 -0
- package/deps/librdkafka/tests/run-test-batches.py +157 -0
- package/deps/librdkafka/tests/run-test.sh +140 -0
- package/deps/librdkafka/tests/rusage.c +249 -0
- package/deps/librdkafka/tests/sasl_test.py +289 -0
- package/deps/librdkafka/tests/scenarios/README.md +6 -0
- package/deps/librdkafka/tests/scenarios/ak23.json +6 -0
- package/deps/librdkafka/tests/scenarios/default.json +5 -0
- package/deps/librdkafka/tests/scenarios/noautocreate.json +5 -0
- package/deps/librdkafka/tests/sockem.c +801 -0
- package/deps/librdkafka/tests/sockem.h +85 -0
- package/deps/librdkafka/tests/sockem_ctrl.c +145 -0
- package/deps/librdkafka/tests/sockem_ctrl.h +61 -0
- package/deps/librdkafka/tests/test.c +7778 -0
- package/deps/librdkafka/tests/test.conf.example +27 -0
- package/deps/librdkafka/tests/test.h +1028 -0
- package/deps/librdkafka/tests/testcpp.cpp +131 -0
- package/deps/librdkafka/tests/testcpp.h +388 -0
- package/deps/librdkafka/tests/testshared.h +416 -0
- package/deps/librdkafka/tests/tools/README.md +4 -0
- package/deps/librdkafka/tests/tools/stats/README.md +21 -0
- package/deps/librdkafka/tests/tools/stats/filter.jq +42 -0
- package/deps/librdkafka/tests/tools/stats/graph.py +150 -0
- package/deps/librdkafka/tests/tools/stats/requirements.txt +3 -0
- package/deps/librdkafka/tests/tools/stats/to_csv.py +124 -0
- package/deps/librdkafka/tests/trivup/trivup-0.14.0.tar.gz +0 -0
- package/deps/librdkafka/tests/until-fail.sh +87 -0
- package/deps/librdkafka/tests/xxxx-assign_partition.c +122 -0
- package/deps/librdkafka/tests/xxxx-metadata.cpp +159 -0
- package/deps/librdkafka/vcpkg.json +23 -0
- package/deps/librdkafka/win32/README.md +5 -0
- package/deps/librdkafka/win32/build-package.bat +3 -0
- package/deps/librdkafka/win32/build.bat +19 -0
- package/deps/librdkafka/win32/common.vcxproj +84 -0
- package/deps/librdkafka/win32/interceptor_test/interceptor_test.vcxproj +87 -0
- package/deps/librdkafka/win32/librdkafka.autopkg.template +54 -0
- package/deps/librdkafka/win32/librdkafka.master.testing.targets +13 -0
- package/deps/librdkafka/win32/librdkafka.sln +226 -0
- package/deps/librdkafka/win32/librdkafka.vcxproj +276 -0
- package/deps/librdkafka/win32/librdkafkacpp/librdkafkacpp.vcxproj +104 -0
- package/deps/librdkafka/win32/msbuild.ps1 +15 -0
- package/deps/librdkafka/win32/openssl_engine_example/openssl_engine_example.vcxproj +132 -0
- package/deps/librdkafka/win32/package-zip.ps1 +46 -0
- package/deps/librdkafka/win32/packages/repositories.config +4 -0
- package/deps/librdkafka/win32/push-package.bat +4 -0
- package/deps/librdkafka/win32/rdkafka_complex_consumer_example_cpp/rdkafka_complex_consumer_example_cpp.vcxproj +67 -0
- package/deps/librdkafka/win32/rdkafka_example/rdkafka_example.vcxproj +97 -0
- package/deps/librdkafka/win32/rdkafka_performance/rdkafka_performance.vcxproj +97 -0
- package/deps/librdkafka/win32/setup-msys2.ps1 +47 -0
- package/deps/librdkafka/win32/setup-vcpkg.ps1 +34 -0
- package/deps/librdkafka/win32/tests/test.conf.example +25 -0
- package/deps/librdkafka/win32/tests/tests.vcxproj +253 -0
- package/deps/librdkafka/win32/win_ssl_cert_store/win_ssl_cert_store.vcxproj +132 -0
- package/deps/librdkafka/win32/wingetopt.c +564 -0
- package/deps/librdkafka/win32/wingetopt.h +101 -0
- package/deps/librdkafka/win32/wintime.h +33 -0
- package/deps/librdkafka.gyp +62 -0
- package/lib/admin.js +233 -0
- package/lib/client.js +573 -0
- package/lib/error.js +500 -0
- package/lib/index.js +34 -0
- package/lib/kafka-consumer-stream.js +397 -0
- package/lib/kafka-consumer.js +698 -0
- package/lib/producer/high-level-producer.js +323 -0
- package/lib/producer-stream.js +307 -0
- package/lib/producer.js +375 -0
- package/lib/tools/ref-counter.js +52 -0
- package/lib/topic-partition.js +88 -0
- package/lib/topic.js +42 -0
- package/lib/util.js +29 -0
- package/package.json +61 -0
- package/prebuilds/darwin-arm64/@point3+node-rdkafka.node +0 -0
- package/prebuilds/linux-x64/@point3+node-rdkafka.node +0 -0
- package/util/configure.js +30 -0
- package/util/get-env.js +6 -0
- package/util/test-compile.js +11 -0
- package/util/test-producer-delivery.js +100 -0
|
@@ -0,0 +1,1876 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* librdkafka - Apache Kafka C library
|
|
3
|
+
*
|
|
4
|
+
* Copyright (c) 2020-2022, Magnus Edenhill
|
|
5
|
+
* 2023, Confluent Inc.
|
|
6
|
+
* All rights reserved.
|
|
7
|
+
*
|
|
8
|
+
* Redistribution and use in source and binary forms, with or without
|
|
9
|
+
* modification, are permitted provided that the following conditions are met:
|
|
10
|
+
*
|
|
11
|
+
* 1. Redistributions of source code must retain the above copyright notice,
|
|
12
|
+
* this list of conditions and the following disclaimer.
|
|
13
|
+
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
14
|
+
* this list of conditions and the following disclaimer in the documentation
|
|
15
|
+
* and/or other materials provided with the distribution.
|
|
16
|
+
*
|
|
17
|
+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
18
|
+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
19
|
+
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
20
|
+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
|
21
|
+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
22
|
+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
23
|
+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
24
|
+
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
25
|
+
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
26
|
+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
27
|
+
* POSSIBILITY OF SUCH DAMAGE.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Mocks
|
|
32
|
+
*
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
#include "rdkafka_int.h"
|
|
36
|
+
#include "rdbuf.h"
|
|
37
|
+
#include "rdkafka_mock_int.h"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
static const char *rd_kafka_mock_cgrp_classic_state_names[] = {
|
|
41
|
+
"Empty", "Joining", "Syncing", "Rebalancing", "Up"};
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
static void
|
|
45
|
+
rd_kafka_mock_cgrp_classic_rebalance(rd_kafka_mock_cgrp_classic_t *mcgrp,
|
|
46
|
+
const char *reason);
|
|
47
|
+
static void rd_kafka_mock_cgrp_classic_member_destroy(
|
|
48
|
+
rd_kafka_mock_cgrp_classic_t *mcgrp,
|
|
49
|
+
rd_kafka_mock_cgrp_classic_member_t *member);
|
|
50
|
+
|
|
51
|
+
static void
|
|
52
|
+
rd_kafka_mock_cgrp_classic_set_state(rd_kafka_mock_cgrp_classic_t *mcgrp,
|
|
53
|
+
unsigned int new_state,
|
|
54
|
+
const char *reason) {
|
|
55
|
+
if (mcgrp->state == new_state)
|
|
56
|
+
return;
|
|
57
|
+
|
|
58
|
+
rd_kafka_dbg(mcgrp->cluster->rk, MOCK, "MOCK",
|
|
59
|
+
"Mock consumer group %s with %d member(s) "
|
|
60
|
+
"changing state %s -> %s: %s",
|
|
61
|
+
mcgrp->id, mcgrp->member_cnt,
|
|
62
|
+
rd_kafka_mock_cgrp_classic_state_names[mcgrp->state],
|
|
63
|
+
rd_kafka_mock_cgrp_classic_state_names[new_state], reason);
|
|
64
|
+
|
|
65
|
+
mcgrp->state = new_state;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* @brief Mark member as active (restart session timer)
|
|
71
|
+
*/
|
|
72
|
+
void rd_kafka_mock_cgrp_classic_member_active(
|
|
73
|
+
rd_kafka_mock_cgrp_classic_t *mcgrp,
|
|
74
|
+
rd_kafka_mock_cgrp_classic_member_t *member) {
|
|
75
|
+
rd_kafka_dbg(mcgrp->cluster->rk, MOCK, "MOCK",
|
|
76
|
+
"Marking mock consumer group member %s as active",
|
|
77
|
+
member->id);
|
|
78
|
+
member->ts_last_activity = rd_clock();
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* @brief Verify that the protocol request is valid in the current state.
|
|
84
|
+
*
|
|
85
|
+
* @param member may be NULL.
|
|
86
|
+
*/
|
|
87
|
+
rd_kafka_resp_err_t rd_kafka_mock_cgrp_classic_check_state(
|
|
88
|
+
rd_kafka_mock_cgrp_classic_t *mcgrp,
|
|
89
|
+
rd_kafka_mock_cgrp_classic_member_t *member,
|
|
90
|
+
const rd_kafka_buf_t *request,
|
|
91
|
+
int32_t generation_id) {
|
|
92
|
+
int16_t ApiKey = request->rkbuf_reqhdr.ApiKey;
|
|
93
|
+
rd_bool_t has_generation_id = ApiKey == RD_KAFKAP_SyncGroup ||
|
|
94
|
+
ApiKey == RD_KAFKAP_Heartbeat ||
|
|
95
|
+
ApiKey == RD_KAFKAP_OffsetCommit;
|
|
96
|
+
|
|
97
|
+
if (has_generation_id && generation_id != mcgrp->generation_id)
|
|
98
|
+
return RD_KAFKA_RESP_ERR_ILLEGAL_GENERATION;
|
|
99
|
+
|
|
100
|
+
if (ApiKey == RD_KAFKAP_OffsetCommit && !member)
|
|
101
|
+
return RD_KAFKA_RESP_ERR_UNKNOWN_MEMBER_ID;
|
|
102
|
+
|
|
103
|
+
switch (mcgrp->state) {
|
|
104
|
+
case RD_KAFKA_MOCK_CGRP_STATE_EMPTY:
|
|
105
|
+
if (ApiKey == RD_KAFKAP_JoinGroup)
|
|
106
|
+
return RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
107
|
+
break;
|
|
108
|
+
|
|
109
|
+
case RD_KAFKA_MOCK_CGRP_STATE_JOINING:
|
|
110
|
+
if (ApiKey == RD_KAFKAP_JoinGroup ||
|
|
111
|
+
ApiKey == RD_KAFKAP_LeaveGroup)
|
|
112
|
+
return RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
113
|
+
else
|
|
114
|
+
return RD_KAFKA_RESP_ERR_REBALANCE_IN_PROGRESS;
|
|
115
|
+
|
|
116
|
+
case RD_KAFKA_MOCK_CGRP_STATE_SYNCING:
|
|
117
|
+
if (ApiKey == RD_KAFKAP_SyncGroup ||
|
|
118
|
+
ApiKey == RD_KAFKAP_JoinGroup ||
|
|
119
|
+
ApiKey == RD_KAFKAP_LeaveGroup)
|
|
120
|
+
return RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
121
|
+
else
|
|
122
|
+
return RD_KAFKA_RESP_ERR_REBALANCE_IN_PROGRESS;
|
|
123
|
+
|
|
124
|
+
case RD_KAFKA_MOCK_CGRP_STATE_REBALANCING:
|
|
125
|
+
if (ApiKey == RD_KAFKAP_JoinGroup ||
|
|
126
|
+
ApiKey == RD_KAFKAP_LeaveGroup ||
|
|
127
|
+
ApiKey == RD_KAFKAP_OffsetCommit)
|
|
128
|
+
return RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
129
|
+
else
|
|
130
|
+
return RD_KAFKA_RESP_ERR_REBALANCE_IN_PROGRESS;
|
|
131
|
+
|
|
132
|
+
case RD_KAFKA_MOCK_CGRP_STATE_UP:
|
|
133
|
+
if (ApiKey == RD_KAFKAP_JoinGroup ||
|
|
134
|
+
ApiKey == RD_KAFKAP_LeaveGroup ||
|
|
135
|
+
ApiKey == RD_KAFKAP_Heartbeat ||
|
|
136
|
+
ApiKey == RD_KAFKAP_OffsetCommit)
|
|
137
|
+
return RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
138
|
+
break;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
return RD_KAFKA_RESP_ERR_INVALID_REQUEST;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* @brief Set a member's assignment (from leader's SyncGroupRequest)
|
|
147
|
+
*/
|
|
148
|
+
void rd_kafka_mock_cgrp_classic_member_assignment_set(
|
|
149
|
+
rd_kafka_mock_cgrp_classic_t *mcgrp,
|
|
150
|
+
rd_kafka_mock_cgrp_classic_member_t *member,
|
|
151
|
+
const rd_kafkap_bytes_t *Metadata) {
|
|
152
|
+
if (member->assignment) {
|
|
153
|
+
rd_assert(mcgrp->assignment_cnt > 0);
|
|
154
|
+
mcgrp->assignment_cnt--;
|
|
155
|
+
rd_kafkap_bytes_destroy(member->assignment);
|
|
156
|
+
member->assignment = NULL;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
if (Metadata) {
|
|
160
|
+
mcgrp->assignment_cnt++;
|
|
161
|
+
member->assignment = rd_kafkap_bytes_copy(Metadata);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* @brief Sync done (successfully) or failed, send responses back to members.
|
|
168
|
+
*/
|
|
169
|
+
static void
|
|
170
|
+
rd_kafka_mock_cgrp_classic_sync_done(rd_kafka_mock_cgrp_classic_t *mcgrp,
|
|
171
|
+
rd_kafka_resp_err_t err) {
|
|
172
|
+
rd_kafka_mock_cgrp_classic_member_t *member;
|
|
173
|
+
|
|
174
|
+
TAILQ_FOREACH(member, &mcgrp->members, link) {
|
|
175
|
+
rd_kafka_buf_t *resp;
|
|
176
|
+
|
|
177
|
+
if ((resp = member->resp)) {
|
|
178
|
+
member->resp = NULL;
|
|
179
|
+
rd_assert(resp->rkbuf_reqhdr.ApiKey ==
|
|
180
|
+
RD_KAFKAP_SyncGroup);
|
|
181
|
+
|
|
182
|
+
rd_kafka_buf_write_i16(resp, err); /* ErrorCode */
|
|
183
|
+
/* MemberState */
|
|
184
|
+
rd_kafka_buf_write_kbytes(
|
|
185
|
+
resp, !err ? member->assignment : NULL);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
rd_kafka_mock_cgrp_classic_member_assignment_set(mcgrp, member,
|
|
189
|
+
NULL);
|
|
190
|
+
|
|
191
|
+
if (member->conn) {
|
|
192
|
+
rd_kafka_mock_connection_set_blocking(member->conn,
|
|
193
|
+
rd_false);
|
|
194
|
+
if (resp)
|
|
195
|
+
rd_kafka_mock_connection_send_response(
|
|
196
|
+
member->conn, resp);
|
|
197
|
+
} else if (resp) {
|
|
198
|
+
/* Member has disconnected. */
|
|
199
|
+
rd_kafka_buf_destroy(resp);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* @brief Check if all members have sent SyncGroupRequests, if so, propagate
|
|
207
|
+
* assignment to members.
|
|
208
|
+
*/
|
|
209
|
+
static void
|
|
210
|
+
rd_kafka_mock_cgrp_classic_sync_check(rd_kafka_mock_cgrp_classic_t *mcgrp) {
|
|
211
|
+
|
|
212
|
+
rd_kafka_dbg(mcgrp->cluster->rk, MOCK, "MOCK",
|
|
213
|
+
"Mock consumer group %s: awaiting %d/%d syncing members "
|
|
214
|
+
"in state %s",
|
|
215
|
+
mcgrp->id, mcgrp->assignment_cnt, mcgrp->member_cnt,
|
|
216
|
+
rd_kafka_mock_cgrp_classic_state_names[mcgrp->state]);
|
|
217
|
+
|
|
218
|
+
if (mcgrp->assignment_cnt < mcgrp->member_cnt)
|
|
219
|
+
return;
|
|
220
|
+
|
|
221
|
+
rd_kafka_mock_cgrp_classic_sync_done(mcgrp, RD_KAFKA_RESP_ERR_NO_ERROR);
|
|
222
|
+
rd_kafka_mock_cgrp_classic_set_state(mcgrp, RD_KAFKA_MOCK_CGRP_STATE_UP,
|
|
223
|
+
"all members synced");
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* @brief Member has sent SyncGroupRequest and is waiting for a response,
|
|
229
|
+
* which will be sent when the all group member SyncGroupRequest are
|
|
230
|
+
* received.
|
|
231
|
+
*/
|
|
232
|
+
rd_kafka_resp_err_t rd_kafka_mock_cgrp_classic_member_sync_set(
|
|
233
|
+
rd_kafka_mock_cgrp_classic_t *mcgrp,
|
|
234
|
+
rd_kafka_mock_cgrp_classic_member_t *member,
|
|
235
|
+
rd_kafka_mock_connection_t *mconn,
|
|
236
|
+
rd_kafka_buf_t *resp) {
|
|
237
|
+
|
|
238
|
+
if (mcgrp->state != RD_KAFKA_MOCK_CGRP_STATE_SYNCING)
|
|
239
|
+
return RD_KAFKA_RESP_ERR_REBALANCE_IN_PROGRESS; /* FIXME */
|
|
240
|
+
|
|
241
|
+
rd_kafka_mock_cgrp_classic_member_active(mcgrp, member);
|
|
242
|
+
|
|
243
|
+
rd_assert(!member->resp);
|
|
244
|
+
|
|
245
|
+
member->resp = resp;
|
|
246
|
+
member->conn = mconn;
|
|
247
|
+
rd_kafka_mock_connection_set_blocking(member->conn, rd_true);
|
|
248
|
+
|
|
249
|
+
/* Check if all members now have an assignment, if so, send responses */
|
|
250
|
+
rd_kafka_mock_cgrp_classic_sync_check(mcgrp);
|
|
251
|
+
|
|
252
|
+
return RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* @brief Member is explicitly leaving the group (through LeaveGroupRequest)
|
|
258
|
+
*/
|
|
259
|
+
rd_kafka_resp_err_t rd_kafka_mock_cgrp_classic_member_leave(
|
|
260
|
+
rd_kafka_mock_cgrp_classic_t *mcgrp,
|
|
261
|
+
rd_kafka_mock_cgrp_classic_member_t *member) {
|
|
262
|
+
|
|
263
|
+
rd_kafka_dbg(mcgrp->cluster->rk, MOCK, "MOCK",
|
|
264
|
+
"Member %s is leaving group %s", member->id, mcgrp->id);
|
|
265
|
+
|
|
266
|
+
rd_kafka_mock_cgrp_classic_member_destroy(mcgrp, member);
|
|
267
|
+
|
|
268
|
+
rd_kafka_mock_cgrp_classic_rebalance(mcgrp, "explicit member leave");
|
|
269
|
+
|
|
270
|
+
return RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* @brief Destroys/frees an array of protocols, including the array itself.
|
|
275
|
+
*/
|
|
276
|
+
void rd_kafka_mock_cgrp_classic_protos_destroy(
|
|
277
|
+
rd_kafka_mock_cgrp_classic_proto_t *protos,
|
|
278
|
+
int proto_cnt) {
|
|
279
|
+
int i;
|
|
280
|
+
|
|
281
|
+
for (i = 0; i < proto_cnt; i++) {
|
|
282
|
+
rd_free(protos[i].name);
|
|
283
|
+
if (protos[i].metadata)
|
|
284
|
+
rd_free(protos[i].metadata);
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
rd_free(protos);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
static void rd_kafka_mock_cgrp_classic_rebalance_timer_restart(
|
|
291
|
+
rd_kafka_mock_cgrp_classic_t *mcgrp,
|
|
292
|
+
int timeout_ms);
|
|
293
|
+
|
|
294
|
+
/**
|
|
295
|
+
* @brief Elect consumer group leader and send JoinGroup responses
|
|
296
|
+
*/
|
|
297
|
+
static void
|
|
298
|
+
rd_kafka_mock_cgrp_classic_elect_leader(rd_kafka_mock_cgrp_classic_t *mcgrp) {
|
|
299
|
+
rd_kafka_mock_cgrp_classic_member_t *member;
|
|
300
|
+
|
|
301
|
+
rd_assert(mcgrp->state == RD_KAFKA_MOCK_CGRP_STATE_JOINING);
|
|
302
|
+
rd_assert(!TAILQ_EMPTY(&mcgrp->members));
|
|
303
|
+
|
|
304
|
+
mcgrp->generation_id++;
|
|
305
|
+
|
|
306
|
+
/* Elect a leader deterministically if the group.instance.id is
|
|
307
|
+
* available, using the lexicographic order of group.instance.ids.
|
|
308
|
+
* This is not how it's done on a real broker, which uses the first
|
|
309
|
+
* member joined. But we use a determinstic method for better testing,
|
|
310
|
+
* (in case we want to enforce a some consumer to be the group leader).
|
|
311
|
+
* If group.instance.id is not specified for any consumer, we use the
|
|
312
|
+
* first one joined, similar to the real broker. */
|
|
313
|
+
mcgrp->leader = NULL;
|
|
314
|
+
TAILQ_FOREACH(member, &mcgrp->members, link) {
|
|
315
|
+
if (!mcgrp->leader)
|
|
316
|
+
mcgrp->leader = member;
|
|
317
|
+
else if (mcgrp->leader->group_instance_id &&
|
|
318
|
+
member->group_instance_id &&
|
|
319
|
+
(rd_strcmp(mcgrp->leader->group_instance_id,
|
|
320
|
+
member->group_instance_id) > 0))
|
|
321
|
+
mcgrp->leader = member;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
rd_kafka_dbg(
|
|
325
|
+
mcgrp->cluster->rk, MOCK, "MOCK",
|
|
326
|
+
"Consumer group %s with %d member(s) is rebalancing: "
|
|
327
|
+
"elected leader is %s (group.instance.id = %s), generation id %d",
|
|
328
|
+
mcgrp->id, mcgrp->member_cnt, mcgrp->leader->id,
|
|
329
|
+
mcgrp->leader->group_instance_id, mcgrp->generation_id);
|
|
330
|
+
|
|
331
|
+
/* Find the most commonly supported protocol name among the members.
|
|
332
|
+
* FIXME: For now we'll blindly use the first protocol of the leader. */
|
|
333
|
+
if (mcgrp->protocol_name)
|
|
334
|
+
rd_free(mcgrp->protocol_name);
|
|
335
|
+
mcgrp->protocol_name = RD_KAFKAP_STR_DUP(mcgrp->leader->protos[0].name);
|
|
336
|
+
|
|
337
|
+
/* Send JoinGroupResponses to all members */
|
|
338
|
+
TAILQ_FOREACH(member, &mcgrp->members, link) {
|
|
339
|
+
rd_bool_t is_leader = member == mcgrp->leader;
|
|
340
|
+
int member_cnt = is_leader ? mcgrp->member_cnt : 0;
|
|
341
|
+
rd_kafka_buf_t *resp;
|
|
342
|
+
rd_kafka_mock_cgrp_classic_member_t *member2;
|
|
343
|
+
rd_kafka_mock_connection_t *mconn;
|
|
344
|
+
|
|
345
|
+
/* Member connection has been closed, it will eventually
|
|
346
|
+
* reconnect or time out from the group. */
|
|
347
|
+
if (!member->conn || !member->resp)
|
|
348
|
+
continue;
|
|
349
|
+
mconn = member->conn;
|
|
350
|
+
member->conn = NULL;
|
|
351
|
+
resp = member->resp;
|
|
352
|
+
member->resp = NULL;
|
|
353
|
+
|
|
354
|
+
rd_assert(resp->rkbuf_reqhdr.ApiKey == RD_KAFKAP_JoinGroup);
|
|
355
|
+
|
|
356
|
+
rd_kafka_buf_write_i16(resp, 0); /* ErrorCode */
|
|
357
|
+
rd_kafka_buf_write_i32(resp, mcgrp->generation_id);
|
|
358
|
+
rd_kafka_buf_write_str(resp, mcgrp->protocol_name, -1);
|
|
359
|
+
rd_kafka_buf_write_str(resp, mcgrp->leader->id, -1);
|
|
360
|
+
rd_kafka_buf_write_str(resp, member->id, -1);
|
|
361
|
+
rd_kafka_buf_write_i32(resp, member_cnt);
|
|
362
|
+
|
|
363
|
+
/* Send full member list to leader */
|
|
364
|
+
if (member_cnt > 0) {
|
|
365
|
+
TAILQ_FOREACH(member2, &mcgrp->members, link) {
|
|
366
|
+
rd_kafka_buf_write_str(resp, member2->id, -1);
|
|
367
|
+
if (resp->rkbuf_reqhdr.ApiVersion >= 5)
|
|
368
|
+
rd_kafka_buf_write_str(
|
|
369
|
+
resp, member2->group_instance_id,
|
|
370
|
+
-1);
|
|
371
|
+
/* FIXME: look up correct protocol name */
|
|
372
|
+
rd_assert(!rd_kafkap_str_cmp_str(
|
|
373
|
+
member2->protos[0].name,
|
|
374
|
+
mcgrp->protocol_name));
|
|
375
|
+
|
|
376
|
+
rd_kafka_buf_write_kbytes(
|
|
377
|
+
resp, member2->protos[0].metadata);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/* Mark each member as active to avoid them timing out
|
|
382
|
+
* at the same time as a JoinGroup handler that blocks
|
|
383
|
+
* session.timeout.ms to elect a leader. */
|
|
384
|
+
rd_kafka_mock_cgrp_classic_member_active(mcgrp, member);
|
|
385
|
+
|
|
386
|
+
rd_kafka_mock_connection_set_blocking(mconn, rd_false);
|
|
387
|
+
rd_kafka_mock_connection_send_response(mconn, resp);
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
mcgrp->last_member_cnt = mcgrp->member_cnt;
|
|
391
|
+
|
|
392
|
+
rd_kafka_mock_cgrp_classic_set_state(mcgrp,
|
|
393
|
+
RD_KAFKA_MOCK_CGRP_STATE_SYNCING,
|
|
394
|
+
"leader elected, waiting for all "
|
|
395
|
+
"members to sync");
|
|
396
|
+
|
|
397
|
+
rd_kafka_mock_cgrp_classic_rebalance_timer_restart(
|
|
398
|
+
mcgrp, mcgrp->session_timeout_ms);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
/**
|
|
403
|
+
* @brief Trigger group rebalance.
|
|
404
|
+
*/
|
|
405
|
+
static void
|
|
406
|
+
rd_kafka_mock_cgrp_classic_rebalance(rd_kafka_mock_cgrp_classic_t *mcgrp,
|
|
407
|
+
const char *reason) {
|
|
408
|
+
int timeout_ms;
|
|
409
|
+
|
|
410
|
+
if (mcgrp->state == RD_KAFKA_MOCK_CGRP_STATE_JOINING)
|
|
411
|
+
return; /* Do nothing, group is already rebalancing. */
|
|
412
|
+
else if (mcgrp->state == RD_KAFKA_MOCK_CGRP_STATE_EMPTY)
|
|
413
|
+
/* First join, low timeout.
|
|
414
|
+
* Same as group.initial.rebalance.delay.ms
|
|
415
|
+
* on the broker. */
|
|
416
|
+
timeout_ms =
|
|
417
|
+
mcgrp->cluster->defaults.group_initial_rebalance_delay_ms;
|
|
418
|
+
else if (mcgrp->state == RD_KAFKA_MOCK_CGRP_STATE_REBALANCING &&
|
|
419
|
+
mcgrp->member_cnt == mcgrp->last_member_cnt)
|
|
420
|
+
timeout_ms = 100; /* All members rejoined, quickly transition
|
|
421
|
+
* to election. */
|
|
422
|
+
else /* Let the rebalance delay be a bit shorter than the
|
|
423
|
+
* session timeout so that we don't time out waiting members
|
|
424
|
+
* who are also subject to the session timeout. */
|
|
425
|
+
timeout_ms = mcgrp->session_timeout_ms > 1000
|
|
426
|
+
? mcgrp->session_timeout_ms - 1000
|
|
427
|
+
: mcgrp->session_timeout_ms;
|
|
428
|
+
|
|
429
|
+
if (mcgrp->state == RD_KAFKA_MOCK_CGRP_STATE_SYNCING)
|
|
430
|
+
/* Abort current Syncing state */
|
|
431
|
+
rd_kafka_mock_cgrp_classic_sync_done(
|
|
432
|
+
mcgrp, RD_KAFKA_RESP_ERR_REBALANCE_IN_PROGRESS);
|
|
433
|
+
|
|
434
|
+
rd_kafka_mock_cgrp_classic_set_state(
|
|
435
|
+
mcgrp, RD_KAFKA_MOCK_CGRP_STATE_JOINING, reason);
|
|
436
|
+
rd_kafka_mock_cgrp_classic_rebalance_timer_restart(mcgrp, timeout_ms);
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/**
|
|
440
|
+
* @brief Consumer group state machine triggered by timer events.
|
|
441
|
+
*/
|
|
442
|
+
static void
|
|
443
|
+
rd_kafka_mock_cgrp_classic_fsm_timeout(rd_kafka_mock_cgrp_classic_t *mcgrp) {
|
|
444
|
+
rd_kafka_dbg(mcgrp->cluster->rk, MOCK, "MOCK",
|
|
445
|
+
"Mock consumer group %s FSM timeout in state %s",
|
|
446
|
+
mcgrp->id,
|
|
447
|
+
rd_kafka_mock_cgrp_classic_state_names[mcgrp->state]);
|
|
448
|
+
|
|
449
|
+
switch (mcgrp->state) {
|
|
450
|
+
case RD_KAFKA_MOCK_CGRP_STATE_EMPTY:
|
|
451
|
+
/* No members, do nothing */
|
|
452
|
+
break;
|
|
453
|
+
case RD_KAFKA_MOCK_CGRP_STATE_JOINING:
|
|
454
|
+
/* Timed out waiting for more members, elect a leader */
|
|
455
|
+
if (mcgrp->member_cnt > 0)
|
|
456
|
+
rd_kafka_mock_cgrp_classic_elect_leader(mcgrp);
|
|
457
|
+
else
|
|
458
|
+
rd_kafka_mock_cgrp_classic_set_state(
|
|
459
|
+
mcgrp, RD_KAFKA_MOCK_CGRP_STATE_EMPTY,
|
|
460
|
+
"no members joined");
|
|
461
|
+
break;
|
|
462
|
+
|
|
463
|
+
case RD_KAFKA_MOCK_CGRP_STATE_SYNCING:
|
|
464
|
+
/* Timed out waiting for all members to sync */
|
|
465
|
+
|
|
466
|
+
/* Send error response to all waiting members */
|
|
467
|
+
rd_kafka_mock_cgrp_classic_sync_done(
|
|
468
|
+
mcgrp, RD_KAFKA_RESP_ERR_REBALANCE_IN_PROGRESS /* FIXME */);
|
|
469
|
+
|
|
470
|
+
rd_kafka_mock_cgrp_classic_set_state(
|
|
471
|
+
mcgrp, RD_KAFKA_MOCK_CGRP_STATE_REBALANCING,
|
|
472
|
+
"timed out waiting for all members to synchronize");
|
|
473
|
+
break;
|
|
474
|
+
|
|
475
|
+
case RD_KAFKA_MOCK_CGRP_STATE_REBALANCING:
|
|
476
|
+
/* Timed out waiting for all members to Leave or re-Join */
|
|
477
|
+
rd_kafka_mock_cgrp_classic_set_state(
|
|
478
|
+
mcgrp, RD_KAFKA_MOCK_CGRP_STATE_JOINING,
|
|
479
|
+
"timed out waiting for all "
|
|
480
|
+
"members to re-Join or Leave");
|
|
481
|
+
break;
|
|
482
|
+
|
|
483
|
+
case RD_KAFKA_MOCK_CGRP_STATE_UP:
|
|
484
|
+
/* No fsm timers triggered in this state, see
|
|
485
|
+
* the session_tmr instead */
|
|
486
|
+
break;
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
static void rd_kafka_mcgrp_rebalance_timer_cb(rd_kafka_timers_t *rkts,
|
|
491
|
+
void *arg) {
|
|
492
|
+
rd_kafka_mock_cgrp_classic_t *mcgrp = arg;
|
|
493
|
+
|
|
494
|
+
rd_kafka_mock_cgrp_classic_fsm_timeout(mcgrp);
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
/**
|
|
499
|
+
* @brief Restart the rebalance timer, postponing leader election.
|
|
500
|
+
*/
|
|
501
|
+
static void rd_kafka_mock_cgrp_classic_rebalance_timer_restart(
|
|
502
|
+
rd_kafka_mock_cgrp_classic_t *mcgrp,
|
|
503
|
+
int timeout_ms) {
|
|
504
|
+
rd_kafka_timer_start_oneshot(
|
|
505
|
+
&mcgrp->cluster->timers, &mcgrp->rebalance_tmr, rd_true,
|
|
506
|
+
timeout_ms * 1000, rd_kafka_mcgrp_rebalance_timer_cb, mcgrp);
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
static void rd_kafka_mock_cgrp_classic_member_destroy(
|
|
511
|
+
rd_kafka_mock_cgrp_classic_t *mcgrp,
|
|
512
|
+
rd_kafka_mock_cgrp_classic_member_t *member) {
|
|
513
|
+
rd_assert(mcgrp->member_cnt > 0);
|
|
514
|
+
TAILQ_REMOVE(&mcgrp->members, member, link);
|
|
515
|
+
mcgrp->member_cnt--;
|
|
516
|
+
|
|
517
|
+
rd_free(member->id);
|
|
518
|
+
|
|
519
|
+
if (member->resp)
|
|
520
|
+
rd_kafka_buf_destroy(member->resp);
|
|
521
|
+
|
|
522
|
+
if (member->group_instance_id)
|
|
523
|
+
rd_free(member->group_instance_id);
|
|
524
|
+
|
|
525
|
+
rd_kafka_mock_cgrp_classic_member_assignment_set(mcgrp, member, NULL);
|
|
526
|
+
|
|
527
|
+
rd_kafka_mock_cgrp_classic_protos_destroy(member->protos,
|
|
528
|
+
member->proto_cnt);
|
|
529
|
+
|
|
530
|
+
rd_free(member);
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
/**
|
|
535
|
+
* @brief Find member in group.
|
|
536
|
+
*/
|
|
537
|
+
rd_kafka_mock_cgrp_classic_member_t *rd_kafka_mock_cgrp_classic_member_find(
|
|
538
|
+
const rd_kafka_mock_cgrp_classic_t *mcgrp,
|
|
539
|
+
const rd_kafkap_str_t *MemberId) {
|
|
540
|
+
const rd_kafka_mock_cgrp_classic_member_t *member;
|
|
541
|
+
TAILQ_FOREACH(member, &mcgrp->members, link) {
|
|
542
|
+
if (!rd_kafkap_str_cmp_str(MemberId, member->id))
|
|
543
|
+
return (rd_kafka_mock_cgrp_classic_member_t *)member;
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
return NULL;
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
/**
|
|
551
|
+
* @brief Update or add member to consumer group
|
|
552
|
+
*/
|
|
553
|
+
rd_kafka_resp_err_t rd_kafka_mock_cgrp_classic_member_add(
|
|
554
|
+
rd_kafka_mock_cgrp_classic_t *mcgrp,
|
|
555
|
+
rd_kafka_mock_connection_t *mconn,
|
|
556
|
+
rd_kafka_buf_t *resp,
|
|
557
|
+
const rd_kafkap_str_t *MemberId,
|
|
558
|
+
const rd_kafkap_str_t *ProtocolType,
|
|
559
|
+
const rd_kafkap_str_t *GroupInstanceId,
|
|
560
|
+
rd_kafka_mock_cgrp_classic_proto_t *protos,
|
|
561
|
+
int proto_cnt,
|
|
562
|
+
int session_timeout_ms) {
|
|
563
|
+
rd_kafka_mock_cgrp_classic_member_t *member;
|
|
564
|
+
rd_kafka_resp_err_t err;
|
|
565
|
+
|
|
566
|
+
err = rd_kafka_mock_cgrp_classic_check_state(mcgrp, NULL, resp, -1);
|
|
567
|
+
if (err)
|
|
568
|
+
return err;
|
|
569
|
+
|
|
570
|
+
/* Find member */
|
|
571
|
+
member = rd_kafka_mock_cgrp_classic_member_find(mcgrp, MemberId);
|
|
572
|
+
if (!member) {
|
|
573
|
+
/* Not found, add member */
|
|
574
|
+
member = rd_calloc(1, sizeof(*member));
|
|
575
|
+
|
|
576
|
+
if (!RD_KAFKAP_STR_LEN(MemberId)) {
|
|
577
|
+
/* Generate a member id */
|
|
578
|
+
char memberid[32];
|
|
579
|
+
rd_snprintf(memberid, sizeof(memberid), "%p", member);
|
|
580
|
+
member->id = rd_strdup(memberid);
|
|
581
|
+
} else
|
|
582
|
+
member->id = RD_KAFKAP_STR_DUP(MemberId);
|
|
583
|
+
|
|
584
|
+
if (RD_KAFKAP_STR_LEN(GroupInstanceId))
|
|
585
|
+
member->group_instance_id =
|
|
586
|
+
RD_KAFKAP_STR_DUP(GroupInstanceId);
|
|
587
|
+
|
|
588
|
+
TAILQ_INSERT_TAIL(&mcgrp->members, member, link);
|
|
589
|
+
mcgrp->member_cnt++;
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
if (mcgrp->state != RD_KAFKA_MOCK_CGRP_STATE_JOINING)
|
|
593
|
+
rd_kafka_mock_cgrp_classic_rebalance(mcgrp, "member join");
|
|
594
|
+
|
|
595
|
+
mcgrp->session_timeout_ms = session_timeout_ms;
|
|
596
|
+
|
|
597
|
+
if (member->protos)
|
|
598
|
+
rd_kafka_mock_cgrp_classic_protos_destroy(member->protos,
|
|
599
|
+
member->proto_cnt);
|
|
600
|
+
member->protos = protos;
|
|
601
|
+
member->proto_cnt = proto_cnt;
|
|
602
|
+
|
|
603
|
+
rd_assert(!member->resp);
|
|
604
|
+
member->resp = resp;
|
|
605
|
+
member->conn = mconn;
|
|
606
|
+
rd_kafka_mock_cgrp_classic_member_active(mcgrp, member);
|
|
607
|
+
|
|
608
|
+
return RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
/**
|
|
612
|
+
* @brief Check if any members have exceeded the session timeout.
|
|
613
|
+
*/
|
|
614
|
+
static void rd_kafka_mock_cgrp_classic_session_tmr_cb(rd_kafka_timers_t *rkts,
|
|
615
|
+
void *arg) {
|
|
616
|
+
rd_kafka_mock_cgrp_classic_t *mcgrp = arg;
|
|
617
|
+
rd_kafka_mock_cgrp_classic_member_t *member, *tmp;
|
|
618
|
+
rd_ts_t now = rd_clock();
|
|
619
|
+
int timeout_cnt = 0;
|
|
620
|
+
|
|
621
|
+
TAILQ_FOREACH_SAFE(member, &mcgrp->members, link, tmp) {
|
|
622
|
+
if (member->ts_last_activity +
|
|
623
|
+
(mcgrp->session_timeout_ms * 1000) >
|
|
624
|
+
now)
|
|
625
|
+
continue;
|
|
626
|
+
|
|
627
|
+
rd_kafka_dbg(mcgrp->cluster->rk, MOCK, "MOCK",
|
|
628
|
+
"Member %s session timed out for group %s",
|
|
629
|
+
member->id, mcgrp->id);
|
|
630
|
+
|
|
631
|
+
rd_kafka_mock_cgrp_classic_member_destroy(mcgrp, member);
|
|
632
|
+
timeout_cnt++;
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
if (timeout_cnt)
|
|
636
|
+
rd_kafka_mock_cgrp_classic_rebalance(mcgrp, "member timeout");
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
void rd_kafka_mock_cgrp_classic_destroy(rd_kafka_mock_cgrp_classic_t *mcgrp) {
|
|
641
|
+
rd_kafka_mock_cgrp_classic_member_t *member;
|
|
642
|
+
|
|
643
|
+
TAILQ_REMOVE(&mcgrp->cluster->cgrps_classic, mcgrp, link);
|
|
644
|
+
|
|
645
|
+
rd_kafka_timer_stop(&mcgrp->cluster->timers, &mcgrp->rebalance_tmr,
|
|
646
|
+
rd_true);
|
|
647
|
+
rd_kafka_timer_stop(&mcgrp->cluster->timers, &mcgrp->session_tmr,
|
|
648
|
+
rd_true);
|
|
649
|
+
rd_free(mcgrp->id);
|
|
650
|
+
rd_free(mcgrp->protocol_type);
|
|
651
|
+
if (mcgrp->protocol_name)
|
|
652
|
+
rd_free(mcgrp->protocol_name);
|
|
653
|
+
while ((member = TAILQ_FIRST(&mcgrp->members)))
|
|
654
|
+
rd_kafka_mock_cgrp_classic_member_destroy(mcgrp, member);
|
|
655
|
+
rd_free(mcgrp);
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
rd_kafka_mock_cgrp_classic_t *
|
|
660
|
+
rd_kafka_mock_cgrp_classic_find(rd_kafka_mock_cluster_t *mcluster,
|
|
661
|
+
const rd_kafkap_str_t *GroupId) {
|
|
662
|
+
rd_kafka_mock_cgrp_classic_t *mcgrp;
|
|
663
|
+
TAILQ_FOREACH(mcgrp, &mcluster->cgrps_classic, link) {
|
|
664
|
+
if (!rd_kafkap_str_cmp_str(GroupId, mcgrp->id))
|
|
665
|
+
return mcgrp;
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
return NULL;
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
|
|
672
|
+
/**
|
|
673
|
+
* @brief Find or create a classic consumer group
|
|
674
|
+
*/
|
|
675
|
+
rd_kafka_mock_cgrp_classic_t *
|
|
676
|
+
rd_kafka_mock_cgrp_classic_get(rd_kafka_mock_cluster_t *mcluster,
|
|
677
|
+
const rd_kafkap_str_t *GroupId,
|
|
678
|
+
const rd_kafkap_str_t *ProtocolType) {
|
|
679
|
+
rd_kafka_mock_cgrp_classic_t *mcgrp;
|
|
680
|
+
|
|
681
|
+
mcgrp = rd_kafka_mock_cgrp_classic_find(mcluster, GroupId);
|
|
682
|
+
if (mcgrp)
|
|
683
|
+
return mcgrp;
|
|
684
|
+
|
|
685
|
+
/* FIXME: What to do with mismatching ProtocolTypes? */
|
|
686
|
+
|
|
687
|
+
mcgrp = rd_calloc(1, sizeof(*mcgrp));
|
|
688
|
+
|
|
689
|
+
mcgrp->cluster = mcluster;
|
|
690
|
+
mcgrp->id = RD_KAFKAP_STR_DUP(GroupId);
|
|
691
|
+
mcgrp->protocol_type = RD_KAFKAP_STR_DUP(ProtocolType);
|
|
692
|
+
mcgrp->generation_id = 1;
|
|
693
|
+
TAILQ_INIT(&mcgrp->members);
|
|
694
|
+
rd_kafka_timer_start(&mcluster->timers, &mcgrp->session_tmr,
|
|
695
|
+
1000 * 1000 /*1s*/,
|
|
696
|
+
rd_kafka_mock_cgrp_classic_session_tmr_cb, mcgrp);
|
|
697
|
+
|
|
698
|
+
TAILQ_INSERT_TAIL(&mcluster->cgrps_classic, mcgrp, link);
|
|
699
|
+
|
|
700
|
+
return mcgrp;
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
|
|
704
|
+
/**
|
|
705
|
+
* @brief A client connection closed, check if any classic cgrp has any state
|
|
706
|
+
* for this connection that needs to be cleared.
|
|
707
|
+
*/
|
|
708
|
+
void rd_kafka_mock_cgrps_classic_connection_closed(
|
|
709
|
+
rd_kafka_mock_cluster_t *mcluster,
|
|
710
|
+
rd_kafka_mock_connection_t *mconn) {
|
|
711
|
+
rd_kafka_mock_cgrp_classic_t *mcgrp;
|
|
712
|
+
|
|
713
|
+
TAILQ_FOREACH(mcgrp, &mcluster->cgrps_classic, link) {
|
|
714
|
+
rd_kafka_mock_cgrp_classic_member_t *member, *tmp;
|
|
715
|
+
TAILQ_FOREACH_SAFE(member, &mcgrp->members, link, tmp) {
|
|
716
|
+
if (member->conn == mconn) {
|
|
717
|
+
member->conn = NULL;
|
|
718
|
+
if (member->resp) {
|
|
719
|
+
rd_kafka_buf_destroy(member->resp);
|
|
720
|
+
member->resp = NULL;
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
/**
|
|
728
|
+
* @struct Target assignment for a consumer group.
|
|
729
|
+
* `member_ids` and `assignment` are in the same order
|
|
730
|
+
* and have the same count.
|
|
731
|
+
*/
|
|
732
|
+
typedef struct rd_kafka_mock_cgrp_consumer_target_assignment_s {
|
|
733
|
+
rd_list_t *member_ids; /**< Member id list (char *). */
|
|
734
|
+
rd_list_t *assignment; /**< Assingment list
|
|
735
|
+
(rd_kafka_topic_partition_list_t *). */
|
|
736
|
+
} rd_kafka_mock_cgrp_consumer_target_assignment_t;
|
|
737
|
+
|
|
738
|
+
static rd_kafka_mock_cgrp_consumer_target_assignment_t *
|
|
739
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_new0(rd_list_t *member_ids,
|
|
740
|
+
rd_list_t *assignment) {
|
|
741
|
+
rd_assert(member_ids->rl_cnt == assignment->rl_cnt);
|
|
742
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_t *target_assignment =
|
|
743
|
+
rd_calloc(1, sizeof(*target_assignment));
|
|
744
|
+
target_assignment->member_ids =
|
|
745
|
+
rd_list_copy(member_ids, rd_list_string_copy, NULL);
|
|
746
|
+
target_assignment->assignment = rd_list_copy(
|
|
747
|
+
assignment, rd_kafka_topic_partition_list_copy_opaque, NULL);
|
|
748
|
+
return target_assignment;
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_t *
|
|
752
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_new(
|
|
753
|
+
char **member_ids,
|
|
754
|
+
int member_cnt,
|
|
755
|
+
rd_kafka_topic_partition_list_t **assignment) {
|
|
756
|
+
int i;
|
|
757
|
+
rd_list_t *member_id_list, *assignment_list;
|
|
758
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_t *ret;
|
|
759
|
+
|
|
760
|
+
member_id_list = rd_list_new(member_cnt, rd_free);
|
|
761
|
+
assignment_list =
|
|
762
|
+
rd_list_new(member_cnt, rd_kafka_topic_partition_list_destroy_free);
|
|
763
|
+
for (i = 0; i < member_cnt; i++) {
|
|
764
|
+
rd_list_add(member_id_list, rd_strdup(member_ids[i]));
|
|
765
|
+
rd_list_add(assignment_list,
|
|
766
|
+
rd_kafka_topic_partition_list_copy(assignment[i]));
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
ret = rd_kafka_mock_cgrp_consumer_target_assignment_new0(
|
|
770
|
+
member_id_list, assignment_list);
|
|
771
|
+
rd_list_destroy(member_id_list);
|
|
772
|
+
rd_list_destroy(assignment_list);
|
|
773
|
+
return ret;
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
void rd_kafka_mock_cgrp_consumer_target_assignment_destroy(
|
|
777
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_t *target_assignment) {
|
|
778
|
+
rd_list_destroy(target_assignment->member_ids);
|
|
779
|
+
rd_list_destroy(target_assignment->assignment);
|
|
780
|
+
rd_free(target_assignment);
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
/**
|
|
784
|
+
* @brief Sets next target assignment and member epoch for \p member
|
|
785
|
+
* to a copy of partition list \p rktparlist,
|
|
786
|
+
* filling its topic ids if not provided, using \p cgrp cluster topics.
|
|
787
|
+
*
|
|
788
|
+
* @param mcgrp The consumer group containing the member.
|
|
789
|
+
* @param member A consumer group member.
|
|
790
|
+
* @param target_member_epoch New member epoch.
|
|
791
|
+
* @param rktparlist Next target assignment.
|
|
792
|
+
*
|
|
793
|
+
* @locks mcluster->lock MUST be held.
|
|
794
|
+
*/
|
|
795
|
+
static void rd_kafka_mock_cgrp_consumer_member_target_assignment_set(
|
|
796
|
+
rd_kafka_mock_cgrp_consumer_t *mcgrp,
|
|
797
|
+
rd_kafka_mock_cgrp_consumer_member_t *member,
|
|
798
|
+
int target_member_epoch,
|
|
799
|
+
const rd_kafka_topic_partition_list_t *rktparlist) {
|
|
800
|
+
rd_kafka_topic_partition_t *rktpar;
|
|
801
|
+
if (member->target_assignment) {
|
|
802
|
+
rd_kafka_topic_partition_list_destroy(
|
|
803
|
+
member->target_assignment);
|
|
804
|
+
}
|
|
805
|
+
member->target_member_epoch = target_member_epoch;
|
|
806
|
+
member->target_assignment =
|
|
807
|
+
rd_kafka_topic_partition_list_copy(rktparlist);
|
|
808
|
+
|
|
809
|
+
/* If not present, fill topic ids using names */
|
|
810
|
+
RD_KAFKA_TPLIST_FOREACH(rktpar, member->target_assignment) {
|
|
811
|
+
rd_kafka_Uuid_t topic_id =
|
|
812
|
+
rd_kafka_topic_partition_get_topic_id(rktpar);
|
|
813
|
+
if (!rd_kafka_Uuid_cmp(topic_id, RD_KAFKA_UUID_ZERO)) {
|
|
814
|
+
rd_kafka_mock_topic_t *mtopic =
|
|
815
|
+
rd_kafka_mock_topic_find(mcgrp->cluster,
|
|
816
|
+
rktpar->topic);
|
|
817
|
+
if (mtopic)
|
|
818
|
+
rd_kafka_topic_partition_set_topic_id(
|
|
819
|
+
rktpar, mtopic->id);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
/**
|
|
825
|
+
* @brief Sets next target assignment for group \p mcgrp
|
|
826
|
+
* to a copy of \p target_assignment partition lists.
|
|
827
|
+
*
|
|
828
|
+
* @param mcgrp The consumer group.
|
|
829
|
+
* @param target_assignment Target assignment for all members.
|
|
830
|
+
*
|
|
831
|
+
* @locks mcluster->lock MUST be held.
|
|
832
|
+
*/
|
|
833
|
+
static void rd_kafka_mock_cgrp_consumer_target_assignment_set(
|
|
834
|
+
rd_kafka_mock_cgrp_consumer_t *mcgrp,
|
|
835
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_t *target_assignment) {
|
|
836
|
+
int i = 0;
|
|
837
|
+
int32_t new_target_member_epoch;
|
|
838
|
+
const char *member_id;
|
|
839
|
+
rd_kafka_mock_cgrp_consumer_member_t *member;
|
|
840
|
+
|
|
841
|
+
mcgrp->group_epoch++;
|
|
842
|
+
new_target_member_epoch = mcgrp->group_epoch;
|
|
843
|
+
RD_LIST_FOREACH(member_id, target_assignment->member_ids, i) {
|
|
844
|
+
rd_kafkap_str_t *member_id_str =
|
|
845
|
+
rd_kafkap_str_new(member_id, strlen(member_id));
|
|
846
|
+
rd_kafka_topic_partition_list_t *member_assignment =
|
|
847
|
+
rd_list_elem(target_assignment->assignment, i);
|
|
848
|
+
member = rd_kafka_mock_cgrp_consumer_member_find(mcgrp,
|
|
849
|
+
member_id_str);
|
|
850
|
+
rd_kafkap_str_destroy(member_id_str);
|
|
851
|
+
|
|
852
|
+
if (!member)
|
|
853
|
+
continue;
|
|
854
|
+
|
|
855
|
+
rd_kafka_mock_cgrp_consumer_member_target_assignment_set(
|
|
856
|
+
mcgrp, member, new_target_member_epoch, member_assignment);
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
typedef RD_MAP_TYPE(const char *, rd_list_t *) map_str_list;
|
|
861
|
+
typedef RD_MAP_TYPE(const char *, int *) map_str_int;
|
|
862
|
+
|
|
863
|
+
/**
|
|
864
|
+
* @brief Calculate a simple range target assignment for the consumer group \p
|
|
865
|
+
* mcgrp. This isn't replicating any given broker assignor but is used
|
|
866
|
+
* when the test doesn't need a specific type of assignment.
|
|
867
|
+
*
|
|
868
|
+
* If the test needs it, instead of replicating same conditions with all the
|
|
869
|
+
* members, one can mock the assignment directly with
|
|
870
|
+
* `rd_kafka_mock_cgrp_consumer_target_assignment`.
|
|
871
|
+
*/
|
|
872
|
+
static rd_kafka_mock_cgrp_consumer_target_assignment_t *
|
|
873
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_calculate_range(
|
|
874
|
+
const rd_kafka_mock_cgrp_consumer_t *mcgrp) {
|
|
875
|
+
int i, *i_pointer;
|
|
876
|
+
const char *topic;
|
|
877
|
+
rd_list_t *members;
|
|
878
|
+
rd_kafka_mock_cgrp_consumer_member_t *member;
|
|
879
|
+
rd_kafka_mock_cluster_t *mcluster = mcgrp->cluster;
|
|
880
|
+
/* List of member ids (char *) */
|
|
881
|
+
rd_list_t *member_ids = rd_list_new(mcgrp->member_cnt, rd_free);
|
|
882
|
+
/* List of member assignment (rd_kafka_topic_partition_list_t *) */
|
|
883
|
+
rd_list_t *assignment = rd_list_new(
|
|
884
|
+
mcgrp->member_cnt, rd_kafka_topic_partition_list_destroy_free);
|
|
885
|
+
/* Map from topic name to list of members */
|
|
886
|
+
map_str_list topic_members =
|
|
887
|
+
RD_MAP_INITIALIZER(mcgrp->member_cnt, rd_map_str_cmp,
|
|
888
|
+
rd_map_str_hash, NULL, rd_list_destroy_free);
|
|
889
|
+
/* Map from member id to index in the members and assignment lists. */
|
|
890
|
+
map_str_int member_idx = RD_MAP_INITIALIZER(
|
|
891
|
+
mcgrp->member_cnt, rd_map_str_cmp, rd_map_str_hash, NULL, rd_free);
|
|
892
|
+
|
|
893
|
+
i = 0;
|
|
894
|
+
|
|
895
|
+
/* First create a map with topics associated to the list of members
|
|
896
|
+
* and save the member idx in the `member_idx` map. */
|
|
897
|
+
TAILQ_FOREACH(member, &mcgrp->members, link) {
|
|
898
|
+
int j;
|
|
899
|
+
rd_list_add(member_ids, rd_strdup(member->id));
|
|
900
|
+
rd_list_add(assignment, rd_kafka_topic_partition_list_new(0));
|
|
901
|
+
|
|
902
|
+
RD_LIST_FOREACH(topic, member->subscribed_topics, j) {
|
|
903
|
+
if (!RD_MAP_GET(&topic_members, topic)) {
|
|
904
|
+
members = rd_list_new(0, NULL);
|
|
905
|
+
RD_MAP_SET(&topic_members, topic, members);
|
|
906
|
+
} else
|
|
907
|
+
members = RD_MAP_GET(&topic_members, topic);
|
|
908
|
+
rd_list_add(members, member);
|
|
909
|
+
}
|
|
910
|
+
i_pointer = rd_calloc(1, sizeof(*i_pointer));
|
|
911
|
+
*i_pointer = i;
|
|
912
|
+
RD_MAP_SET(&member_idx, member->id, i_pointer);
|
|
913
|
+
i++;
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
/* For each topic to a range assignment and add the
|
|
917
|
+
* corresponding partitions to the assignment for that member.
|
|
918
|
+
* Finds the list index using the `member_idx` map. */
|
|
919
|
+
RD_MAP_FOREACH(topic, members, &topic_members) {
|
|
920
|
+
rd_kafka_Uuid_t topic_id;
|
|
921
|
+
rd_kafka_topic_partition_list_t *member_assignment;
|
|
922
|
+
int members_cnt = rd_list_cnt(members);
|
|
923
|
+
int common, one_more, assigned = 0;
|
|
924
|
+
rd_kafkap_str_t Topic = {.str = topic, .len = strlen(topic)};
|
|
925
|
+
rd_kafka_mock_topic_t *mock_topic =
|
|
926
|
+
rd_kafka_mock_topic_find_by_kstr(mcluster, &Topic);
|
|
927
|
+
if (!mock_topic)
|
|
928
|
+
continue;
|
|
929
|
+
|
|
930
|
+
topic_id = mock_topic->id;
|
|
931
|
+
|
|
932
|
+
/* Assign one partition more
|
|
933
|
+
* to the first mock_topic->partition_cnt % members_cnt
|
|
934
|
+
* members. */
|
|
935
|
+
common = mock_topic->partition_cnt / members_cnt;
|
|
936
|
+
one_more = mock_topic->partition_cnt % members_cnt;
|
|
937
|
+
|
|
938
|
+
RD_LIST_FOREACH(member, members, i) {
|
|
939
|
+
int j, num_partitions = common;
|
|
940
|
+
int idx = *RD_MAP_GET(&member_idx, member->id);
|
|
941
|
+
member_assignment = rd_list_elem(assignment, idx);
|
|
942
|
+
if (idx < one_more)
|
|
943
|
+
num_partitions++;
|
|
944
|
+
for (j = 0; j < num_partitions; j++) {
|
|
945
|
+
rd_kafka_topic_partition_t *rktpar =
|
|
946
|
+
rd_kafka_topic_partition_list_add(
|
|
947
|
+
member_assignment, topic, assigned + j);
|
|
948
|
+
rd_kafka_topic_partition_set_topic_id(rktpar,
|
|
949
|
+
topic_id);
|
|
950
|
+
}
|
|
951
|
+
assigned += num_partitions;
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_t *ret =
|
|
956
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_new0(member_ids,
|
|
957
|
+
assignment);
|
|
958
|
+
|
|
959
|
+
RD_MAP_DESTROY(&topic_members);
|
|
960
|
+
RD_MAP_DESTROY(&member_idx);
|
|
961
|
+
|
|
962
|
+
rd_list_destroy(member_ids);
|
|
963
|
+
rd_list_destroy(assignment);
|
|
964
|
+
|
|
965
|
+
return ret;
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
/**
|
|
969
|
+
* @brief Recalculate and set a target assignment for \p mcgrp
|
|
970
|
+
* only if `mcgrp->manual_assignment` isn't set.
|
|
971
|
+
*
|
|
972
|
+
* @locks mcluster->lock MUST be held.
|
|
973
|
+
*/
|
|
974
|
+
static void rd_kafka_mock_cgrp_consumer_target_assignment_recalculate(
|
|
975
|
+
rd_kafka_mock_cgrp_consumer_t *mcgrp) {
|
|
976
|
+
if (mcgrp->manual_assignment)
|
|
977
|
+
return;
|
|
978
|
+
|
|
979
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_t *target_assignment =
|
|
980
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_calculate_range(
|
|
981
|
+
mcgrp);
|
|
982
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_set(mcgrp,
|
|
983
|
+
target_assignment);
|
|
984
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_destroy(
|
|
985
|
+
target_assignment);
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
/**
|
|
989
|
+
* @brief Set manual target assignment \p target_assignment
|
|
990
|
+
* to the consumer group \p mcgrp .
|
|
991
|
+
*
|
|
992
|
+
* @param mcgrp Consumer group
|
|
993
|
+
* @param target_assignment Target assignment to set.
|
|
994
|
+
* Pass NULL to return to automatic assignment.
|
|
995
|
+
*
|
|
996
|
+
* @locks mcluster->lock MUST be held.
|
|
997
|
+
*/
|
|
998
|
+
static void rd_kafka_mock_cgrp_consumer_target_assignment_set_manual(
|
|
999
|
+
rd_kafka_mock_cgrp_consumer_t *mcgrp,
|
|
1000
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_t *target_assignment) {
|
|
1001
|
+
if (!target_assignment) {
|
|
1002
|
+
mcgrp->manual_assignment = rd_false;
|
|
1003
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_recalculate(
|
|
1004
|
+
mcgrp);
|
|
1005
|
+
return;
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
mcgrp->manual_assignment = rd_true;
|
|
1009
|
+
|
|
1010
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_set(mcgrp,
|
|
1011
|
+
target_assignment);
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
/**
|
|
1015
|
+
* @brief Sets \p member current assignment to a copy of
|
|
1016
|
+
* \p current_assignment.
|
|
1017
|
+
*
|
|
1018
|
+
* @param member A consumer group member.
|
|
1019
|
+
* @param current_assignment Current assignment to set.
|
|
1020
|
+
*
|
|
1021
|
+
* @locks mcluster->lock MUST be held.
|
|
1022
|
+
*/
|
|
1023
|
+
static void rd_kafka_mock_cgrp_consumer_member_current_assignment_set(
|
|
1024
|
+
rd_kafka_mock_cgrp_consumer_member_t *member,
|
|
1025
|
+
const rd_kafka_topic_partition_list_t *current_assignment) {
|
|
1026
|
+
if (member->current_assignment) {
|
|
1027
|
+
rd_kafka_topic_partition_list_destroy(
|
|
1028
|
+
member->current_assignment);
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
member->current_assignment =
|
|
1032
|
+
current_assignment
|
|
1033
|
+
? rd_kafka_topic_partition_list_copy(current_assignment)
|
|
1034
|
+
: NULL;
|
|
1035
|
+
}
|
|
1036
|
+
|
|
1037
|
+
/**
|
|
1038
|
+
* @brief Sets \p member returned assignment to a
|
|
1039
|
+
* copy of \p returned_assignment.
|
|
1040
|
+
*
|
|
1041
|
+
* @param member A consumer group member.
|
|
1042
|
+
* @param returned_assignment Returned assignment to set.
|
|
1043
|
+
*
|
|
1044
|
+
* @locks mcluster->lock MUST be held.
|
|
1045
|
+
*/
|
|
1046
|
+
static void rd_kafka_mock_cgrp_consumer_member_returned_assignment_set(
|
|
1047
|
+
rd_kafka_mock_cgrp_consumer_member_t *member,
|
|
1048
|
+
const rd_kafka_topic_partition_list_t *returned_assignment) {
|
|
1049
|
+
if (member->returned_assignment) {
|
|
1050
|
+
rd_kafka_topic_partition_list_destroy(
|
|
1051
|
+
member->returned_assignment);
|
|
1052
|
+
}
|
|
1053
|
+
member->returned_assignment =
|
|
1054
|
+
returned_assignment
|
|
1055
|
+
? rd_kafka_topic_partition_list_copy(returned_assignment)
|
|
1056
|
+
: NULL;
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
/**
|
|
1060
|
+
* @brief Returns a copy of \p member target assignment containing only
|
|
1061
|
+
* partitions that can be assignment, whose topic id is non-zero.
|
|
1062
|
+
*
|
|
1063
|
+
* @param member The group member.
|
|
1064
|
+
*
|
|
1065
|
+
* @remark The returned pointer ownership is transferred to the caller.
|
|
1066
|
+
*
|
|
1067
|
+
* @locks mcluster->lock MUST be held.
|
|
1068
|
+
*/
|
|
1069
|
+
static rd_kafka_topic_partition_list_t *
|
|
1070
|
+
rd_kafka_mock_cgrp_consumer_member_target_assignment_assignable(
|
|
1071
|
+
rd_kafka_mock_cgrp_consumer_member_t *member) {
|
|
1072
|
+
rd_kafka_topic_partition_list_t *assignment = member->target_assignment;
|
|
1073
|
+
rd_kafka_topic_partition_t *rktpar;
|
|
1074
|
+
rd_kafka_topic_partition_list_t *ret =
|
|
1075
|
+
rd_kafka_topic_partition_list_new(assignment->cnt);
|
|
1076
|
+
|
|
1077
|
+
RD_KAFKA_TPLIST_FOREACH(rktpar, assignment) {
|
|
1078
|
+
rd_kafka_Uuid_t topic_id =
|
|
1079
|
+
rd_kafka_topic_partition_get_topic_id(rktpar);
|
|
1080
|
+
if (rd_kafka_Uuid_cmp(topic_id, RD_KAFKA_UUID_ZERO)) {
|
|
1081
|
+
rd_kafka_topic_partition_list_add_copy(ret, rktpar);
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
return ret;
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
/**
|
|
1089
|
+
* Returns true iff \p new_assignment doesn't have any intersection with any
|
|
1090
|
+
* other member current assignment.
|
|
1091
|
+
*
|
|
1092
|
+
* If there's an intersection, it means we cannot bump the epoch at the moment,
|
|
1093
|
+
* because some of these partitions are held by a different member. They have
|
|
1094
|
+
* to be revoked from that member before it's possible to increase the epoch
|
|
1095
|
+
* and assign additional partitions to this member.
|
|
1096
|
+
*/
|
|
1097
|
+
rd_bool_t rd_kafka_mock_cgrp_consumer_member_next_assignment_can_bump_epoch(
|
|
1098
|
+
rd_kafka_mock_cgrp_consumer_member_t *member,
|
|
1099
|
+
rd_kafka_topic_partition_list_t *new_assignment) {
|
|
1100
|
+
rd_kafka_topic_partition_list_t *double_assignment,
|
|
1101
|
+
*assigned_partitions = rd_kafka_topic_partition_list_new(0);
|
|
1102
|
+
rd_kafka_mock_cgrp_consumer_member_t *other_member;
|
|
1103
|
+
rd_kafka_mock_cgrp_consumer_t *mcgrp = member->mcgrp;
|
|
1104
|
+
rd_bool_t ret;
|
|
1105
|
+
|
|
1106
|
+
TAILQ_FOREACH(other_member, &mcgrp->members, link) {
|
|
1107
|
+
int other_current_assignment_cnt = 0,
|
|
1108
|
+
other_returned_assignment_cnt = 0;
|
|
1109
|
+
if (member == other_member)
|
|
1110
|
+
continue;
|
|
1111
|
+
if (other_member->current_assignment)
|
|
1112
|
+
other_current_assignment_cnt =
|
|
1113
|
+
other_member->current_assignment->cnt;
|
|
1114
|
+
if (other_member->returned_assignment)
|
|
1115
|
+
other_returned_assignment_cnt =
|
|
1116
|
+
other_member->returned_assignment->cnt;
|
|
1117
|
+
|
|
1118
|
+
if (other_current_assignment_cnt > 0 &&
|
|
1119
|
+
other_current_assignment_cnt >
|
|
1120
|
+
other_returned_assignment_cnt) {
|
|
1121
|
+
/* This is the case where we're revoking
|
|
1122
|
+
* some partitions.
|
|
1123
|
+
* returned_assignment < current_assignment. */
|
|
1124
|
+
rd_kafka_topic_partition_list_add_list(
|
|
1125
|
+
assigned_partitions,
|
|
1126
|
+
other_member->current_assignment);
|
|
1127
|
+
} else if (other_returned_assignment_cnt > 0) {
|
|
1128
|
+
/* This is the case where we're assigning
|
|
1129
|
+
* some partitions.
|
|
1130
|
+
* returned_assignment >= current_assignment. */
|
|
1131
|
+
rd_kafka_topic_partition_list_add_list(
|
|
1132
|
+
assigned_partitions,
|
|
1133
|
+
other_member->returned_assignment);
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
double_assignment = rd_kafka_topic_partition_list_intersection_by_id(
|
|
1137
|
+
new_assignment, assigned_partitions);
|
|
1138
|
+
ret = double_assignment->cnt == 0;
|
|
1139
|
+
|
|
1140
|
+
rd_kafka_topic_partition_list_destroy(assigned_partitions);
|
|
1141
|
+
rd_kafka_topic_partition_list_destroy(double_assignment);
|
|
1142
|
+
return ret;
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
/**
|
|
1146
|
+
* @brief Calculates if \p member,
|
|
1147
|
+
* needs a revocation, that is if its current assignment
|
|
1148
|
+
* isn't a subset of its target assignment.
|
|
1149
|
+
* In case it needs a revocation, it returns
|
|
1150
|
+
* the intersection between the two assignments,
|
|
1151
|
+
* that is the remaining partitions after revocation
|
|
1152
|
+
* of those not included in target assignment.
|
|
1153
|
+
*
|
|
1154
|
+
* @param member The group member.
|
|
1155
|
+
*
|
|
1156
|
+
* @return The remaining set of partitions, or NULL in case no revocation
|
|
1157
|
+
* is needed.
|
|
1158
|
+
*
|
|
1159
|
+
* @remark The returned pointer ownership is transferred to the caller.
|
|
1160
|
+
*
|
|
1161
|
+
* @locks mcluster->lock MUST be held.
|
|
1162
|
+
*/
|
|
1163
|
+
static rd_kafka_topic_partition_list_t *
|
|
1164
|
+
rd_kafka_mock_cgrp_consumer_member_needs_revocation(
|
|
1165
|
+
rd_kafka_mock_cgrp_consumer_member_t *member) {
|
|
1166
|
+
rd_kafka_topic_partition_list_t *intersection;
|
|
1167
|
+
rd_bool_t needs_revocation;
|
|
1168
|
+
|
|
1169
|
+
if (member->current_assignment)
|
|
1170
|
+
/* If we have a current assignment we
|
|
1171
|
+
* calculate the intersection with
|
|
1172
|
+
* target assignment. */
|
|
1173
|
+
intersection = rd_kafka_topic_partition_list_intersection_by_id(
|
|
1174
|
+
member->current_assignment, member->target_assignment);
|
|
1175
|
+
else
|
|
1176
|
+
/* Otherwise intersection is empty. */
|
|
1177
|
+
intersection = rd_kafka_topic_partition_list_new(0);
|
|
1178
|
+
|
|
1179
|
+
needs_revocation = member->current_assignment &&
|
|
1180
|
+
intersection->cnt < member->current_assignment->cnt;
|
|
1181
|
+
if (needs_revocation) {
|
|
1182
|
+
return intersection;
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
rd_kafka_topic_partition_list_destroy(intersection);
|
|
1186
|
+
return NULL;
|
|
1187
|
+
}
|
|
1188
|
+
|
|
1189
|
+
/**
|
|
1190
|
+
* @brief Calculates if \p member,
|
|
1191
|
+
* can receive new partitions, given revocation is completed.
|
|
1192
|
+
* In case new partitions aren't held by other members it
|
|
1193
|
+
* returns the assignable target assignment and bumps current
|
|
1194
|
+
* member epoch, otherwise it returns NULL and
|
|
1195
|
+
* doesn't change current member epoch.
|
|
1196
|
+
*
|
|
1197
|
+
* @param member The group member.
|
|
1198
|
+
*
|
|
1199
|
+
* @return The assignable set of partitions, or NULL in case new partitions
|
|
1200
|
+
* cannot be assigned yet.
|
|
1201
|
+
*
|
|
1202
|
+
* @remark The returned pointer ownership is transferred to the caller.
|
|
1203
|
+
*
|
|
1204
|
+
* @locks mcluster->lock MUST be held.
|
|
1205
|
+
*/
|
|
1206
|
+
static rd_kafka_topic_partition_list_t *
|
|
1207
|
+
rd_kafka_mock_cgrp_consumer_member_needs_assignment(
|
|
1208
|
+
rd_kafka_mock_cgrp_consumer_member_t *member) {
|
|
1209
|
+
rd_kafka_topic_partition_list_t *returned_assignment =
|
|
1210
|
+
rd_kafka_mock_cgrp_consumer_member_target_assignment_assignable(
|
|
1211
|
+
member);
|
|
1212
|
+
|
|
1213
|
+
if (!rd_kafka_mock_cgrp_consumer_member_next_assignment_can_bump_epoch(
|
|
1214
|
+
member, returned_assignment)) {
|
|
1215
|
+
/* We can't bump the epoch still,
|
|
1216
|
+
* there are some partitions held by other members.
|
|
1217
|
+
* We have to return NULL. */
|
|
1218
|
+
rd_kafka_topic_partition_list_destroy(returned_assignment);
|
|
1219
|
+
return NULL;
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
/* No partitions to remove, return
|
|
1223
|
+
* target assignment and reconcile the
|
|
1224
|
+
* epochs */
|
|
1225
|
+
member->current_member_epoch = member->target_member_epoch;
|
|
1226
|
+
return returned_assignment;
|
|
1227
|
+
}
|
|
1228
|
+
|
|
1229
|
+
/**
|
|
1230
|
+
* @brief Calculates next assignment and member epoch for a \p member,
|
|
1231
|
+
* given \p current_assignment.
|
|
1232
|
+
*
|
|
1233
|
+
* @param member The group member.
|
|
1234
|
+
* @param current_assignment The assignment sent by the member, or NULL if it
|
|
1235
|
+
* didn't change. Must be NULL if *member_epoch is 0.
|
|
1236
|
+
* @param member_epoch Pointer to client reported member epoch. Can be updated.
|
|
1237
|
+
*
|
|
1238
|
+
* @return The new assignment to return to the member.
|
|
1239
|
+
*
|
|
1240
|
+
* @remark The returned pointer ownership is transferred to the caller.
|
|
1241
|
+
*
|
|
1242
|
+
* @locks mcluster->lock MUST be held.
|
|
1243
|
+
*/
|
|
1244
|
+
rd_kafka_topic_partition_list_t *
|
|
1245
|
+
rd_kafka_mock_cgrp_consumer_member_next_assignment(
|
|
1246
|
+
rd_kafka_mock_cgrp_consumer_member_t *member,
|
|
1247
|
+
rd_kafka_topic_partition_list_t *current_assignment,
|
|
1248
|
+
int *member_epoch) {
|
|
1249
|
+
rd_kafka_topic_partition_list_t *assignment_to_return = NULL;
|
|
1250
|
+
|
|
1251
|
+
if (current_assignment) {
|
|
1252
|
+
/* Update current assignment to reflect what is provided
|
|
1253
|
+
* by the client. */
|
|
1254
|
+
rd_kafka_mock_cgrp_consumer_member_current_assignment_set(
|
|
1255
|
+
member, current_assignment);
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
if (*member_epoch > 0 &&
|
|
1259
|
+
member->current_member_epoch != *member_epoch) {
|
|
1260
|
+
/* Member epoch is different from the one we expect,
|
|
1261
|
+
* that means we have to fence the member. */
|
|
1262
|
+
*member_epoch = -1; /* FENCED_MEMBER_EPOCH */
|
|
1263
|
+
return NULL;
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
if (member->target_assignment) {
|
|
1267
|
+
/* We have a target assignment,
|
|
1268
|
+
* let's check if we can assign it. */
|
|
1269
|
+
|
|
1270
|
+
if (*member_epoch != member->current_member_epoch ||
|
|
1271
|
+
member->current_member_epoch !=
|
|
1272
|
+
member->target_member_epoch) {
|
|
1273
|
+
/* Epochs are different, that means we have to bump the
|
|
1274
|
+
* epoch immediately or do some revocations
|
|
1275
|
+
* before that. */
|
|
1276
|
+
|
|
1277
|
+
assignment_to_return =
|
|
1278
|
+
rd_kafka_mock_cgrp_consumer_member_needs_revocation(
|
|
1279
|
+
member);
|
|
1280
|
+
if (!assignment_to_return) {
|
|
1281
|
+
/* After revocation we only have to
|
|
1282
|
+
* add new partitions.
|
|
1283
|
+
* In case these new partitions are held
|
|
1284
|
+
* by other members we still cannot do it. */
|
|
1285
|
+
assignment_to_return =
|
|
1286
|
+
rd_kafka_mock_cgrp_consumer_member_needs_assignment(
|
|
1287
|
+
member);
|
|
1288
|
+
}
|
|
1289
|
+
} else if (!member->returned_assignment) {
|
|
1290
|
+
/* If all the epochs are the same, the only case
|
|
1291
|
+
* where we have to return the assignment is
|
|
1292
|
+
* after a disconnection, when returned_assignment has
|
|
1293
|
+
* been reset to NULL. */
|
|
1294
|
+
assignment_to_return =
|
|
1295
|
+
rd_kafka_mock_cgrp_consumer_member_target_assignment_assignable(
|
|
1296
|
+
member);
|
|
1297
|
+
}
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1300
|
+
*member_epoch = member->current_member_epoch;
|
|
1301
|
+
if (assignment_to_return) {
|
|
1302
|
+
/* Compare assignment_to_return with last returned_assignment.
|
|
1303
|
+
* If equal, return NULL, otherwise return assignment_to_return
|
|
1304
|
+
* and update last returned_assignment. */
|
|
1305
|
+
rd_bool_t same_returned_assignment =
|
|
1306
|
+
member->returned_assignment &&
|
|
1307
|
+
!rd_kafka_topic_partition_list_cmp(
|
|
1308
|
+
member->returned_assignment, assignment_to_return,
|
|
1309
|
+
rd_kafka_topic_partition_by_id_cmp);
|
|
1310
|
+
|
|
1311
|
+
if (same_returned_assignment) {
|
|
1312
|
+
/* Returned assignment is the same as previous
|
|
1313
|
+
* one, we return NULL instead to show no change. */
|
|
1314
|
+
rd_kafka_topic_partition_list_destroy(
|
|
1315
|
+
assignment_to_return);
|
|
1316
|
+
assignment_to_return = NULL;
|
|
1317
|
+
} else {
|
|
1318
|
+
/* We store returned assignment
|
|
1319
|
+
* for later comparison. */
|
|
1320
|
+
rd_kafka_mock_cgrp_consumer_member_returned_assignment_set(
|
|
1321
|
+
member, assignment_to_return);
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
return assignment_to_return;
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
/**
|
|
1328
|
+
* @brief Mark member as active (restart session timer).
|
|
1329
|
+
*
|
|
1330
|
+
* @param mcgrp Member's consumer group.
|
|
1331
|
+
* @param member Member to set as active.
|
|
1332
|
+
*
|
|
1333
|
+
* @locks mcluster->lock MUST be held.
|
|
1334
|
+
*/
|
|
1335
|
+
void rd_kafka_mock_cgrp_consumer_member_active(
|
|
1336
|
+
rd_kafka_mock_cgrp_consumer_t *mcgrp,
|
|
1337
|
+
rd_kafka_mock_cgrp_consumer_member_t *member) {
|
|
1338
|
+
rd_kafka_dbg(mcgrp->cluster->rk, MOCK, "MOCK",
|
|
1339
|
+
"Marking mock consumer group member %s as active",
|
|
1340
|
+
member->id);
|
|
1341
|
+
member->ts_last_activity = rd_clock();
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
/**
|
|
1345
|
+
* @brief Finds a member in consumer group \p mcgrp by \p MemberId.
|
|
1346
|
+
*
|
|
1347
|
+
* @param mcgrp Consumer group to search.
|
|
1348
|
+
* @param MemberId Member id to look for.
|
|
1349
|
+
* @return Found member or NULL.
|
|
1350
|
+
*
|
|
1351
|
+
* @locks mcluster->lock MUST be held.
|
|
1352
|
+
*/
|
|
1353
|
+
rd_kafka_mock_cgrp_consumer_member_t *rd_kafka_mock_cgrp_consumer_member_find(
|
|
1354
|
+
const rd_kafka_mock_cgrp_consumer_t *mcgrp,
|
|
1355
|
+
const rd_kafkap_str_t *MemberId) {
|
|
1356
|
+
const rd_kafka_mock_cgrp_consumer_member_t *member;
|
|
1357
|
+
TAILQ_FOREACH(member, &mcgrp->members, link) {
|
|
1358
|
+
if (!rd_kafkap_str_cmp_str(MemberId, member->id))
|
|
1359
|
+
return (rd_kafka_mock_cgrp_consumer_member_t *)member;
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
return NULL;
|
|
1363
|
+
}
|
|
1364
|
+
|
|
1365
|
+
/**
|
|
1366
|
+
* @brief Finds a member in consumer group \p mcgrp by \p InstanceId.
|
|
1367
|
+
*
|
|
1368
|
+
* @param mcgrp Consumer group to search.
|
|
1369
|
+
* @param InstanceId Instance id to look for.
|
|
1370
|
+
* @return Found member or NULL.
|
|
1371
|
+
*
|
|
1372
|
+
* @locks mcluster->lock MUST be held.
|
|
1373
|
+
*/
|
|
1374
|
+
rd_kafka_mock_cgrp_consumer_member_t *
|
|
1375
|
+
rd_kafka_mock_cgrp_consumer_member_find_by_instance_id(
|
|
1376
|
+
const rd_kafka_mock_cgrp_consumer_t *mcgrp,
|
|
1377
|
+
const rd_kafkap_str_t *InstanceId) {
|
|
1378
|
+
if (RD_KAFKAP_STR_IS_NULL(InstanceId))
|
|
1379
|
+
return NULL;
|
|
1380
|
+
|
|
1381
|
+
const rd_kafka_mock_cgrp_consumer_member_t *member;
|
|
1382
|
+
TAILQ_FOREACH(member, &mcgrp->members, link) {
|
|
1383
|
+
if (!member->instance_id)
|
|
1384
|
+
continue;
|
|
1385
|
+
|
|
1386
|
+
if (!rd_kafkap_str_cmp_str(InstanceId, member->instance_id))
|
|
1387
|
+
return (rd_kafka_mock_cgrp_consumer_member_t *)member;
|
|
1388
|
+
}
|
|
1389
|
+
|
|
1390
|
+
return NULL;
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
static void validate_subscription(const rd_kafkap_str_t *SubscribedTopicNames,
|
|
1394
|
+
int32_t SubscribedTopicNamesCnt,
|
|
1395
|
+
const rd_kafkap_str_t *SubscribedTopicRegex) {
|
|
1396
|
+
/* Either they are both NULL
|
|
1397
|
+
* or both non-NULL. */
|
|
1398
|
+
rd_assert((SubscribedTopicNames == NULL) ==
|
|
1399
|
+
RD_KAFKAP_STR_IS_NULL(SubscribedTopicRegex));
|
|
1400
|
+
/* If they're not NULL at least one should be non-empty */
|
|
1401
|
+
rd_assert(SubscribedTopicNames == NULL || SubscribedTopicNamesCnt > 0 ||
|
|
1402
|
+
RD_KAFKAP_STR_LEN(SubscribedTopicRegex) > 0);
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
/**
|
|
1406
|
+
* @brief Set the subscribed topics for the member \p member based on \p
|
|
1407
|
+
* SubscribedTopicNames and \p SubscribedTopicRegex. Deduplicates the list after
|
|
1408
|
+
* sorting it.
|
|
1409
|
+
* @return `rd_true` if the subscription was changed, that happens
|
|
1410
|
+
* if it's set and different from previous one.
|
|
1411
|
+
*
|
|
1412
|
+
* @locks mcluster->lock MUST be held.
|
|
1413
|
+
*/
|
|
1414
|
+
static rd_bool_t rd_kafka_mock_cgrp_consumer_member_subscribed_topic_names_set(
|
|
1415
|
+
rd_kafka_mock_cgrp_consumer_member_t *member,
|
|
1416
|
+
rd_kafkap_str_t *SubscribedTopicNames,
|
|
1417
|
+
int32_t SubscribedTopicNamesCnt,
|
|
1418
|
+
const rd_kafkap_str_t *SubscribedTopicRegex) {
|
|
1419
|
+
rd_bool_t changed = rd_false;
|
|
1420
|
+
rd_list_t *new_subscription;
|
|
1421
|
+
int32_t i;
|
|
1422
|
+
|
|
1423
|
+
validate_subscription(SubscribedTopicNames, SubscribedTopicNamesCnt,
|
|
1424
|
+
SubscribedTopicRegex);
|
|
1425
|
+
|
|
1426
|
+
if (!SubscribedTopicNames &&
|
|
1427
|
+
RD_KAFKAP_STR_IS_NULL(SubscribedTopicRegex) &&
|
|
1428
|
+
!member->subscribed_topic_regex) {
|
|
1429
|
+
/* When client is sending NULL for SubscribedTopicNames and
|
|
1430
|
+
* SubscribedTopicRegex, its subscription didn't change. If we
|
|
1431
|
+
* already had a regex, we need to compute the regex again. */
|
|
1432
|
+
return changed;
|
|
1433
|
+
}
|
|
1434
|
+
|
|
1435
|
+
if (SubscribedTopicNames) {
|
|
1436
|
+
RD_IF_FREE(member->subscribed_topic_names, rd_list_destroy);
|
|
1437
|
+
member->subscribed_topic_names =
|
|
1438
|
+
rd_list_new(SubscribedTopicNamesCnt, rd_free);
|
|
1439
|
+
for (i = 0; i < SubscribedTopicNamesCnt; i++) {
|
|
1440
|
+
rd_list_add(
|
|
1441
|
+
member->subscribed_topic_names,
|
|
1442
|
+
RD_KAFKAP_STR_DUP(&SubscribedTopicNames[i]));
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
if (!RD_KAFKAP_STR_IS_NULL(SubscribedTopicRegex)) {
|
|
1447
|
+
RD_IF_FREE(member->subscribed_topic_regex, rd_free);
|
|
1448
|
+
member->subscribed_topic_regex =
|
|
1449
|
+
RD_KAFKAP_STR_DUP(SubscribedTopicRegex);
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
new_subscription =
|
|
1453
|
+
rd_list_new(rd_list_cnt(member->subscribed_topic_names), rd_free);
|
|
1454
|
+
|
|
1455
|
+
rd_list_copy_to(new_subscription, member->subscribed_topic_names,
|
|
1456
|
+
rd_list_string_copy, NULL);
|
|
1457
|
+
|
|
1458
|
+
if (member->subscribed_topic_regex[0]) {
|
|
1459
|
+
rd_kafka_mock_cluster_t *mcluster = member->mcgrp->cluster;
|
|
1460
|
+
rd_kafka_mock_topic_t *mtopic;
|
|
1461
|
+
char errstr[1];
|
|
1462
|
+
rd_regex_t *re = rd_regex_comp(member->subscribed_topic_regex,
|
|
1463
|
+
errstr, sizeof(errstr));
|
|
1464
|
+
|
|
1465
|
+
TAILQ_FOREACH(mtopic, &mcluster->topics, link) {
|
|
1466
|
+
if (rd_regex_exec(re, mtopic->name))
|
|
1467
|
+
rd_list_add(new_subscription,
|
|
1468
|
+
rd_strdup(mtopic->name));
|
|
1469
|
+
}
|
|
1470
|
+
|
|
1471
|
+
rd_regex_destroy(re);
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1474
|
+
rd_list_deduplicate(&new_subscription, rd_strcmp2);
|
|
1475
|
+
|
|
1476
|
+
if (!member->subscribed_topics ||
|
|
1477
|
+
rd_list_cmp(new_subscription, member->subscribed_topics,
|
|
1478
|
+
rd_list_cmp_str)) {
|
|
1479
|
+
if (member->subscribed_topics)
|
|
1480
|
+
rd_list_destroy(member->subscribed_topics);
|
|
1481
|
+
member->subscribed_topics =
|
|
1482
|
+
rd_list_copy(new_subscription, rd_list_string_copy, NULL);
|
|
1483
|
+
changed = rd_true;
|
|
1484
|
+
}
|
|
1485
|
+
rd_list_destroy(new_subscription);
|
|
1486
|
+
return changed;
|
|
1487
|
+
}
|
|
1488
|
+
|
|
1489
|
+
static void rd_kafka_mock_cgrp_consumer_member_topic_id_set(
|
|
1490
|
+
rd_kafka_mock_cgrp_consumer_member_t *member,
|
|
1491
|
+
const rd_kafkap_str_t *MemberId) {
|
|
1492
|
+
/* KIP 1082: MemberId is generated by the client */
|
|
1493
|
+
rd_assert(RD_KAFKAP_STR_LEN(MemberId) > 0);
|
|
1494
|
+
RD_IF_FREE(member->id, rd_free);
|
|
1495
|
+
member->id = RD_KAFKAP_STR_DUP(MemberId);
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
/**
|
|
1499
|
+
* @brief Adds a member to consumer group \p mcgrp. If member with same
|
|
1500
|
+
* \p MemberId is already present, only updates the connection and
|
|
1501
|
+
* sets it as active.
|
|
1502
|
+
*
|
|
1503
|
+
* @param mcgrp Consumer group to add the member to.
|
|
1504
|
+
* @param conn Member connection.
|
|
1505
|
+
* @param MemberId Member id.
|
|
1506
|
+
* @param InstanceId Group instance id (optional).
|
|
1507
|
+
* @param session_timeout_ms Session timeout to use.
|
|
1508
|
+
* @param SubscribedTopicNames Array of subscribed topics.
|
|
1509
|
+
* Mandatory if the member is a new one.
|
|
1510
|
+
* @param SubscribedTopicNamesCnt Number of elements in \p SubscribedTopicNames.
|
|
1511
|
+
* @param SubscribedTopicRegex Subscribed topic regex.
|
|
1512
|
+
*
|
|
1513
|
+
* @return New or existing member, NULL if the member cannot be added.
|
|
1514
|
+
*
|
|
1515
|
+
* @locks mcluster->lock MUST be held.
|
|
1516
|
+
*/
|
|
1517
|
+
rd_kafka_mock_cgrp_consumer_member_t *rd_kafka_mock_cgrp_consumer_member_add(
|
|
1518
|
+
rd_kafka_mock_cgrp_consumer_t *mcgrp,
|
|
1519
|
+
struct rd_kafka_mock_connection_s *conn,
|
|
1520
|
+
const rd_kafkap_str_t *MemberId,
|
|
1521
|
+
const rd_kafkap_str_t *InstanceId,
|
|
1522
|
+
rd_kafkap_str_t *SubscribedTopicNames,
|
|
1523
|
+
int32_t SubscribedTopicNamesCnt,
|
|
1524
|
+
const rd_kafkap_str_t *SubscribedTopicRegex) {
|
|
1525
|
+
rd_kafka_mock_cgrp_consumer_member_t *member = NULL;
|
|
1526
|
+
rd_bool_t changed = rd_false;
|
|
1527
|
+
|
|
1528
|
+
/* Find member */
|
|
1529
|
+
member = rd_kafka_mock_cgrp_consumer_member_find(mcgrp, MemberId);
|
|
1530
|
+
if (!member) {
|
|
1531
|
+
member = rd_kafka_mock_cgrp_consumer_member_find_by_instance_id(
|
|
1532
|
+
mcgrp, InstanceId);
|
|
1533
|
+
|
|
1534
|
+
if (member) {
|
|
1535
|
+
if (!member->left_static_membership) {
|
|
1536
|
+
/* Old member still active,
|
|
1537
|
+
* fence this one */
|
|
1538
|
+
return NULL;
|
|
1539
|
+
}
|
|
1540
|
+
|
|
1541
|
+
if (rd_kafkap_str_cmp_str(MemberId, member->id) != 0) {
|
|
1542
|
+
/* Member is a new instance and is rejoining
|
|
1543
|
+
* with a new MemberId. */
|
|
1544
|
+
rd_kafka_mock_cgrp_consumer_member_topic_id_set(
|
|
1545
|
+
member, MemberId);
|
|
1546
|
+
}
|
|
1547
|
+
member->left_static_membership = rd_false;
|
|
1548
|
+
}
|
|
1549
|
+
} else {
|
|
1550
|
+
member->left_static_membership = rd_false;
|
|
1551
|
+
}
|
|
1552
|
+
|
|
1553
|
+
if (!member) {
|
|
1554
|
+
validate_subscription(SubscribedTopicNames,
|
|
1555
|
+
SubscribedTopicNamesCnt,
|
|
1556
|
+
SubscribedTopicRegex);
|
|
1557
|
+
|
|
1558
|
+
/* In case of session timeout
|
|
1559
|
+
* where the member isn't aware it's been fenced. */
|
|
1560
|
+
if (SubscribedTopicNames == NULL)
|
|
1561
|
+
return NULL;
|
|
1562
|
+
|
|
1563
|
+
/* Not found, add member */
|
|
1564
|
+
member = rd_calloc(1, sizeof(*member));
|
|
1565
|
+
member->mcgrp = mcgrp;
|
|
1566
|
+
|
|
1567
|
+
rd_kafka_mock_cgrp_consumer_member_topic_id_set(member,
|
|
1568
|
+
MemberId);
|
|
1569
|
+
|
|
1570
|
+
if (!RD_KAFKAP_STR_IS_NULL(InstanceId))
|
|
1571
|
+
member->instance_id = RD_KAFKAP_STR_DUP(InstanceId);
|
|
1572
|
+
|
|
1573
|
+
TAILQ_INSERT_TAIL(&mcgrp->members, member, link);
|
|
1574
|
+
mcgrp->member_cnt++;
|
|
1575
|
+
changed = rd_true;
|
|
1576
|
+
member->target_member_epoch = mcgrp->group_epoch;
|
|
1577
|
+
}
|
|
1578
|
+
|
|
1579
|
+
changed |=
|
|
1580
|
+
rd_kafka_mock_cgrp_consumer_member_subscribed_topic_names_set(
|
|
1581
|
+
member, SubscribedTopicNames, SubscribedTopicNamesCnt,
|
|
1582
|
+
SubscribedTopicRegex);
|
|
1583
|
+
|
|
1584
|
+
mcgrp->session_timeout_ms =
|
|
1585
|
+
mcgrp->cluster->defaults.group_consumer_session_timeout_ms;
|
|
1586
|
+
mcgrp->heartbeat_interval_ms =
|
|
1587
|
+
mcgrp->cluster->defaults.group_consumer_heartbeat_interval_ms;
|
|
1588
|
+
|
|
1589
|
+
member->conn = conn;
|
|
1590
|
+
|
|
1591
|
+
rd_kafka_mock_cgrp_consumer_member_active(mcgrp, member);
|
|
1592
|
+
|
|
1593
|
+
if (changed)
|
|
1594
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_recalculate(
|
|
1595
|
+
mcgrp);
|
|
1596
|
+
|
|
1597
|
+
return member;
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
/**
|
|
1601
|
+
* @brief Destroys a consumer group member, removing from its consumer group.
|
|
1602
|
+
*
|
|
1603
|
+
* @param mcgrp Member consumer group.
|
|
1604
|
+
* @param member Member to destroy.
|
|
1605
|
+
*
|
|
1606
|
+
* @locks mcluster->lock MUST be held.
|
|
1607
|
+
*/
|
|
1608
|
+
static void rd_kafka_mock_cgrp_consumer_member_destroy(
|
|
1609
|
+
rd_kafka_mock_cgrp_consumer_t *mcgrp,
|
|
1610
|
+
rd_kafka_mock_cgrp_consumer_member_t *member) {
|
|
1611
|
+
rd_assert(mcgrp->member_cnt > 0);
|
|
1612
|
+
TAILQ_REMOVE(&mcgrp->members, member, link);
|
|
1613
|
+
mcgrp->member_cnt--;
|
|
1614
|
+
|
|
1615
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_recalculate(mcgrp);
|
|
1616
|
+
|
|
1617
|
+
rd_free(member->id);
|
|
1618
|
+
|
|
1619
|
+
if (member->instance_id)
|
|
1620
|
+
rd_free(member->instance_id);
|
|
1621
|
+
|
|
1622
|
+
RD_IF_FREE(member->target_assignment,
|
|
1623
|
+
rd_kafka_topic_partition_list_destroy);
|
|
1624
|
+
RD_IF_FREE(member->current_assignment,
|
|
1625
|
+
rd_kafka_topic_partition_list_destroy);
|
|
1626
|
+
RD_IF_FREE(member->returned_assignment,
|
|
1627
|
+
rd_kafka_topic_partition_list_destroy);
|
|
1628
|
+
RD_IF_FREE(member->subscribed_topics, rd_list_destroy_free);
|
|
1629
|
+
|
|
1630
|
+
RD_IF_FREE(member->subscribed_topic_names, rd_list_destroy_free);
|
|
1631
|
+
|
|
1632
|
+
RD_IF_FREE(member->subscribed_topic_regex, rd_free);
|
|
1633
|
+
|
|
1634
|
+
rd_free(member);
|
|
1635
|
+
}
|
|
1636
|
+
|
|
1637
|
+
static void rd_kafka_mock_cgrp_consumer_member_leave_static(
|
|
1638
|
+
rd_kafka_mock_cgrp_consumer_member_t *member) {
|
|
1639
|
+
member->left_static_membership = rd_true;
|
|
1640
|
+
rd_kafka_mock_cgrp_consumer_member_returned_assignment_set(member,
|
|
1641
|
+
NULL);
|
|
1642
|
+
}
|
|
1643
|
+
|
|
1644
|
+
|
|
1645
|
+
/**
|
|
1646
|
+
* @brief Called when a member must leave a consumer group.
|
|
1647
|
+
*
|
|
1648
|
+
* @param mcgrp Consumer group to leave.
|
|
1649
|
+
* @param member Member that leaves.
|
|
1650
|
+
* @param leave_static If true, the member is leaving with static group
|
|
1651
|
+
* membership.
|
|
1652
|
+
*
|
|
1653
|
+
* @locks mcluster->lock MUST be held.
|
|
1654
|
+
*/
|
|
1655
|
+
void rd_kafka_mock_cgrp_consumer_member_leave(
|
|
1656
|
+
rd_kafka_mock_cgrp_consumer_t *mcgrp,
|
|
1657
|
+
rd_kafka_mock_cgrp_consumer_member_t *member,
|
|
1658
|
+
rd_bool_t leave_static) {
|
|
1659
|
+
rd_bool_t is_static = member->instance_id != NULL;
|
|
1660
|
+
|
|
1661
|
+
rd_kafka_dbg(mcgrp->cluster->rk, MOCK, "MOCK",
|
|
1662
|
+
"Member %s is leaving group %s, is static: %s, "
|
|
1663
|
+
"static leave: %s",
|
|
1664
|
+
member->id, mcgrp->id, RD_STR_ToF(is_static),
|
|
1665
|
+
RD_STR_ToF(leave_static));
|
|
1666
|
+
if (!is_static || !leave_static)
|
|
1667
|
+
rd_kafka_mock_cgrp_consumer_member_destroy(mcgrp, member);
|
|
1668
|
+
else
|
|
1669
|
+
rd_kafka_mock_cgrp_consumer_member_leave_static(member);
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
/**
|
|
1673
|
+
* @brief Called when a member is fenced from a consumer group.
|
|
1674
|
+
*
|
|
1675
|
+
* @param mcgrp Consumer group.
|
|
1676
|
+
* @param member Member to fence.
|
|
1677
|
+
*
|
|
1678
|
+
* @locks mcluster->lock MUST be held.
|
|
1679
|
+
*/
|
|
1680
|
+
void rd_kafka_mock_cgrp_consumer_member_fenced(
|
|
1681
|
+
rd_kafka_mock_cgrp_consumer_t *mcgrp,
|
|
1682
|
+
rd_kafka_mock_cgrp_consumer_member_t *member) {
|
|
1683
|
+
|
|
1684
|
+
rd_kafka_dbg(mcgrp->cluster->rk, MOCK, "MOCK",
|
|
1685
|
+
"Member %s is fenced from group %s", member->id,
|
|
1686
|
+
mcgrp->id);
|
|
1687
|
+
|
|
1688
|
+
rd_kafka_mock_cgrp_consumer_member_destroy(mcgrp, member);
|
|
1689
|
+
}
|
|
1690
|
+
|
|
1691
|
+
/**
|
|
1692
|
+
* @brief Find a consumer group in cluster \p mcluster by \p GroupId.
|
|
1693
|
+
*
|
|
1694
|
+
* @param mcluster Cluster to search in.
|
|
1695
|
+
* @param GroupId Group id to search.
|
|
1696
|
+
* @return Found group or NULL.
|
|
1697
|
+
*
|
|
1698
|
+
* @locks mcluster->lock MUST be held.
|
|
1699
|
+
*/
|
|
1700
|
+
rd_kafka_mock_cgrp_consumer_t *
|
|
1701
|
+
rd_kafka_mock_cgrp_consumer_find(const rd_kafka_mock_cluster_t *mcluster,
|
|
1702
|
+
const rd_kafkap_str_t *GroupId) {
|
|
1703
|
+
rd_kafka_mock_cgrp_consumer_t *mcgrp;
|
|
1704
|
+
TAILQ_FOREACH(mcgrp, &mcluster->cgrps_consumer, link) {
|
|
1705
|
+
if (!rd_kafkap_str_cmp_str(GroupId, mcgrp->id))
|
|
1706
|
+
return mcgrp;
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
return NULL;
|
|
1710
|
+
}
|
|
1711
|
+
|
|
1712
|
+
/**
|
|
1713
|
+
* @brief Check if any members have exceeded the session timeout.
|
|
1714
|
+
*
|
|
1715
|
+
* @param rkts Timers.
|
|
1716
|
+
* @param arg Consumer group.
|
|
1717
|
+
*
|
|
1718
|
+
* @locks mcluster->lock is acquired and released.
|
|
1719
|
+
*/
|
|
1720
|
+
static void rd_kafka_mock_cgrp_consumer_session_tmr_cb(rd_kafka_timers_t *rkts,
|
|
1721
|
+
void *arg) {
|
|
1722
|
+
rd_kafka_mock_cgrp_consumer_t *mcgrp = arg;
|
|
1723
|
+
rd_kafka_mock_cgrp_consumer_member_t *member, *tmp;
|
|
1724
|
+
rd_ts_t now = rd_clock();
|
|
1725
|
+
rd_kafka_mock_cluster_t *mcluster = mcgrp->cluster;
|
|
1726
|
+
|
|
1727
|
+
mtx_unlock(&mcluster->lock);
|
|
1728
|
+
TAILQ_FOREACH_SAFE(member, &mcgrp->members, link, tmp) {
|
|
1729
|
+
if (member->ts_last_activity +
|
|
1730
|
+
(mcgrp->session_timeout_ms * 1000) >
|
|
1731
|
+
now)
|
|
1732
|
+
continue;
|
|
1733
|
+
|
|
1734
|
+
rd_kafka_dbg(mcgrp->cluster->rk, MOCK, "MOCK",
|
|
1735
|
+
"Member %s session timed out for group %s",
|
|
1736
|
+
member->id, mcgrp->id);
|
|
1737
|
+
|
|
1738
|
+
rd_kafka_mock_cgrp_consumer_member_fenced(mcgrp, member);
|
|
1739
|
+
}
|
|
1740
|
+
mtx_unlock(&mcluster->lock);
|
|
1741
|
+
}
|
|
1742
|
+
|
|
1743
|
+
|
|
1744
|
+
/**
|
|
1745
|
+
* @brief Find or create a "consumer" consumer group.
|
|
1746
|
+
*
|
|
1747
|
+
* @param mcluster Cluster to search in.
|
|
1748
|
+
* @param GroupId Group id to look for.
|
|
1749
|
+
* @return Found or new consumer group.
|
|
1750
|
+
*
|
|
1751
|
+
* @locks mcluster->lock MUST be held.
|
|
1752
|
+
*/
|
|
1753
|
+
rd_kafka_mock_cgrp_consumer_t *
|
|
1754
|
+
rd_kafka_mock_cgrp_consumer_get(rd_kafka_mock_cluster_t *mcluster,
|
|
1755
|
+
const rd_kafkap_str_t *GroupId) {
|
|
1756
|
+
rd_kafka_mock_cgrp_consumer_t *mcgrp;
|
|
1757
|
+
|
|
1758
|
+
mcgrp = rd_kafka_mock_cgrp_consumer_find(mcluster, GroupId);
|
|
1759
|
+
if (mcgrp)
|
|
1760
|
+
return mcgrp;
|
|
1761
|
+
|
|
1762
|
+
mcgrp = rd_calloc(1, sizeof(*mcgrp));
|
|
1763
|
+
mcgrp->cluster = mcluster;
|
|
1764
|
+
mcgrp->id = RD_KAFKAP_STR_DUP(GroupId);
|
|
1765
|
+
mcgrp->group_epoch = 1;
|
|
1766
|
+
TAILQ_INIT(&mcgrp->members);
|
|
1767
|
+
rd_kafka_timer_start(&mcluster->timers, &mcgrp->session_tmr,
|
|
1768
|
+
1000 * 1000 /*1s*/,
|
|
1769
|
+
rd_kafka_mock_cgrp_consumer_session_tmr_cb, mcgrp);
|
|
1770
|
+
|
|
1771
|
+
TAILQ_INSERT_TAIL(&mcluster->cgrps_consumer, mcgrp, link);
|
|
1772
|
+
|
|
1773
|
+
return mcgrp;
|
|
1774
|
+
}
|
|
1775
|
+
|
|
1776
|
+
|
|
1777
|
+
void rd_kafka_mock_cgrp_consumer_target_assignment(
|
|
1778
|
+
rd_kafka_mock_cluster_t *mcluster,
|
|
1779
|
+
const char *group_id,
|
|
1780
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_t *target_assignment) {
|
|
1781
|
+
rd_kafka_mock_cgrp_consumer_t *mcgrp;
|
|
1782
|
+
rd_kafkap_str_t *group_id_str =
|
|
1783
|
+
rd_kafkap_str_new(group_id, strlen(group_id));
|
|
1784
|
+
|
|
1785
|
+
mtx_lock(&mcluster->lock);
|
|
1786
|
+
|
|
1787
|
+
mcgrp = rd_kafka_mock_cgrp_consumer_find(mcluster, group_id_str);
|
|
1788
|
+
if (!mcgrp)
|
|
1789
|
+
goto destroy;
|
|
1790
|
+
|
|
1791
|
+
rd_kafka_mock_cgrp_consumer_target_assignment_set_manual(
|
|
1792
|
+
mcgrp, target_assignment);
|
|
1793
|
+
|
|
1794
|
+
destroy:
|
|
1795
|
+
rd_kafkap_str_destroy(group_id_str);
|
|
1796
|
+
mtx_unlock(&mcluster->lock);
|
|
1797
|
+
}
|
|
1798
|
+
|
|
1799
|
+
void rd_kafka_mock_set_group_consumer_session_timeout_ms(
|
|
1800
|
+
rd_kafka_mock_cluster_t *mcluster,
|
|
1801
|
+
int group_consumer_session_timeout_ms) {
|
|
1802
|
+
mtx_lock(&mcluster->lock);
|
|
1803
|
+
mcluster->defaults.group_consumer_session_timeout_ms =
|
|
1804
|
+
group_consumer_session_timeout_ms;
|
|
1805
|
+
mtx_unlock(&mcluster->lock);
|
|
1806
|
+
}
|
|
1807
|
+
|
|
1808
|
+
void rd_kafka_mock_set_group_consumer_heartbeat_interval_ms(
|
|
1809
|
+
rd_kafka_mock_cluster_t *mcluster,
|
|
1810
|
+
int group_consumer_heartbeat_interval_ms) {
|
|
1811
|
+
mtx_lock(&mcluster->lock);
|
|
1812
|
+
mcluster->defaults.group_consumer_heartbeat_interval_ms =
|
|
1813
|
+
group_consumer_heartbeat_interval_ms;
|
|
1814
|
+
mtx_unlock(&mcluster->lock);
|
|
1815
|
+
}
|
|
1816
|
+
|
|
1817
|
+
/**
|
|
1818
|
+
* @brief A client connection closed, check if any consumer cgrp has any state
|
|
1819
|
+
* for this connection that needs to be cleared.
|
|
1820
|
+
*
|
|
1821
|
+
* @param mcluster Cluster to search in.
|
|
1822
|
+
* @param mconn Connection that was closed.
|
|
1823
|
+
*
|
|
1824
|
+
* @locks mcluster->lock MUST be held.
|
|
1825
|
+
*/
|
|
1826
|
+
void rd_kafka_mock_cgrps_consumer_connection_closed(
|
|
1827
|
+
rd_kafka_mock_cluster_t *mcluster,
|
|
1828
|
+
rd_kafka_mock_connection_t *mconn) {
|
|
1829
|
+
rd_kafka_mock_cgrp_consumer_t *mcgrp;
|
|
1830
|
+
|
|
1831
|
+
TAILQ_FOREACH(mcgrp, &mcluster->cgrps_consumer, link) {
|
|
1832
|
+
rd_kafka_mock_cgrp_consumer_member_t *member, *tmp;
|
|
1833
|
+
TAILQ_FOREACH_SAFE(member, &mcgrp->members, link, tmp) {
|
|
1834
|
+
if (member->conn == mconn) {
|
|
1835
|
+
member->conn = NULL;
|
|
1836
|
+
rd_kafka_mock_cgrp_consumer_member_returned_assignment_set(
|
|
1837
|
+
member, NULL);
|
|
1838
|
+
rd_kafka_mock_cgrp_consumer_member_current_assignment_set(
|
|
1839
|
+
member, NULL);
|
|
1840
|
+
}
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
/**
|
|
1846
|
+
* @brief Destroys consumer group \p mcgrp and all of its members.
|
|
1847
|
+
*
|
|
1848
|
+
* @param mcgrp Consumer group to destroy.
|
|
1849
|
+
*
|
|
1850
|
+
* @locks mcluster->lock MUST be held.
|
|
1851
|
+
*/
|
|
1852
|
+
void rd_kafka_mock_cgrp_consumer_destroy(rd_kafka_mock_cgrp_consumer_t *mcgrp) {
|
|
1853
|
+
rd_kafka_mock_cgrp_consumer_member_t *member;
|
|
1854
|
+
|
|
1855
|
+
TAILQ_REMOVE(&mcgrp->cluster->cgrps_consumer, mcgrp, link);
|
|
1856
|
+
|
|
1857
|
+
rd_kafka_timer_stop(&mcgrp->cluster->timers, &mcgrp->session_tmr,
|
|
1858
|
+
rd_true);
|
|
1859
|
+
rd_free(mcgrp->id);
|
|
1860
|
+
while ((member = TAILQ_FIRST(&mcgrp->members)))
|
|
1861
|
+
rd_kafka_mock_cgrp_consumer_member_destroy(mcgrp, member);
|
|
1862
|
+
rd_free(mcgrp);
|
|
1863
|
+
}
|
|
1864
|
+
|
|
1865
|
+
/**
|
|
1866
|
+
* @brief A client connection closed, check if any cgrp has any state
|
|
1867
|
+
* for this connection that needs to be cleared.
|
|
1868
|
+
*
|
|
1869
|
+
* @param mcluster Mock cluster.
|
|
1870
|
+
* @param mconn Connection that was closed.
|
|
1871
|
+
*/
|
|
1872
|
+
void rd_kafka_mock_cgrps_connection_closed(rd_kafka_mock_cluster_t *mcluster,
|
|
1873
|
+
rd_kafka_mock_connection_t *mconn) {
|
|
1874
|
+
rd_kafka_mock_cgrps_classic_connection_closed(mcluster, mconn);
|
|
1875
|
+
rd_kafka_mock_cgrps_consumer_connection_closed(mcluster, mconn);
|
|
1876
|
+
}
|