@point3/node-rdkafka 3.6.0-1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (707) hide show
  1. package/LICENSE.txt +20 -0
  2. package/README.md +636 -0
  3. package/binding.gyp +154 -0
  4. package/deps/librdkafka/.clang-format +136 -0
  5. package/deps/librdkafka/.clang-format-cpp +103 -0
  6. package/deps/librdkafka/.dir-locals.el +10 -0
  7. package/deps/librdkafka/.formatignore +33 -0
  8. package/deps/librdkafka/.gdbmacros +19 -0
  9. package/deps/librdkafka/.github/CODEOWNERS +1 -0
  10. package/deps/librdkafka/.github/ISSUE_TEMPLATE +34 -0
  11. package/deps/librdkafka/.semaphore/run-all-tests.yml +77 -0
  12. package/deps/librdkafka/.semaphore/semaphore-integration.yml +250 -0
  13. package/deps/librdkafka/.semaphore/semaphore.yml +378 -0
  14. package/deps/librdkafka/.semaphore/verify-linux-packages.yml +41 -0
  15. package/deps/librdkafka/CHANGELOG.md +2208 -0
  16. package/deps/librdkafka/CMakeLists.txt +291 -0
  17. package/deps/librdkafka/CODE_OF_CONDUCT.md +46 -0
  18. package/deps/librdkafka/CONFIGURATION.md +209 -0
  19. package/deps/librdkafka/CONTRIBUTING.md +431 -0
  20. package/deps/librdkafka/Doxyfile +2375 -0
  21. package/deps/librdkafka/INTRODUCTION.md +2481 -0
  22. package/deps/librdkafka/LICENSE +26 -0
  23. package/deps/librdkafka/LICENSE.cjson +22 -0
  24. package/deps/librdkafka/LICENSE.crc32c +28 -0
  25. package/deps/librdkafka/LICENSE.fnv1a +18 -0
  26. package/deps/librdkafka/LICENSE.hdrhistogram +27 -0
  27. package/deps/librdkafka/LICENSE.lz4 +26 -0
  28. package/deps/librdkafka/LICENSE.murmur2 +25 -0
  29. package/deps/librdkafka/LICENSE.nanopb +22 -0
  30. package/deps/librdkafka/LICENSE.opentelemetry +203 -0
  31. package/deps/librdkafka/LICENSE.pycrc +23 -0
  32. package/deps/librdkafka/LICENSE.queue +31 -0
  33. package/deps/librdkafka/LICENSE.regexp +5 -0
  34. package/deps/librdkafka/LICENSE.snappy +36 -0
  35. package/deps/librdkafka/LICENSE.tinycthread +26 -0
  36. package/deps/librdkafka/LICENSE.wingetopt +49 -0
  37. package/deps/librdkafka/LICENSES.txt +625 -0
  38. package/deps/librdkafka/Makefile +125 -0
  39. package/deps/librdkafka/README.md +199 -0
  40. package/deps/librdkafka/README.win32 +26 -0
  41. package/deps/librdkafka/STATISTICS.md +624 -0
  42. package/deps/librdkafka/configure +214 -0
  43. package/deps/librdkafka/configure.self +331 -0
  44. package/deps/librdkafka/debian/changelog +111 -0
  45. package/deps/librdkafka/debian/compat +1 -0
  46. package/deps/librdkafka/debian/control +71 -0
  47. package/deps/librdkafka/debian/copyright +99 -0
  48. package/deps/librdkafka/debian/gbp.conf +9 -0
  49. package/deps/librdkafka/debian/librdkafka++1.install +1 -0
  50. package/deps/librdkafka/debian/librdkafka-dev.examples +2 -0
  51. package/deps/librdkafka/debian/librdkafka-dev.install +9 -0
  52. package/deps/librdkafka/debian/librdkafka1.docs +5 -0
  53. package/deps/librdkafka/debian/librdkafka1.install +1 -0
  54. package/deps/librdkafka/debian/librdkafka1.symbols +135 -0
  55. package/deps/librdkafka/debian/rules +19 -0
  56. package/deps/librdkafka/debian/source/format +1 -0
  57. package/deps/librdkafka/debian/watch +2 -0
  58. package/deps/librdkafka/dev-conf.sh +123 -0
  59. package/deps/librdkafka/examples/CMakeLists.txt +79 -0
  60. package/deps/librdkafka/examples/Makefile +167 -0
  61. package/deps/librdkafka/examples/README.md +42 -0
  62. package/deps/librdkafka/examples/alter_consumer_group_offsets.c +338 -0
  63. package/deps/librdkafka/examples/consumer.c +271 -0
  64. package/deps/librdkafka/examples/delete_records.c +233 -0
  65. package/deps/librdkafka/examples/describe_cluster.c +322 -0
  66. package/deps/librdkafka/examples/describe_consumer_groups.c +455 -0
  67. package/deps/librdkafka/examples/describe_topics.c +427 -0
  68. package/deps/librdkafka/examples/elect_leaders.c +317 -0
  69. package/deps/librdkafka/examples/globals.json +11 -0
  70. package/deps/librdkafka/examples/idempotent_producer.c +344 -0
  71. package/deps/librdkafka/examples/incremental_alter_configs.c +347 -0
  72. package/deps/librdkafka/examples/kafkatest_verifiable_client.cpp +945 -0
  73. package/deps/librdkafka/examples/list_consumer_group_offsets.c +359 -0
  74. package/deps/librdkafka/examples/list_consumer_groups.c +365 -0
  75. package/deps/librdkafka/examples/list_offsets.c +327 -0
  76. package/deps/librdkafka/examples/misc.c +287 -0
  77. package/deps/librdkafka/examples/openssl_engine_example.cpp +248 -0
  78. package/deps/librdkafka/examples/producer.c +251 -0
  79. package/deps/librdkafka/examples/producer.cpp +228 -0
  80. package/deps/librdkafka/examples/rdkafka_complex_consumer_example.c +617 -0
  81. package/deps/librdkafka/examples/rdkafka_complex_consumer_example.cpp +467 -0
  82. package/deps/librdkafka/examples/rdkafka_consume_batch.cpp +264 -0
  83. package/deps/librdkafka/examples/rdkafka_example.c +853 -0
  84. package/deps/librdkafka/examples/rdkafka_example.cpp +679 -0
  85. package/deps/librdkafka/examples/rdkafka_performance.c +1781 -0
  86. package/deps/librdkafka/examples/transactions-older-broker.c +668 -0
  87. package/deps/librdkafka/examples/transactions.c +665 -0
  88. package/deps/librdkafka/examples/user_scram.c +491 -0
  89. package/deps/librdkafka/examples/win_ssl_cert_store.cpp +396 -0
  90. package/deps/librdkafka/lds-gen.py +73 -0
  91. package/deps/librdkafka/mainpage.doxy +40 -0
  92. package/deps/librdkafka/mklove/Makefile.base +329 -0
  93. package/deps/librdkafka/mklove/modules/configure.atomics +144 -0
  94. package/deps/librdkafka/mklove/modules/configure.base +2484 -0
  95. package/deps/librdkafka/mklove/modules/configure.builtin +70 -0
  96. package/deps/librdkafka/mklove/modules/configure.cc +186 -0
  97. package/deps/librdkafka/mklove/modules/configure.cxx +8 -0
  98. package/deps/librdkafka/mklove/modules/configure.fileversion +65 -0
  99. package/deps/librdkafka/mklove/modules/configure.gitversion +29 -0
  100. package/deps/librdkafka/mklove/modules/configure.good_cflags +18 -0
  101. package/deps/librdkafka/mklove/modules/configure.host +132 -0
  102. package/deps/librdkafka/mklove/modules/configure.lib +49 -0
  103. package/deps/librdkafka/mklove/modules/configure.libcurl +99 -0
  104. package/deps/librdkafka/mklove/modules/configure.libsasl2 +36 -0
  105. package/deps/librdkafka/mklove/modules/configure.libssl +147 -0
  106. package/deps/librdkafka/mklove/modules/configure.libzstd +58 -0
  107. package/deps/librdkafka/mklove/modules/configure.parseversion +95 -0
  108. package/deps/librdkafka/mklove/modules/configure.pic +16 -0
  109. package/deps/librdkafka/mklove/modules/configure.socket +20 -0
  110. package/deps/librdkafka/mklove/modules/configure.zlib +61 -0
  111. package/deps/librdkafka/mklove/modules/patches/README.md +8 -0
  112. package/deps/librdkafka/mklove/modules/patches/libcurl.0000-no-runtime-linking-check.patch +11 -0
  113. package/deps/librdkafka/mklove/modules/patches/libssl.0000-osx-rand-include-fix-OpenSSL-PR16409.patch +56 -0
  114. package/deps/librdkafka/packaging/RELEASE.md +319 -0
  115. package/deps/librdkafka/packaging/alpine/build-alpine.sh +38 -0
  116. package/deps/librdkafka/packaging/archlinux/PKGBUILD +30 -0
  117. package/deps/librdkafka/packaging/cmake/Config.cmake.in +37 -0
  118. package/deps/librdkafka/packaging/cmake/Modules/FindLZ4.cmake +38 -0
  119. package/deps/librdkafka/packaging/cmake/Modules/FindZSTD.cmake +27 -0
  120. package/deps/librdkafka/packaging/cmake/Modules/LICENSE.FindZstd +178 -0
  121. package/deps/librdkafka/packaging/cmake/README.md +38 -0
  122. package/deps/librdkafka/packaging/cmake/config.h.in +52 -0
  123. package/deps/librdkafka/packaging/cmake/parseversion.cmake +60 -0
  124. package/deps/librdkafka/packaging/cmake/rdkafka.pc.in +12 -0
  125. package/deps/librdkafka/packaging/cmake/try_compile/atomic_32_test.c +8 -0
  126. package/deps/librdkafka/packaging/cmake/try_compile/atomic_64_test.c +8 -0
  127. package/deps/librdkafka/packaging/cmake/try_compile/c11threads_test.c +14 -0
  128. package/deps/librdkafka/packaging/cmake/try_compile/crc32c_hw_test.c +27 -0
  129. package/deps/librdkafka/packaging/cmake/try_compile/dlopen_test.c +11 -0
  130. package/deps/librdkafka/packaging/cmake/try_compile/libsasl2_test.c +7 -0
  131. package/deps/librdkafka/packaging/cmake/try_compile/pthread_setname_darwin_test.c +6 -0
  132. package/deps/librdkafka/packaging/cmake/try_compile/pthread_setname_freebsd_test.c +7 -0
  133. package/deps/librdkafka/packaging/cmake/try_compile/pthread_setname_gnu_test.c +5 -0
  134. package/deps/librdkafka/packaging/cmake/try_compile/rand_r_test.c +7 -0
  135. package/deps/librdkafka/packaging/cmake/try_compile/rdkafka_setup.cmake +122 -0
  136. package/deps/librdkafka/packaging/cmake/try_compile/regex_test.c +10 -0
  137. package/deps/librdkafka/packaging/cmake/try_compile/strndup_test.c +5 -0
  138. package/deps/librdkafka/packaging/cmake/try_compile/sync_32_test.c +8 -0
  139. package/deps/librdkafka/packaging/cmake/try_compile/sync_64_test.c +8 -0
  140. package/deps/librdkafka/packaging/cp/README.md +16 -0
  141. package/deps/librdkafka/packaging/cp/check_features.c +72 -0
  142. package/deps/librdkafka/packaging/cp/verify-deb.sh +33 -0
  143. package/deps/librdkafka/packaging/cp/verify-packages.sh +69 -0
  144. package/deps/librdkafka/packaging/cp/verify-rpm.sh +32 -0
  145. package/deps/librdkafka/packaging/debian/changelog +66 -0
  146. package/deps/librdkafka/packaging/debian/compat +1 -0
  147. package/deps/librdkafka/packaging/debian/control +49 -0
  148. package/deps/librdkafka/packaging/debian/copyright +84 -0
  149. package/deps/librdkafka/packaging/debian/docs +5 -0
  150. package/deps/librdkafka/packaging/debian/gbp.conf +9 -0
  151. package/deps/librdkafka/packaging/debian/librdkafka-dev.dirs +2 -0
  152. package/deps/librdkafka/packaging/debian/librdkafka-dev.examples +2 -0
  153. package/deps/librdkafka/packaging/debian/librdkafka-dev.install +6 -0
  154. package/deps/librdkafka/packaging/debian/librdkafka-dev.substvars +1 -0
  155. package/deps/librdkafka/packaging/debian/librdkafka.dsc +16 -0
  156. package/deps/librdkafka/packaging/debian/librdkafka1-dbg.substvars +1 -0
  157. package/deps/librdkafka/packaging/debian/librdkafka1.dirs +1 -0
  158. package/deps/librdkafka/packaging/debian/librdkafka1.install +2 -0
  159. package/deps/librdkafka/packaging/debian/librdkafka1.postinst.debhelper +5 -0
  160. package/deps/librdkafka/packaging/debian/librdkafka1.postrm.debhelper +5 -0
  161. package/deps/librdkafka/packaging/debian/librdkafka1.symbols +64 -0
  162. package/deps/librdkafka/packaging/debian/rules +19 -0
  163. package/deps/librdkafka/packaging/debian/source/format +1 -0
  164. package/deps/librdkafka/packaging/debian/watch +2 -0
  165. package/deps/librdkafka/packaging/get_version.py +21 -0
  166. package/deps/librdkafka/packaging/homebrew/README.md +15 -0
  167. package/deps/librdkafka/packaging/homebrew/brew-update-pr.sh +31 -0
  168. package/deps/librdkafka/packaging/mingw-w64/configure-build-msys2-mingw-static.sh +52 -0
  169. package/deps/librdkafka/packaging/mingw-w64/configure-build-msys2-mingw.sh +21 -0
  170. package/deps/librdkafka/packaging/mingw-w64/export-variables.sh +13 -0
  171. package/deps/librdkafka/packaging/mingw-w64/run-tests.sh +6 -0
  172. package/deps/librdkafka/packaging/mingw-w64/semaphoreci-build.sh +38 -0
  173. package/deps/librdkafka/packaging/nuget/README.md +84 -0
  174. package/deps/librdkafka/packaging/nuget/artifact.py +177 -0
  175. package/deps/librdkafka/packaging/nuget/cleanup-s3.py +143 -0
  176. package/deps/librdkafka/packaging/nuget/common/p-common__plat-windows__arch-win32__bldtype-Release/msvcr120.zip +0 -0
  177. package/deps/librdkafka/packaging/nuget/common/p-common__plat-windows__arch-win32__bldtype-Release/msvcr140.zip +0 -0
  178. package/deps/librdkafka/packaging/nuget/common/p-common__plat-windows__arch-x64__bldtype-Release/msvcr120.zip +0 -0
  179. package/deps/librdkafka/packaging/nuget/common/p-common__plat-windows__arch-x64__bldtype-Release/msvcr140.zip +0 -0
  180. package/deps/librdkafka/packaging/nuget/nuget.sh +21 -0
  181. package/deps/librdkafka/packaging/nuget/nugetpackage.py +278 -0
  182. package/deps/librdkafka/packaging/nuget/packaging.py +448 -0
  183. package/deps/librdkafka/packaging/nuget/push-to-nuget.sh +21 -0
  184. package/deps/librdkafka/packaging/nuget/release.py +167 -0
  185. package/deps/librdkafka/packaging/nuget/requirements.txt +3 -0
  186. package/deps/librdkafka/packaging/nuget/staticpackage.py +178 -0
  187. package/deps/librdkafka/packaging/nuget/templates/librdkafka.redist.nuspec +21 -0
  188. package/deps/librdkafka/packaging/nuget/templates/librdkafka.redist.props +18 -0
  189. package/deps/librdkafka/packaging/nuget/templates/librdkafka.redist.targets +19 -0
  190. package/deps/librdkafka/packaging/nuget/zfile/__init__.py +0 -0
  191. package/deps/librdkafka/packaging/nuget/zfile/zfile.py +98 -0
  192. package/deps/librdkafka/packaging/rpm/Makefile +92 -0
  193. package/deps/librdkafka/packaging/rpm/README.md +23 -0
  194. package/deps/librdkafka/packaging/rpm/el7-x86_64.cfg +40 -0
  195. package/deps/librdkafka/packaging/rpm/librdkafka.spec +118 -0
  196. package/deps/librdkafka/packaging/rpm/mock-on-docker.sh +96 -0
  197. package/deps/librdkafka/packaging/rpm/tests/Makefile +25 -0
  198. package/deps/librdkafka/packaging/rpm/tests/README.md +8 -0
  199. package/deps/librdkafka/packaging/rpm/tests/run-test.sh +42 -0
  200. package/deps/librdkafka/packaging/rpm/tests/test-on-docker.sh +56 -0
  201. package/deps/librdkafka/packaging/rpm/tests/test.c +77 -0
  202. package/deps/librdkafka/packaging/rpm/tests/test.cpp +34 -0
  203. package/deps/librdkafka/packaging/tools/Dockerfile +31 -0
  204. package/deps/librdkafka/packaging/tools/build-configurations-checks.sh +12 -0
  205. package/deps/librdkafka/packaging/tools/build-deb-package.sh +64 -0
  206. package/deps/librdkafka/packaging/tools/build-debian.sh +65 -0
  207. package/deps/librdkafka/packaging/tools/build-manylinux.sh +68 -0
  208. package/deps/librdkafka/packaging/tools/build-release-artifacts.sh +139 -0
  209. package/deps/librdkafka/packaging/tools/distro-build.sh +38 -0
  210. package/deps/librdkafka/packaging/tools/gh-release-checksums.py +39 -0
  211. package/deps/librdkafka/packaging/tools/rdutcoverage.sh +25 -0
  212. package/deps/librdkafka/packaging/tools/requirements.txt +2 -0
  213. package/deps/librdkafka/packaging/tools/run-in-docker.sh +28 -0
  214. package/deps/librdkafka/packaging/tools/run-integration-tests.sh +31 -0
  215. package/deps/librdkafka/packaging/tools/run-style-check.sh +4 -0
  216. package/deps/librdkafka/packaging/tools/style-format.sh +149 -0
  217. package/deps/librdkafka/packaging/tools/update_rpcs_max_versions.py +100 -0
  218. package/deps/librdkafka/service.yml +172 -0
  219. package/deps/librdkafka/src/CMakeLists.txt +374 -0
  220. package/deps/librdkafka/src/Makefile +103 -0
  221. package/deps/librdkafka/src/README.lz4.md +30 -0
  222. package/deps/librdkafka/src/cJSON.c +2834 -0
  223. package/deps/librdkafka/src/cJSON.h +398 -0
  224. package/deps/librdkafka/src/crc32c.c +430 -0
  225. package/deps/librdkafka/src/crc32c.h +38 -0
  226. package/deps/librdkafka/src/generate_proto.sh +66 -0
  227. package/deps/librdkafka/src/librdkafka_cgrp_synch.png +0 -0
  228. package/deps/librdkafka/src/lz4.c +2727 -0
  229. package/deps/librdkafka/src/lz4.h +842 -0
  230. package/deps/librdkafka/src/lz4frame.c +2078 -0
  231. package/deps/librdkafka/src/lz4frame.h +692 -0
  232. package/deps/librdkafka/src/lz4frame_static.h +47 -0
  233. package/deps/librdkafka/src/lz4hc.c +1631 -0
  234. package/deps/librdkafka/src/lz4hc.h +413 -0
  235. package/deps/librdkafka/src/nanopb/pb.h +917 -0
  236. package/deps/librdkafka/src/nanopb/pb_common.c +388 -0
  237. package/deps/librdkafka/src/nanopb/pb_common.h +49 -0
  238. package/deps/librdkafka/src/nanopb/pb_decode.c +1727 -0
  239. package/deps/librdkafka/src/nanopb/pb_decode.h +193 -0
  240. package/deps/librdkafka/src/nanopb/pb_encode.c +1000 -0
  241. package/deps/librdkafka/src/nanopb/pb_encode.h +185 -0
  242. package/deps/librdkafka/src/opentelemetry/common.pb.c +32 -0
  243. package/deps/librdkafka/src/opentelemetry/common.pb.h +170 -0
  244. package/deps/librdkafka/src/opentelemetry/metrics.options +2 -0
  245. package/deps/librdkafka/src/opentelemetry/metrics.pb.c +67 -0
  246. package/deps/librdkafka/src/opentelemetry/metrics.pb.h +966 -0
  247. package/deps/librdkafka/src/opentelemetry/resource.pb.c +12 -0
  248. package/deps/librdkafka/src/opentelemetry/resource.pb.h +58 -0
  249. package/deps/librdkafka/src/queue.h +850 -0
  250. package/deps/librdkafka/src/rd.h +584 -0
  251. package/deps/librdkafka/src/rdaddr.c +255 -0
  252. package/deps/librdkafka/src/rdaddr.h +202 -0
  253. package/deps/librdkafka/src/rdatomic.h +230 -0
  254. package/deps/librdkafka/src/rdavg.h +260 -0
  255. package/deps/librdkafka/src/rdavl.c +210 -0
  256. package/deps/librdkafka/src/rdavl.h +250 -0
  257. package/deps/librdkafka/src/rdbase64.c +200 -0
  258. package/deps/librdkafka/src/rdbase64.h +43 -0
  259. package/deps/librdkafka/src/rdbuf.c +1884 -0
  260. package/deps/librdkafka/src/rdbuf.h +375 -0
  261. package/deps/librdkafka/src/rdcrc32.c +114 -0
  262. package/deps/librdkafka/src/rdcrc32.h +170 -0
  263. package/deps/librdkafka/src/rddl.c +179 -0
  264. package/deps/librdkafka/src/rddl.h +43 -0
  265. package/deps/librdkafka/src/rdendian.h +175 -0
  266. package/deps/librdkafka/src/rdfloat.h +67 -0
  267. package/deps/librdkafka/src/rdfnv1a.c +113 -0
  268. package/deps/librdkafka/src/rdfnv1a.h +35 -0
  269. package/deps/librdkafka/src/rdgz.c +120 -0
  270. package/deps/librdkafka/src/rdgz.h +46 -0
  271. package/deps/librdkafka/src/rdhdrhistogram.c +721 -0
  272. package/deps/librdkafka/src/rdhdrhistogram.h +87 -0
  273. package/deps/librdkafka/src/rdhttp.c +830 -0
  274. package/deps/librdkafka/src/rdhttp.h +101 -0
  275. package/deps/librdkafka/src/rdinterval.h +177 -0
  276. package/deps/librdkafka/src/rdkafka.c +5505 -0
  277. package/deps/librdkafka/src/rdkafka.h +10686 -0
  278. package/deps/librdkafka/src/rdkafka_admin.c +9794 -0
  279. package/deps/librdkafka/src/rdkafka_admin.h +661 -0
  280. package/deps/librdkafka/src/rdkafka_assignment.c +1010 -0
  281. package/deps/librdkafka/src/rdkafka_assignment.h +73 -0
  282. package/deps/librdkafka/src/rdkafka_assignor.c +1786 -0
  283. package/deps/librdkafka/src/rdkafka_assignor.h +402 -0
  284. package/deps/librdkafka/src/rdkafka_aux.c +409 -0
  285. package/deps/librdkafka/src/rdkafka_aux.h +174 -0
  286. package/deps/librdkafka/src/rdkafka_background.c +221 -0
  287. package/deps/librdkafka/src/rdkafka_broker.c +6337 -0
  288. package/deps/librdkafka/src/rdkafka_broker.h +744 -0
  289. package/deps/librdkafka/src/rdkafka_buf.c +543 -0
  290. package/deps/librdkafka/src/rdkafka_buf.h +1525 -0
  291. package/deps/librdkafka/src/rdkafka_cert.c +576 -0
  292. package/deps/librdkafka/src/rdkafka_cert.h +62 -0
  293. package/deps/librdkafka/src/rdkafka_cgrp.c +7587 -0
  294. package/deps/librdkafka/src/rdkafka_cgrp.h +477 -0
  295. package/deps/librdkafka/src/rdkafka_conf.c +4880 -0
  296. package/deps/librdkafka/src/rdkafka_conf.h +732 -0
  297. package/deps/librdkafka/src/rdkafka_confval.h +97 -0
  298. package/deps/librdkafka/src/rdkafka_coord.c +623 -0
  299. package/deps/librdkafka/src/rdkafka_coord.h +132 -0
  300. package/deps/librdkafka/src/rdkafka_error.c +228 -0
  301. package/deps/librdkafka/src/rdkafka_error.h +80 -0
  302. package/deps/librdkafka/src/rdkafka_event.c +502 -0
  303. package/deps/librdkafka/src/rdkafka_event.h +126 -0
  304. package/deps/librdkafka/src/rdkafka_feature.c +898 -0
  305. package/deps/librdkafka/src/rdkafka_feature.h +104 -0
  306. package/deps/librdkafka/src/rdkafka_fetcher.c +1422 -0
  307. package/deps/librdkafka/src/rdkafka_fetcher.h +44 -0
  308. package/deps/librdkafka/src/rdkafka_header.c +220 -0
  309. package/deps/librdkafka/src/rdkafka_header.h +76 -0
  310. package/deps/librdkafka/src/rdkafka_idempotence.c +807 -0
  311. package/deps/librdkafka/src/rdkafka_idempotence.h +144 -0
  312. package/deps/librdkafka/src/rdkafka_int.h +1260 -0
  313. package/deps/librdkafka/src/rdkafka_interceptor.c +819 -0
  314. package/deps/librdkafka/src/rdkafka_interceptor.h +104 -0
  315. package/deps/librdkafka/src/rdkafka_lz4.c +450 -0
  316. package/deps/librdkafka/src/rdkafka_lz4.h +49 -0
  317. package/deps/librdkafka/src/rdkafka_metadata.c +2209 -0
  318. package/deps/librdkafka/src/rdkafka_metadata.h +345 -0
  319. package/deps/librdkafka/src/rdkafka_metadata_cache.c +1183 -0
  320. package/deps/librdkafka/src/rdkafka_mock.c +3661 -0
  321. package/deps/librdkafka/src/rdkafka_mock.h +610 -0
  322. package/deps/librdkafka/src/rdkafka_mock_cgrp.c +1876 -0
  323. package/deps/librdkafka/src/rdkafka_mock_handlers.c +3113 -0
  324. package/deps/librdkafka/src/rdkafka_mock_int.h +710 -0
  325. package/deps/librdkafka/src/rdkafka_msg.c +2589 -0
  326. package/deps/librdkafka/src/rdkafka_msg.h +614 -0
  327. package/deps/librdkafka/src/rdkafka_msgbatch.h +62 -0
  328. package/deps/librdkafka/src/rdkafka_msgset.h +98 -0
  329. package/deps/librdkafka/src/rdkafka_msgset_reader.c +1806 -0
  330. package/deps/librdkafka/src/rdkafka_msgset_writer.c +1474 -0
  331. package/deps/librdkafka/src/rdkafka_offset.c +1565 -0
  332. package/deps/librdkafka/src/rdkafka_offset.h +150 -0
  333. package/deps/librdkafka/src/rdkafka_op.c +997 -0
  334. package/deps/librdkafka/src/rdkafka_op.h +858 -0
  335. package/deps/librdkafka/src/rdkafka_partition.c +4896 -0
  336. package/deps/librdkafka/src/rdkafka_partition.h +1182 -0
  337. package/deps/librdkafka/src/rdkafka_pattern.c +228 -0
  338. package/deps/librdkafka/src/rdkafka_pattern.h +70 -0
  339. package/deps/librdkafka/src/rdkafka_plugin.c +213 -0
  340. package/deps/librdkafka/src/rdkafka_plugin.h +41 -0
  341. package/deps/librdkafka/src/rdkafka_proto.h +736 -0
  342. package/deps/librdkafka/src/rdkafka_protocol.h +128 -0
  343. package/deps/librdkafka/src/rdkafka_queue.c +1230 -0
  344. package/deps/librdkafka/src/rdkafka_queue.h +1220 -0
  345. package/deps/librdkafka/src/rdkafka_range_assignor.c +1748 -0
  346. package/deps/librdkafka/src/rdkafka_request.c +7089 -0
  347. package/deps/librdkafka/src/rdkafka_request.h +732 -0
  348. package/deps/librdkafka/src/rdkafka_roundrobin_assignor.c +123 -0
  349. package/deps/librdkafka/src/rdkafka_sasl.c +530 -0
  350. package/deps/librdkafka/src/rdkafka_sasl.h +63 -0
  351. package/deps/librdkafka/src/rdkafka_sasl_cyrus.c +722 -0
  352. package/deps/librdkafka/src/rdkafka_sasl_int.h +89 -0
  353. package/deps/librdkafka/src/rdkafka_sasl_oauthbearer.c +1833 -0
  354. package/deps/librdkafka/src/rdkafka_sasl_oauthbearer.h +52 -0
  355. package/deps/librdkafka/src/rdkafka_sasl_oauthbearer_oidc.c +1666 -0
  356. package/deps/librdkafka/src/rdkafka_sasl_oauthbearer_oidc.h +47 -0
  357. package/deps/librdkafka/src/rdkafka_sasl_plain.c +142 -0
  358. package/deps/librdkafka/src/rdkafka_sasl_scram.c +858 -0
  359. package/deps/librdkafka/src/rdkafka_sasl_win32.c +550 -0
  360. package/deps/librdkafka/src/rdkafka_ssl.c +2129 -0
  361. package/deps/librdkafka/src/rdkafka_ssl.h +86 -0
  362. package/deps/librdkafka/src/rdkafka_sticky_assignor.c +4785 -0
  363. package/deps/librdkafka/src/rdkafka_subscription.c +278 -0
  364. package/deps/librdkafka/src/rdkafka_telemetry.c +760 -0
  365. package/deps/librdkafka/src/rdkafka_telemetry.h +52 -0
  366. package/deps/librdkafka/src/rdkafka_telemetry_decode.c +1053 -0
  367. package/deps/librdkafka/src/rdkafka_telemetry_decode.h +59 -0
  368. package/deps/librdkafka/src/rdkafka_telemetry_encode.c +997 -0
  369. package/deps/librdkafka/src/rdkafka_telemetry_encode.h +301 -0
  370. package/deps/librdkafka/src/rdkafka_timer.c +402 -0
  371. package/deps/librdkafka/src/rdkafka_timer.h +117 -0
  372. package/deps/librdkafka/src/rdkafka_topic.c +2161 -0
  373. package/deps/librdkafka/src/rdkafka_topic.h +334 -0
  374. package/deps/librdkafka/src/rdkafka_transport.c +1309 -0
  375. package/deps/librdkafka/src/rdkafka_transport.h +99 -0
  376. package/deps/librdkafka/src/rdkafka_transport_int.h +100 -0
  377. package/deps/librdkafka/src/rdkafka_txnmgr.c +3256 -0
  378. package/deps/librdkafka/src/rdkafka_txnmgr.h +171 -0
  379. package/deps/librdkafka/src/rdkafka_zstd.c +226 -0
  380. package/deps/librdkafka/src/rdkafka_zstd.h +57 -0
  381. package/deps/librdkafka/src/rdlist.c +576 -0
  382. package/deps/librdkafka/src/rdlist.h +434 -0
  383. package/deps/librdkafka/src/rdlog.c +89 -0
  384. package/deps/librdkafka/src/rdlog.h +41 -0
  385. package/deps/librdkafka/src/rdmap.c +508 -0
  386. package/deps/librdkafka/src/rdmap.h +492 -0
  387. package/deps/librdkafka/src/rdmurmur2.c +167 -0
  388. package/deps/librdkafka/src/rdmurmur2.h +35 -0
  389. package/deps/librdkafka/src/rdports.c +61 -0
  390. package/deps/librdkafka/src/rdports.h +38 -0
  391. package/deps/librdkafka/src/rdposix.h +250 -0
  392. package/deps/librdkafka/src/rdrand.c +80 -0
  393. package/deps/librdkafka/src/rdrand.h +43 -0
  394. package/deps/librdkafka/src/rdregex.c +156 -0
  395. package/deps/librdkafka/src/rdregex.h +43 -0
  396. package/deps/librdkafka/src/rdsignal.h +57 -0
  397. package/deps/librdkafka/src/rdstring.c +645 -0
  398. package/deps/librdkafka/src/rdstring.h +98 -0
  399. package/deps/librdkafka/src/rdsysqueue.h +404 -0
  400. package/deps/librdkafka/src/rdtime.h +356 -0
  401. package/deps/librdkafka/src/rdtypes.h +86 -0
  402. package/deps/librdkafka/src/rdunittest.c +549 -0
  403. package/deps/librdkafka/src/rdunittest.h +232 -0
  404. package/deps/librdkafka/src/rdvarint.c +134 -0
  405. package/deps/librdkafka/src/rdvarint.h +165 -0
  406. package/deps/librdkafka/src/rdwin32.h +382 -0
  407. package/deps/librdkafka/src/rdxxhash.c +1030 -0
  408. package/deps/librdkafka/src/rdxxhash.h +328 -0
  409. package/deps/librdkafka/src/regexp.c +1352 -0
  410. package/deps/librdkafka/src/regexp.h +41 -0
  411. package/deps/librdkafka/src/snappy.c +1866 -0
  412. package/deps/librdkafka/src/snappy.h +62 -0
  413. package/deps/librdkafka/src/snappy_compat.h +138 -0
  414. package/deps/librdkafka/src/statistics_schema.json +444 -0
  415. package/deps/librdkafka/src/tinycthread.c +932 -0
  416. package/deps/librdkafka/src/tinycthread.h +503 -0
  417. package/deps/librdkafka/src/tinycthread_extra.c +199 -0
  418. package/deps/librdkafka/src/tinycthread_extra.h +212 -0
  419. package/deps/librdkafka/src/win32_config.h +58 -0
  420. package/deps/librdkafka/src-cpp/CMakeLists.txt +90 -0
  421. package/deps/librdkafka/src-cpp/ConfImpl.cpp +84 -0
  422. package/deps/librdkafka/src-cpp/ConsumerImpl.cpp +244 -0
  423. package/deps/librdkafka/src-cpp/HandleImpl.cpp +436 -0
  424. package/deps/librdkafka/src-cpp/HeadersImpl.cpp +48 -0
  425. package/deps/librdkafka/src-cpp/KafkaConsumerImpl.cpp +296 -0
  426. package/deps/librdkafka/src-cpp/Makefile +55 -0
  427. package/deps/librdkafka/src-cpp/MessageImpl.cpp +38 -0
  428. package/deps/librdkafka/src-cpp/MetadataImpl.cpp +170 -0
  429. package/deps/librdkafka/src-cpp/ProducerImpl.cpp +197 -0
  430. package/deps/librdkafka/src-cpp/QueueImpl.cpp +70 -0
  431. package/deps/librdkafka/src-cpp/README.md +16 -0
  432. package/deps/librdkafka/src-cpp/RdKafka.cpp +59 -0
  433. package/deps/librdkafka/src-cpp/TopicImpl.cpp +124 -0
  434. package/deps/librdkafka/src-cpp/TopicPartitionImpl.cpp +57 -0
  435. package/deps/librdkafka/src-cpp/rdkafkacpp.h +3797 -0
  436. package/deps/librdkafka/src-cpp/rdkafkacpp_int.h +1641 -0
  437. package/deps/librdkafka/tests/0000-unittests.c +72 -0
  438. package/deps/librdkafka/tests/0001-multiobj.c +102 -0
  439. package/deps/librdkafka/tests/0002-unkpart.c +244 -0
  440. package/deps/librdkafka/tests/0003-msgmaxsize.c +173 -0
  441. package/deps/librdkafka/tests/0004-conf.c +934 -0
  442. package/deps/librdkafka/tests/0005-order.c +133 -0
  443. package/deps/librdkafka/tests/0006-symbols.c +163 -0
  444. package/deps/librdkafka/tests/0007-autotopic.c +136 -0
  445. package/deps/librdkafka/tests/0008-reqacks.c +179 -0
  446. package/deps/librdkafka/tests/0009-mock_cluster.c +97 -0
  447. package/deps/librdkafka/tests/0011-produce_batch.c +753 -0
  448. package/deps/librdkafka/tests/0012-produce_consume.c +537 -0
  449. package/deps/librdkafka/tests/0013-null-msgs.c +473 -0
  450. package/deps/librdkafka/tests/0014-reconsume-191.c +512 -0
  451. package/deps/librdkafka/tests/0015-offset_seeks.c +172 -0
  452. package/deps/librdkafka/tests/0016-client_swname.c +181 -0
  453. package/deps/librdkafka/tests/0017-compression.c +140 -0
  454. package/deps/librdkafka/tests/0018-cgrp_term.c +338 -0
  455. package/deps/librdkafka/tests/0019-list_groups.c +289 -0
  456. package/deps/librdkafka/tests/0020-destroy_hang.c +162 -0
  457. package/deps/librdkafka/tests/0021-rkt_destroy.c +72 -0
  458. package/deps/librdkafka/tests/0022-consume_batch.c +279 -0
  459. package/deps/librdkafka/tests/0025-timers.c +147 -0
  460. package/deps/librdkafka/tests/0026-consume_pause.c +547 -0
  461. package/deps/librdkafka/tests/0028-long_topicnames.c +79 -0
  462. package/deps/librdkafka/tests/0029-assign_offset.c +202 -0
  463. package/deps/librdkafka/tests/0030-offset_commit.c +589 -0
  464. package/deps/librdkafka/tests/0031-get_offsets.c +235 -0
  465. package/deps/librdkafka/tests/0033-regex_subscribe.c +536 -0
  466. package/deps/librdkafka/tests/0034-offset_reset.c +398 -0
  467. package/deps/librdkafka/tests/0035-api_version.c +73 -0
  468. package/deps/librdkafka/tests/0036-partial_fetch.c +87 -0
  469. package/deps/librdkafka/tests/0037-destroy_hang_local.c +85 -0
  470. package/deps/librdkafka/tests/0038-performance.c +121 -0
  471. package/deps/librdkafka/tests/0039-event.c +284 -0
  472. package/deps/librdkafka/tests/0040-io_event.c +257 -0
  473. package/deps/librdkafka/tests/0041-fetch_max_bytes.c +97 -0
  474. package/deps/librdkafka/tests/0042-many_topics.c +252 -0
  475. package/deps/librdkafka/tests/0043-no_connection.c +77 -0
  476. package/deps/librdkafka/tests/0044-partition_cnt.c +94 -0
  477. package/deps/librdkafka/tests/0045-subscribe_update.c +1010 -0
  478. package/deps/librdkafka/tests/0046-rkt_cache.c +65 -0
  479. package/deps/librdkafka/tests/0047-partial_buf_tmout.c +98 -0
  480. package/deps/librdkafka/tests/0048-partitioner.c +283 -0
  481. package/deps/librdkafka/tests/0049-consume_conn_close.c +162 -0
  482. package/deps/librdkafka/tests/0050-subscribe_adds.c +145 -0
  483. package/deps/librdkafka/tests/0051-assign_adds.c +126 -0
  484. package/deps/librdkafka/tests/0052-msg_timestamps.c +238 -0
  485. package/deps/librdkafka/tests/0053-stats_cb.cpp +527 -0
  486. package/deps/librdkafka/tests/0054-offset_time.cpp +236 -0
  487. package/deps/librdkafka/tests/0055-producer_latency.c +539 -0
  488. package/deps/librdkafka/tests/0056-balanced_group_mt.c +315 -0
  489. package/deps/librdkafka/tests/0057-invalid_topic.cpp +112 -0
  490. package/deps/librdkafka/tests/0058-log.cpp +123 -0
  491. package/deps/librdkafka/tests/0059-bsearch.cpp +241 -0
  492. package/deps/librdkafka/tests/0060-op_prio.cpp +163 -0
  493. package/deps/librdkafka/tests/0061-consumer_lag.cpp +295 -0
  494. package/deps/librdkafka/tests/0062-stats_event.c +126 -0
  495. package/deps/librdkafka/tests/0063-clusterid.cpp +180 -0
  496. package/deps/librdkafka/tests/0064-interceptors.c +481 -0
  497. package/deps/librdkafka/tests/0065-yield.cpp +140 -0
  498. package/deps/librdkafka/tests/0066-plugins.cpp +129 -0
  499. package/deps/librdkafka/tests/0067-empty_topic.cpp +151 -0
  500. package/deps/librdkafka/tests/0068-produce_timeout.c +136 -0
  501. package/deps/librdkafka/tests/0069-consumer_add_parts.c +119 -0
  502. package/deps/librdkafka/tests/0070-null_empty.cpp +197 -0
  503. package/deps/librdkafka/tests/0072-headers_ut.c +448 -0
  504. package/deps/librdkafka/tests/0073-headers.c +381 -0
  505. package/deps/librdkafka/tests/0074-producev.c +87 -0
  506. package/deps/librdkafka/tests/0075-retry.c +290 -0
  507. package/deps/librdkafka/tests/0076-produce_retry.c +452 -0
  508. package/deps/librdkafka/tests/0077-compaction.c +363 -0
  509. package/deps/librdkafka/tests/0078-c_from_cpp.cpp +96 -0
  510. package/deps/librdkafka/tests/0079-fork.c +93 -0
  511. package/deps/librdkafka/tests/0080-admin_ut.c +3095 -0
  512. package/deps/librdkafka/tests/0081-admin.c +5633 -0
  513. package/deps/librdkafka/tests/0082-fetch_max_bytes.cpp +137 -0
  514. package/deps/librdkafka/tests/0083-cb_event.c +233 -0
  515. package/deps/librdkafka/tests/0084-destroy_flags.c +208 -0
  516. package/deps/librdkafka/tests/0085-headers.cpp +392 -0
  517. package/deps/librdkafka/tests/0086-purge.c +368 -0
  518. package/deps/librdkafka/tests/0088-produce_metadata_timeout.c +162 -0
  519. package/deps/librdkafka/tests/0089-max_poll_interval.c +511 -0
  520. package/deps/librdkafka/tests/0090-idempotence.c +171 -0
  521. package/deps/librdkafka/tests/0091-max_poll_interval_timeout.c +295 -0
  522. package/deps/librdkafka/tests/0092-mixed_msgver.c +103 -0
  523. package/deps/librdkafka/tests/0093-holb.c +200 -0
  524. package/deps/librdkafka/tests/0094-idempotence_msg_timeout.c +231 -0
  525. package/deps/librdkafka/tests/0095-all_brokers_down.cpp +122 -0
  526. package/deps/librdkafka/tests/0097-ssl_verify.cpp +658 -0
  527. package/deps/librdkafka/tests/0098-consumer-txn.cpp +1218 -0
  528. package/deps/librdkafka/tests/0099-commit_metadata.c +194 -0
  529. package/deps/librdkafka/tests/0100-thread_interceptors.cpp +195 -0
  530. package/deps/librdkafka/tests/0101-fetch-from-follower.cpp +446 -0
  531. package/deps/librdkafka/tests/0102-static_group_rebalance.c +836 -0
  532. package/deps/librdkafka/tests/0103-transactions.c +1383 -0
  533. package/deps/librdkafka/tests/0104-fetch_from_follower_mock.c +625 -0
  534. package/deps/librdkafka/tests/0105-transactions_mock.c +3930 -0
  535. package/deps/librdkafka/tests/0106-cgrp_sess_timeout.c +318 -0
  536. package/deps/librdkafka/tests/0107-topic_recreate.c +259 -0
  537. package/deps/librdkafka/tests/0109-auto_create_topics.cpp +278 -0
  538. package/deps/librdkafka/tests/0110-batch_size.cpp +182 -0
  539. package/deps/librdkafka/tests/0111-delay_create_topics.cpp +127 -0
  540. package/deps/librdkafka/tests/0112-assign_unknown_part.c +87 -0
  541. package/deps/librdkafka/tests/0113-cooperative_rebalance.cpp +3473 -0
  542. package/deps/librdkafka/tests/0114-sticky_partitioning.cpp +176 -0
  543. package/deps/librdkafka/tests/0115-producer_auth.cpp +182 -0
  544. package/deps/librdkafka/tests/0116-kafkaconsumer_close.cpp +216 -0
  545. package/deps/librdkafka/tests/0117-mock_errors.c +331 -0
  546. package/deps/librdkafka/tests/0118-commit_rebalance.c +154 -0
  547. package/deps/librdkafka/tests/0119-consumer_auth.cpp +167 -0
  548. package/deps/librdkafka/tests/0120-asymmetric_subscription.c +185 -0
  549. package/deps/librdkafka/tests/0121-clusterid.c +115 -0
  550. package/deps/librdkafka/tests/0122-buffer_cleaning_after_rebalance.c +227 -0
  551. package/deps/librdkafka/tests/0123-connections_max_idle.c +98 -0
  552. package/deps/librdkafka/tests/0124-openssl_invalid_engine.c +69 -0
  553. package/deps/librdkafka/tests/0125-immediate_flush.c +144 -0
  554. package/deps/librdkafka/tests/0126-oauthbearer_oidc.c +528 -0
  555. package/deps/librdkafka/tests/0127-fetch_queue_backoff.cpp +165 -0
  556. package/deps/librdkafka/tests/0128-sasl_callback_queue.cpp +125 -0
  557. package/deps/librdkafka/tests/0129-fetch_aborted_msgs.c +79 -0
  558. package/deps/librdkafka/tests/0130-store_offsets.c +178 -0
  559. package/deps/librdkafka/tests/0131-connect_timeout.c +81 -0
  560. package/deps/librdkafka/tests/0132-strategy_ordering.c +179 -0
  561. package/deps/librdkafka/tests/0133-ssl_keys.c +150 -0
  562. package/deps/librdkafka/tests/0134-ssl_provider.c +92 -0
  563. package/deps/librdkafka/tests/0135-sasl_credentials.cpp +143 -0
  564. package/deps/librdkafka/tests/0136-resolve_cb.c +181 -0
  565. package/deps/librdkafka/tests/0137-barrier_batch_consume.c +619 -0
  566. package/deps/librdkafka/tests/0138-admin_mock.c +281 -0
  567. package/deps/librdkafka/tests/0139-offset_validation_mock.c +950 -0
  568. package/deps/librdkafka/tests/0140-commit_metadata.cpp +108 -0
  569. package/deps/librdkafka/tests/0142-reauthentication.c +515 -0
  570. package/deps/librdkafka/tests/0143-exponential_backoff_mock.c +552 -0
  571. package/deps/librdkafka/tests/0144-idempotence_mock.c +373 -0
  572. package/deps/librdkafka/tests/0145-pause_resume_mock.c +119 -0
  573. package/deps/librdkafka/tests/0146-metadata_mock.c +505 -0
  574. package/deps/librdkafka/tests/0147-consumer_group_consumer_mock.c +952 -0
  575. package/deps/librdkafka/tests/0148-offset_fetch_commit_error_mock.c +563 -0
  576. package/deps/librdkafka/tests/0149-broker-same-host-port.c +140 -0
  577. package/deps/librdkafka/tests/0150-telemetry_mock.c +651 -0
  578. package/deps/librdkafka/tests/0151-purge-brokers.c +566 -0
  579. package/deps/librdkafka/tests/0152-rebootstrap.c +59 -0
  580. package/deps/librdkafka/tests/0153-memberid.c +128 -0
  581. package/deps/librdkafka/tests/1000-unktopic.c +164 -0
  582. package/deps/librdkafka/tests/8000-idle.cpp +60 -0
  583. package/deps/librdkafka/tests/8001-fetch_from_follower_mock_manual.c +113 -0
  584. package/deps/librdkafka/tests/CMakeLists.txt +170 -0
  585. package/deps/librdkafka/tests/LibrdkafkaTestApp.py +291 -0
  586. package/deps/librdkafka/tests/Makefile +182 -0
  587. package/deps/librdkafka/tests/README.md +509 -0
  588. package/deps/librdkafka/tests/autotest.sh +33 -0
  589. package/deps/librdkafka/tests/backtrace.gdb +30 -0
  590. package/deps/librdkafka/tests/broker_version_tests.py +315 -0
  591. package/deps/librdkafka/tests/buildbox.sh +17 -0
  592. package/deps/librdkafka/tests/cleanup-checker-tests.sh +20 -0
  593. package/deps/librdkafka/tests/cluster_testing.py +191 -0
  594. package/deps/librdkafka/tests/delete-test-topics.sh +56 -0
  595. package/deps/librdkafka/tests/fixtures/oauthbearer/jwt_assertion_template.json +10 -0
  596. package/deps/librdkafka/tests/fixtures/ssl/Makefile +8 -0
  597. package/deps/librdkafka/tests/fixtures/ssl/README.md +13 -0
  598. package/deps/librdkafka/tests/fixtures/ssl/client.keystore.intermediate.p12 +0 -0
  599. package/deps/librdkafka/tests/fixtures/ssl/client.keystore.p12 +0 -0
  600. package/deps/librdkafka/tests/fixtures/ssl/client2.certificate.intermediate.pem +72 -0
  601. package/deps/librdkafka/tests/fixtures/ssl/client2.certificate.pem +50 -0
  602. package/deps/librdkafka/tests/fixtures/ssl/client2.intermediate.key +46 -0
  603. package/deps/librdkafka/tests/fixtures/ssl/client2.key +46 -0
  604. package/deps/librdkafka/tests/fixtures/ssl/create_keys.sh +168 -0
  605. package/deps/librdkafka/tests/fuzzers/Makefile +12 -0
  606. package/deps/librdkafka/tests/fuzzers/README.md +31 -0
  607. package/deps/librdkafka/tests/fuzzers/fuzz_regex.c +74 -0
  608. package/deps/librdkafka/tests/fuzzers/helpers.h +90 -0
  609. package/deps/librdkafka/tests/gen-ssl-certs.sh +165 -0
  610. package/deps/librdkafka/tests/interactive_broker_version.py +170 -0
  611. package/deps/librdkafka/tests/interceptor_test/CMakeLists.txt +16 -0
  612. package/deps/librdkafka/tests/interceptor_test/Makefile +22 -0
  613. package/deps/librdkafka/tests/interceptor_test/interceptor_test.c +314 -0
  614. package/deps/librdkafka/tests/interceptor_test/interceptor_test.h +54 -0
  615. package/deps/librdkafka/tests/java/IncrementalRebalanceCli.java +97 -0
  616. package/deps/librdkafka/tests/java/Makefile +13 -0
  617. package/deps/librdkafka/tests/java/Murmur2Cli.java +46 -0
  618. package/deps/librdkafka/tests/java/README.md +14 -0
  619. package/deps/librdkafka/tests/java/TransactionProducerCli.java +162 -0
  620. package/deps/librdkafka/tests/java/run-class.sh +11 -0
  621. package/deps/librdkafka/tests/librdkafka.suppressions +483 -0
  622. package/deps/librdkafka/tests/lz4_manual_test.sh +59 -0
  623. package/deps/librdkafka/tests/multi-broker-version-test.sh +50 -0
  624. package/deps/librdkafka/tests/parse-refcnt.sh +43 -0
  625. package/deps/librdkafka/tests/performance_plot.py +115 -0
  626. package/deps/librdkafka/tests/plugin_test/Makefile +19 -0
  627. package/deps/librdkafka/tests/plugin_test/plugin_test.c +58 -0
  628. package/deps/librdkafka/tests/requirements.txt +2 -0
  629. package/deps/librdkafka/tests/run-all-tests.sh +79 -0
  630. package/deps/librdkafka/tests/run-consumer-tests.sh +16 -0
  631. package/deps/librdkafka/tests/run-producer-tests.sh +16 -0
  632. package/deps/librdkafka/tests/run-test-batches.py +157 -0
  633. package/deps/librdkafka/tests/run-test.sh +140 -0
  634. package/deps/librdkafka/tests/rusage.c +249 -0
  635. package/deps/librdkafka/tests/sasl_test.py +289 -0
  636. package/deps/librdkafka/tests/scenarios/README.md +6 -0
  637. package/deps/librdkafka/tests/scenarios/ak23.json +6 -0
  638. package/deps/librdkafka/tests/scenarios/default.json +5 -0
  639. package/deps/librdkafka/tests/scenarios/noautocreate.json +5 -0
  640. package/deps/librdkafka/tests/sockem.c +801 -0
  641. package/deps/librdkafka/tests/sockem.h +85 -0
  642. package/deps/librdkafka/tests/sockem_ctrl.c +145 -0
  643. package/deps/librdkafka/tests/sockem_ctrl.h +61 -0
  644. package/deps/librdkafka/tests/test.c +7778 -0
  645. package/deps/librdkafka/tests/test.conf.example +27 -0
  646. package/deps/librdkafka/tests/test.h +1028 -0
  647. package/deps/librdkafka/tests/testcpp.cpp +131 -0
  648. package/deps/librdkafka/tests/testcpp.h +388 -0
  649. package/deps/librdkafka/tests/testshared.h +416 -0
  650. package/deps/librdkafka/tests/tools/README.md +4 -0
  651. package/deps/librdkafka/tests/tools/stats/README.md +21 -0
  652. package/deps/librdkafka/tests/tools/stats/filter.jq +42 -0
  653. package/deps/librdkafka/tests/tools/stats/graph.py +150 -0
  654. package/deps/librdkafka/tests/tools/stats/requirements.txt +3 -0
  655. package/deps/librdkafka/tests/tools/stats/to_csv.py +124 -0
  656. package/deps/librdkafka/tests/trivup/trivup-0.14.0.tar.gz +0 -0
  657. package/deps/librdkafka/tests/until-fail.sh +87 -0
  658. package/deps/librdkafka/tests/xxxx-assign_partition.c +122 -0
  659. package/deps/librdkafka/tests/xxxx-metadata.cpp +159 -0
  660. package/deps/librdkafka/vcpkg.json +23 -0
  661. package/deps/librdkafka/win32/README.md +5 -0
  662. package/deps/librdkafka/win32/build-package.bat +3 -0
  663. package/deps/librdkafka/win32/build.bat +19 -0
  664. package/deps/librdkafka/win32/common.vcxproj +84 -0
  665. package/deps/librdkafka/win32/interceptor_test/interceptor_test.vcxproj +87 -0
  666. package/deps/librdkafka/win32/librdkafka.autopkg.template +54 -0
  667. package/deps/librdkafka/win32/librdkafka.master.testing.targets +13 -0
  668. package/deps/librdkafka/win32/librdkafka.sln +226 -0
  669. package/deps/librdkafka/win32/librdkafka.vcxproj +276 -0
  670. package/deps/librdkafka/win32/librdkafkacpp/librdkafkacpp.vcxproj +104 -0
  671. package/deps/librdkafka/win32/msbuild.ps1 +15 -0
  672. package/deps/librdkafka/win32/openssl_engine_example/openssl_engine_example.vcxproj +132 -0
  673. package/deps/librdkafka/win32/package-zip.ps1 +46 -0
  674. package/deps/librdkafka/win32/packages/repositories.config +4 -0
  675. package/deps/librdkafka/win32/push-package.bat +4 -0
  676. package/deps/librdkafka/win32/rdkafka_complex_consumer_example_cpp/rdkafka_complex_consumer_example_cpp.vcxproj +67 -0
  677. package/deps/librdkafka/win32/rdkafka_example/rdkafka_example.vcxproj +97 -0
  678. package/deps/librdkafka/win32/rdkafka_performance/rdkafka_performance.vcxproj +97 -0
  679. package/deps/librdkafka/win32/setup-msys2.ps1 +47 -0
  680. package/deps/librdkafka/win32/setup-vcpkg.ps1 +34 -0
  681. package/deps/librdkafka/win32/tests/test.conf.example +25 -0
  682. package/deps/librdkafka/win32/tests/tests.vcxproj +253 -0
  683. package/deps/librdkafka/win32/win_ssl_cert_store/win_ssl_cert_store.vcxproj +132 -0
  684. package/deps/librdkafka/win32/wingetopt.c +564 -0
  685. package/deps/librdkafka/win32/wingetopt.h +101 -0
  686. package/deps/librdkafka/win32/wintime.h +33 -0
  687. package/deps/librdkafka.gyp +62 -0
  688. package/lib/admin.js +233 -0
  689. package/lib/client.js +573 -0
  690. package/lib/error.js +500 -0
  691. package/lib/index.js +34 -0
  692. package/lib/kafka-consumer-stream.js +397 -0
  693. package/lib/kafka-consumer.js +698 -0
  694. package/lib/producer/high-level-producer.js +323 -0
  695. package/lib/producer-stream.js +307 -0
  696. package/lib/producer.js +375 -0
  697. package/lib/tools/ref-counter.js +52 -0
  698. package/lib/topic-partition.js +88 -0
  699. package/lib/topic.js +42 -0
  700. package/lib/util.js +29 -0
  701. package/package.json +61 -0
  702. package/prebuilds/darwin-arm64/@point3+node-rdkafka.node +0 -0
  703. package/prebuilds/linux-x64/@point3+node-rdkafka.node +0 -0
  704. package/util/configure.js +30 -0
  705. package/util/get-env.js +6 -0
  706. package/util/test-compile.js +11 -0
  707. package/util/test-producer-delivery.js +100 -0
@@ -0,0 +1,1748 @@
1
+ /*
2
+ * librdkafka - The Apache Kafka C/C++ library
3
+ *
4
+ * Copyright (c) 2015-2022, Magnus Edenhill
5
+ * 2023, Confluent Inc.
6
+ * All rights reserved.
7
+ *
8
+ * Redistribution and use in source and binary forms, with or without
9
+ * modification, are permitted provided that the following conditions are met:
10
+ *
11
+ * 1. Redistributions of source code must retain the above copyright notice,
12
+ * this list of conditions and the following disclaimer.
13
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
14
+ * this list of conditions and the following disclaimer in the documentation
15
+ * and/or other materials provided with the distribution.
16
+ *
17
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
+ * POSSIBILITY OF SUCH DAMAGE.
28
+ */
29
+ #include "rdkafka_int.h"
30
+ #include "rdkafka_assignor.h"
31
+ #include "rdunittest.h"
32
+
33
+
34
+ /**
35
+ * Source:
36
+ * https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/clients/consumer/RangeAssignor.java
37
+ *
38
+ * The range assignor works on a per-topic basis. For each topic, we lay out the
39
+ * available partitions in numeric order and the consumers in lexicographic
40
+ * order. We then divide the number of partitions by the total number of
41
+ * consumers to determine the number of partitions to assign to each consumer.
42
+ * If it does not evenly divide, then the first few consumers will have one
43
+ * extra partition.
44
+ *
45
+ * For example, suppose there are two consumers C0 and C1, two topics t0 and t1,
46
+ * and each topic has 3 partitions, resulting in partitions t0p0, t0p1, t0p2,
47
+ * t1p0, t1p1, and t1p2.
48
+ *
49
+ * The assignment will be:
50
+ * C0: [t0p0, t0p1, t1p0, t1p1]
51
+ * C1: [t0p2, t1p2]
52
+ */
53
+
54
+ typedef struct {
55
+ rd_kafkap_str_t *member_id;
56
+ rd_list_t *assigned_partitions; /* Contained Type: int* */
57
+ } rd_kafka_member_assigned_partitions_pair_t;
58
+
59
+ /**
60
+ * @brief Intializes a rd_kafka_member_assigned_partitions_pair_t* with
61
+ * assigned_partitions = [].
62
+ *
63
+ * @param member_id
64
+ *
65
+ * The member_id isn't copied, so the returned value can be used only for the
66
+ * lifetime of this function's arguments.
67
+ * @return rd_kafka_member_assigned_partitions_pair_t*
68
+ */
69
+ static rd_kafka_member_assigned_partitions_pair_t *
70
+ rd_kafka_member_assigned_partitions_pair_new(rd_kafkap_str_t *member_id) {
71
+ rd_kafka_member_assigned_partitions_pair_t *pair =
72
+ rd_calloc(1, sizeof(rd_kafka_member_assigned_partitions_pair_t));
73
+
74
+ pair->member_id = member_id;
75
+ pair->assigned_partitions = rd_list_new(0, NULL);
76
+ return pair;
77
+ }
78
+
79
+ static void rd_kafka_member_assigned_partitions_pair_destroy(void *_pair) {
80
+ rd_kafka_member_assigned_partitions_pair_t *pair =
81
+ (rd_kafka_member_assigned_partitions_pair_t *)_pair;
82
+
83
+ /* Do not destroy the member_id, we don't take ownership. */
84
+ RD_IF_FREE(pair->assigned_partitions, rd_list_destroy);
85
+ RD_IF_FREE(pair, rd_free);
86
+ }
87
+
88
+ static int rd_kafka_member_assigned_partitions_pair_cmp(const void *_a,
89
+ const void *_b) {
90
+ rd_kafka_member_assigned_partitions_pair_t *a =
91
+ (rd_kafka_member_assigned_partitions_pair_t *)_a;
92
+ rd_kafka_member_assigned_partitions_pair_t *b =
93
+ (rd_kafka_member_assigned_partitions_pair_t *)_b;
94
+ return rd_kafkap_str_cmp(a->member_id, b->member_id);
95
+ }
96
+
97
+ static rd_kafka_member_assigned_partitions_pair_t *
98
+ rd_kafka_find_member_assigned_partitions_pair_by_member_id(
99
+ rd_kafkap_str_t *member_id,
100
+ rd_list_t *rd_kafka_member_assigned_partitions_pair_list) {
101
+ rd_kafka_member_assigned_partitions_pair_t search_pair = {member_id,
102
+ NULL};
103
+ return rd_list_find(rd_kafka_member_assigned_partitions_pair_list,
104
+ &search_pair,
105
+ rd_kafka_member_assigned_partitions_pair_cmp);
106
+ }
107
+
108
+ typedef struct {
109
+ /* Contains topic and list of members - sorted by group instance id and
110
+ * member id. Also contains partitions, along with partition replicas,
111
+ * which will help us with the racks. The members also contain their
112
+ * rack id and the partitions they have already been assigned.
113
+ */
114
+ rd_kafka_assignor_topic_t *topic;
115
+ /* unassigned_partitions[i] is true if the ith partition of this topic
116
+ * is not assigned. We prefer using an array rather than using an
117
+ * rd_list and removing elements, because that involves a memmove on
118
+ * each remove. */
119
+ rd_bool_t *unassigned_partitions;
120
+ /* Number of partitions still to be assigned.*/
121
+ size_t unassigned_partitions_left;
122
+ /* An array of char** arrays. The ith element of this array is a sorted
123
+ * char** array, denoting the racks for the ith partition of this topic.
124
+ * The size of this array is equal to the partition_cnt. */
125
+ char ***partition_racks;
126
+ /* The ith element of this array is the size of partition_racks[i]. */
127
+ size_t *racks_cnt;
128
+ /* Contains a pair denoting the partitions assigned to every subscribed
129
+ * consumer (member, [rd_list_t* of int*]). Sorted by member_id.
130
+ * Contained Type: rd_kafka_member_assigned_partitions_pair_t* */
131
+ rd_list_t *member_to_assigned_partitions;
132
+ /* Contains the number of partitions that should be ideally assigned to
133
+ * every subscribing consumer. */
134
+ int num_partitions_per_consumer;
135
+ /* Contains the number of consumers with extra partitions in case number
136
+ * of partitions isn't perfectly divisible by number of consumers. */
137
+ int remaining_consumers_with_extra_partition;
138
+ /* True if we need to perform rack aware assignment. */
139
+ rd_bool_t needs_rack_aware_assignment;
140
+ } rd_kafka_topic_assignment_state_t;
141
+
142
+
143
+ /**
144
+ * @brief Initialize an rd_kafka_topic_assignment_state_t.
145
+ *
146
+ * @param topic
147
+ * @param broker_rack_pair
148
+ * @param broker_rack_pair_cnt
149
+ *
150
+ * The struct rd_kafka_topic_assignment_state_t is mostly for convenience and
151
+ * easy grouping, so we avoid copying values as much as possible. Hence, the
152
+ * returned rd_kafka_topic_assignment_state_t does not own all its values, and
153
+ * should not be used beyond the lifetime of this function's arguments. This
154
+ * function also computes the value of needsRackAwareAssignment given the other
155
+ * information.
156
+ *
157
+ * @return rd_kafka_topic_assignment_state_t*
158
+ */
159
+
160
+ static rd_kafka_topic_assignment_state_t *
161
+ rd_kafka_topic_assignment_state_new(rd_kafka_assignor_topic_t *topic,
162
+ const rd_kafka_metadata_internal_t *mdi) {
163
+ int i;
164
+ rd_kafka_group_member_t *member;
165
+ rd_kafka_topic_assignment_state_t *rktas;
166
+ const int partition_cnt = topic->metadata->partition_cnt;
167
+
168
+ rktas = rd_calloc(1, sizeof(rd_kafka_topic_assignment_state_t));
169
+ rktas->topic = topic; /* don't copy. */
170
+
171
+ rktas->unassigned_partitions =
172
+ rd_malloc(sizeof(rd_bool_t) * partition_cnt);
173
+ rktas->unassigned_partitions_left = partition_cnt;
174
+ for (i = 0; i < partition_cnt; i++) {
175
+ rktas->unassigned_partitions[i] = rd_true;
176
+ }
177
+
178
+ rktas->num_partitions_per_consumer = 0;
179
+ rktas->remaining_consumers_with_extra_partition = 0;
180
+ if (rd_list_cnt(&topic->members)) {
181
+ rktas->num_partitions_per_consumer =
182
+ partition_cnt / rd_list_cnt(&topic->members);
183
+ rktas->remaining_consumers_with_extra_partition =
184
+ partition_cnt % rd_list_cnt(&topic->members);
185
+ }
186
+
187
+ rktas->member_to_assigned_partitions =
188
+ rd_list_new(0, rd_kafka_member_assigned_partitions_pair_destroy);
189
+
190
+ RD_LIST_FOREACH(member, &topic->members, i) {
191
+ rd_list_add(rktas->member_to_assigned_partitions,
192
+ rd_kafka_member_assigned_partitions_pair_new(
193
+ member->rkgm_member_id));
194
+ }
195
+
196
+ rd_list_sort(rktas->member_to_assigned_partitions,
197
+ rd_kafka_member_assigned_partitions_pair_cmp);
198
+
199
+ rktas->partition_racks = rd_calloc(partition_cnt, sizeof(char **));
200
+ rktas->racks_cnt = rd_calloc(partition_cnt, sizeof(size_t));
201
+ for (i = 0; topic->metadata_internal->partitions && i < partition_cnt;
202
+ i++) {
203
+ rktas->racks_cnt[i] =
204
+ topic->metadata_internal->partitions[i].racks_cnt;
205
+ rktas->partition_racks[i] =
206
+ topic->metadata_internal->partitions[i].racks;
207
+ }
208
+
209
+ rktas->needs_rack_aware_assignment =
210
+ rd_kafka_use_rack_aware_assignment(&topic, 1, mdi);
211
+
212
+ return rktas;
213
+ }
214
+
215
+ /* Destroy a rd_kafka_topic_assignment_state_t. */
216
+ static void rd_kafka_topic_assignment_state_destroy(void *_rktas) {
217
+ rd_kafka_topic_assignment_state_t *rktas =
218
+ (rd_kafka_topic_assignment_state_t *)_rktas;
219
+
220
+ rd_free(rktas->unassigned_partitions);
221
+ rd_list_destroy(rktas->member_to_assigned_partitions);
222
+ rd_free(rktas->partition_racks);
223
+ rd_free(rktas->racks_cnt);
224
+ rd_free(rktas);
225
+ }
226
+
227
+ /**
228
+ * Compare two topic_assignment_states, first on the sorted list of consumers
229
+ * (each consumer from the list of consumers is matched till the first point of
230
+ * difference), and if that's equal, compare on the number of partitions.
231
+ *
232
+ * A list sorted with this comparator will group the topic_assignment_states
233
+ * having the same consumers and the same number of partitions together - this
234
+ * is the criteria of co-partitioned topics.
235
+ */
236
+ static int rd_kafka_topic_assignment_state_cmp(const void *_a, const void *_b) {
237
+ int i;
238
+ rd_kafka_topic_assignment_state_t *a =
239
+ (rd_kafka_topic_assignment_state_t *)_a;
240
+ rd_kafka_topic_assignment_state_t *b =
241
+ (rd_kafka_topic_assignment_state_t *)_b;
242
+
243
+ /* This guarantee comes from rd_kafka_range_assignor_assign_cb. */
244
+ rd_assert(a->topic->members.rl_flags & RD_LIST_F_SORTED);
245
+ rd_assert(b->topic->members.rl_flags & RD_LIST_F_SORTED);
246
+
247
+ /* Based on consumers */
248
+ for (i = 0; i < rd_list_cnt(&a->topic->members) &&
249
+ i < rd_list_cnt(&b->topic->members);
250
+ i++) {
251
+ rd_kafka_group_member_t *am =
252
+ rd_list_elem(&a->topic->members, i);
253
+ rd_kafka_group_member_t *bm =
254
+ rd_list_elem(&b->topic->members, i);
255
+ int cmp_res =
256
+ rd_kafkap_str_cmp(am->rkgm_member_id, bm->rkgm_member_id);
257
+ if (cmp_res != 0)
258
+ return cmp_res;
259
+ }
260
+
261
+ if (rd_list_cnt(&a->topic->members) !=
262
+ rd_list_cnt(&b->topic->members)) {
263
+ return RD_CMP(rd_list_cnt(&a->topic->members),
264
+ rd_list_cnt(&b->topic->members));
265
+ }
266
+
267
+ /* Based on number of partitions */
268
+ return RD_CMP(a->topic->metadata->partition_cnt,
269
+ b->topic->metadata->partition_cnt);
270
+ }
271
+
272
+
273
+ /* Helper function to wrap a bsearch on the partition's racks. */
274
+ static char *rd_kafka_topic_assignment_state_rack_search(
275
+ rd_kafka_topic_assignment_state_t *rktas,
276
+ int partition,
277
+ const char *rack) {
278
+ char **partition_racks = rktas->partition_racks[partition];
279
+ size_t cnt = rktas->racks_cnt[partition];
280
+ void *res = NULL;
281
+
282
+ if (!partition_racks)
283
+ return NULL;
284
+
285
+ res = bsearch(&rack, partition_racks, cnt, sizeof(char *), rd_strcmp3);
286
+ if (!res)
287
+ return NULL;
288
+
289
+ return *(char **)res;
290
+ }
291
+
292
+ /*
293
+ * Assigns a partition to a member, and updates fields in rktas for accounting.
294
+ * It's assumed that the partitions assigned to this member don't exceed the
295
+ * allowed number.
296
+ */
297
+ static void rd_kafka_assign_partition(rd_kafka_group_member_t *member,
298
+ rd_kafka_topic_assignment_state_t *rktas,
299
+ int32_t partition) {
300
+ rd_kafka_member_assigned_partitions_pair_t *member_assignment =
301
+ rd_kafka_find_member_assigned_partitions_pair_by_member_id(
302
+ member->rkgm_member_id, rktas->member_to_assigned_partitions);
303
+ rd_assert(member_assignment);
304
+
305
+ /* We can't use &partition, since that's a copy on the stack. */
306
+ rd_list_add(member_assignment->assigned_partitions,
307
+ (void *)&rktas->topic->metadata->partitions[partition].id);
308
+ rd_kafka_topic_partition_list_add_range(member->rkgm_assignment,
309
+ rktas->topic->metadata->topic,
310
+ partition, partition);
311
+
312
+ rd_assert(rktas->unassigned_partitions[partition]);
313
+ rktas->unassigned_partitions[partition] = rd_false;
314
+ rktas->unassigned_partitions_left--;
315
+
316
+ if (rd_list_cnt(member_assignment->assigned_partitions) >
317
+ rktas->num_partitions_per_consumer) {
318
+ rktas->remaining_consumers_with_extra_partition -= 1;
319
+ }
320
+ }
321
+
322
+
323
+ /* Implementation of may_assign for rd_kafka_assign_ranges. True if the consumer
324
+ * rack is empty, or if is exists within the partition racks. */
325
+ static rd_bool_t rd_kafka_racks_match(rd_kafka_group_member_t *member,
326
+ rd_kafka_topic_assignment_state_t *rktas,
327
+ int32_t partition) {
328
+ rd_kafkap_str_t *consumer_rack = member->rkgm_rack_id;
329
+
330
+ if (!consumer_rack || RD_KAFKAP_STR_LEN(consumer_rack) == 0) {
331
+ return rd_true;
332
+ }
333
+
334
+ return rd_kafka_topic_assignment_state_rack_search(
335
+ rktas, partition, consumer_rack->str) != NULL;
336
+ }
337
+
338
+
339
+ /* Implementation of may_assign for rd_kafka_assign_ranges. Always true, used to
340
+ * assign remaining partitions after rack-aware assignment is complete. */
341
+ static rd_bool_t rd_kafka_always(rd_kafka_group_member_t *member,
342
+ rd_kafka_topic_assignment_state_t *rktas,
343
+ int32_t partition) {
344
+ return rd_true;
345
+ }
346
+
347
+ /* Assigns as many partitions as possible for a topic to subscribing members,
348
+ * such that no subscribing member exceeds their limit of allowed partitions,
349
+ * and may_assign(member, rktas, partition) is true for each member and
350
+ * partition.
351
+ */
352
+ static void rd_kafka_assign_ranges(
353
+ rd_kafka_topic_assignment_state_t *rktas,
354
+ rd_bool_t (*may_assign)(rd_kafka_group_member_t *member,
355
+ rd_kafka_topic_assignment_state_t *rktas,
356
+ int32_t partition)) {
357
+ int i;
358
+ rd_kafka_group_member_t *member;
359
+ int32_t *partitions_to_assign =
360
+ rd_alloca(rktas->unassigned_partitions_left * sizeof(int32_t));
361
+
362
+ RD_LIST_FOREACH(member, &rktas->topic->members, i) {
363
+ int j;
364
+ rd_kafka_member_assigned_partitions_pair_t *member_assignment;
365
+ int maximum_assignable_to_consumer;
366
+ int partitions_to_assign_cnt;
367
+
368
+ if (rktas->unassigned_partitions_left == 0)
369
+ break;
370
+
371
+ member_assignment =
372
+ rd_kafka_find_member_assigned_partitions_pair_by_member_id(
373
+ member->rkgm_member_id,
374
+ rktas->member_to_assigned_partitions);
375
+
376
+ maximum_assignable_to_consumer =
377
+ rktas->num_partitions_per_consumer +
378
+ (rktas->remaining_consumers_with_extra_partition > 0) -
379
+ rd_list_cnt(member_assignment->assigned_partitions);
380
+
381
+ if (maximum_assignable_to_consumer <= 0)
382
+ continue;
383
+
384
+ partitions_to_assign_cnt = 0;
385
+ for (j = 0; j < rktas->topic->metadata->partition_cnt; j++) {
386
+ if (!rktas->unassigned_partitions[j]) {
387
+ continue;
388
+ }
389
+
390
+ if (maximum_assignable_to_consumer <= 0)
391
+ break;
392
+
393
+ if (!may_assign(member, rktas, j))
394
+ continue;
395
+
396
+ partitions_to_assign[partitions_to_assign_cnt] = j;
397
+ partitions_to_assign_cnt++;
398
+ maximum_assignable_to_consumer--;
399
+ }
400
+
401
+ for (j = 0; j < partitions_to_assign_cnt; j++)
402
+ rd_kafka_assign_partition(member, rktas,
403
+ partitions_to_assign[j]);
404
+ }
405
+ }
406
+
407
+ /*
408
+ * Assigns partitions for co-partitioned topics in a rack-aware manner on a best
409
+ * effort basis. All partitions may not be assigned to consumers in case a rack
410
+ * aware assignment does not exist.
411
+ */
412
+ static void rd_kafka_assign_co_partitioned(
413
+ rd_list_t *
414
+ rktas_bucket /* Contained Type: rd_kafka_topic_assignment_state_t* */) {
415
+ rd_kafka_topic_assignment_state_t *first_rktas =
416
+ rd_list_elem(rktas_bucket, 0);
417
+ rd_kafka_topic_assignment_state_t *rktas;
418
+ rd_kafka_group_member_t *member;
419
+ int i;
420
+
421
+ /* Since a "bucket" is a group of topic_assignment_states with the same
422
+ * consumers and number of partitions, we can just fetch them from the
423
+ * first member of the bucket. */
424
+ const int partition_cnt = first_rktas->topic->metadata->partition_cnt;
425
+ const rd_list_t *consumers = &first_rktas->topic->members;
426
+
427
+ for (i = 0; i < partition_cnt; i++) {
428
+ /*
429
+ * To assign the ith partition of all the co partitioned topics,
430
+ * we need to find a consumerX that fulfils the criteria:
431
+ * for all topic_assignment_states in the bucket:
432
+ * 1. rack(consumerX) is contained inside racks(partition i)
433
+ * 2. partitions assigned to consumerX does not exceed limits.
434
+ */
435
+ int j;
436
+ RD_LIST_FOREACH(member, consumers, j) {
437
+ int m;
438
+ RD_LIST_FOREACH(rktas, rktas_bucket, m) {
439
+ int maximum_assignable;
440
+ rd_kafka_member_assigned_partitions_pair_t
441
+ *member_assignment;
442
+
443
+ /* Check (1.) */
444
+ if (!member->rkgm_rack_id ||
445
+ RD_KAFKAP_STR_LEN(member->rkgm_rack_id) ==
446
+ 0 ||
447
+ rd_kafka_topic_assignment_state_rack_search(
448
+ rktas, i, member->rkgm_rack_id->str) ==
449
+ NULL) {
450
+ break;
451
+ }
452
+
453
+ /* Check (2.) */
454
+ member_assignment =
455
+ rd_kafka_find_member_assigned_partitions_pair_by_member_id(
456
+ member->rkgm_member_id,
457
+ rktas->member_to_assigned_partitions);
458
+ maximum_assignable =
459
+ rktas->num_partitions_per_consumer +
460
+ (rktas
461
+ ->remaining_consumers_with_extra_partition >
462
+ 0) -
463
+ rd_list_cnt(
464
+ member_assignment->assigned_partitions);
465
+
466
+ if (maximum_assignable <= 0) {
467
+ break;
468
+ }
469
+ }
470
+ if (m == rd_list_cnt(rktas_bucket)) {
471
+ /* Break early - this consumer can be assigned
472
+ * this partition. */
473
+ break;
474
+ }
475
+ }
476
+ if (j == rd_list_cnt(&first_rktas->topic->members)) {
477
+ continue; /* We didn't find a suitable consumer. */
478
+ }
479
+
480
+ rd_assert(member);
481
+
482
+ RD_LIST_FOREACH(rktas, rktas_bucket, j) {
483
+ rd_kafka_assign_partition(member, rktas, i);
484
+ }
485
+
486
+ /* FIXME: A possible optimization: early break here if no
487
+ * consumer remains with maximum_assignable_to_consumer > 0
488
+ * across all topics. */
489
+ }
490
+ }
491
+
492
+
493
+ rd_kafka_resp_err_t
494
+ rd_kafka_range_assignor_assign_cb(rd_kafka_t *rk,
495
+ const rd_kafka_assignor_t *rkas,
496
+ const char *member_id,
497
+ const rd_kafka_metadata_t *metadata,
498
+ rd_kafka_group_member_t *members,
499
+ size_t member_cnt,
500
+ rd_kafka_assignor_topic_t **eligible_topics,
501
+ size_t eligible_topic_cnt,
502
+ char *errstr,
503
+ size_t errstr_size,
504
+ void *opaque) {
505
+ unsigned int ti;
506
+ int i;
507
+ rd_list_t *rktas_list = rd_list_new(
508
+ eligible_topic_cnt, rd_kafka_topic_assignment_state_destroy);
509
+ rd_list_t *rktas_buckets = rd_list_new(0, rd_list_destroy_free);
510
+ rd_list_t
511
+ *rktas_current_bucket; /* Contained Type:
512
+ rd_kafka_topic_assignment_state_t* */
513
+ rd_kafka_topic_assignment_state_t *rktas;
514
+ rd_kafka_topic_assignment_state_t *prev_rktas;
515
+ const rd_kafka_metadata_internal_t *mdi =
516
+ rd_kafka_metadata_get_internal(metadata);
517
+
518
+ /* The range assignor works on a per-topic basis. */
519
+ for (ti = 0; ti < eligible_topic_cnt; ti++) {
520
+ rd_kafka_assignor_topic_t *eligible_topic = eligible_topics[ti];
521
+
522
+ /* For each topic, we sort the consumers in lexicographic order,
523
+ * and create a topic_assignment_state. */
524
+ rd_list_sort(&eligible_topic->members,
525
+ rd_kafka_group_member_cmp);
526
+ rd_list_add(rktas_list, rd_kafka_topic_assignment_state_new(
527
+ eligible_topic, mdi));
528
+ }
529
+
530
+ /* Sort the topic_assignment_states to group the topics which need to be
531
+ * co-partitioned. */
532
+ rd_list_sort(rktas_list, rd_kafka_topic_assignment_state_cmp);
533
+
534
+ /* Use the sorted list of topic_assignment_states and separate them into
535
+ * "buckets". Each bucket contains topics which can be co-partitioned,
536
+ * ie with the same consumers and number of partitions. */
537
+ prev_rktas = NULL;
538
+ rktas_current_bucket = NULL;
539
+ RD_LIST_FOREACH(rktas, rktas_list, i) {
540
+ if (prev_rktas && rd_kafka_topic_assignment_state_cmp(
541
+ rktas, prev_rktas) == 0) {
542
+ rd_list_add(rktas_current_bucket, rktas);
543
+ continue;
544
+ }
545
+
546
+ /* The free function is set to NULL, as we don't copy any of the
547
+ * topic_assignment_states. */
548
+ rktas_current_bucket = rd_list_new(0, NULL);
549
+ rd_list_add(rktas_buckets, rktas_current_bucket);
550
+ prev_rktas = rktas;
551
+ rd_list_add(rktas_current_bucket, rktas);
552
+ }
553
+
554
+ /* Iterate through each bucket. In case there's more than one element in
555
+ * the bucket, we prefer co-partitioning over rack awareness. Otherwise,
556
+ * assign with rack-awareness. */
557
+ rktas = NULL;
558
+ rktas_current_bucket = NULL;
559
+ RD_LIST_FOREACH(rktas_current_bucket, rktas_buckets, i) {
560
+ rd_assert(rd_list_cnt(rktas_current_bucket) > 0);
561
+
562
+ if (rd_list_cnt(rktas_current_bucket) == 1) {
563
+ rktas = rd_list_elem(rktas_current_bucket, 0);
564
+ if (!rktas->needs_rack_aware_assignment)
565
+ continue;
566
+
567
+
568
+ rd_kafka_dbg(rk, CGRP, "ASSIGN",
569
+ "range: Topic %s with %d partition(s) and "
570
+ "%d subscribing member(s), single-topic "
571
+ "rack-aware assignment",
572
+ rktas->topic->metadata->topic,
573
+ rktas->topic->metadata->partition_cnt,
574
+ rd_list_cnt(&rktas->topic->members));
575
+
576
+ rd_kafka_assign_ranges(rktas, rd_kafka_racks_match);
577
+ } else {
578
+ rktas = rd_list_elem(rktas_current_bucket, 0);
579
+ rd_kafka_dbg(
580
+ rk, CGRP, "ASSIGN",
581
+ "range: %d topics with %d partition(s) and "
582
+ "%d subscribing member(s), co-partitioned "
583
+ "rack-aware assignment",
584
+ rd_list_cnt(rktas_current_bucket),
585
+ rktas->topic->metadata->partition_cnt,
586
+ rd_list_cnt(&rktas->topic->members));
587
+
588
+ rd_kafka_assign_co_partitioned(rktas_current_bucket);
589
+ }
590
+ }
591
+
592
+ /* Iterate through each rktas, doing normal assignment for any
593
+ * partitions that might not have gotten a rack-aware assignment.*/
594
+ RD_LIST_FOREACH(rktas, rktas_list, i) {
595
+ rd_kafka_dbg(rk, CGRP, "ASSIGN",
596
+ "range: Topic %s with %d partition(s) and "
597
+ "%d subscribing member(s), single-topic "
598
+ "non-rack-aware assignment for %" PRIusz
599
+ " leftover partitions",
600
+ rktas->topic->metadata->topic,
601
+ rktas->topic->metadata->partition_cnt,
602
+ rd_list_cnt(&rktas->topic->members),
603
+ rktas->unassigned_partitions_left);
604
+ rd_kafka_assign_ranges(rktas, rd_kafka_always);
605
+ }
606
+
607
+ rd_list_destroy(rktas_list);
608
+ rd_list_destroy(rktas_buckets);
609
+
610
+ return 0;
611
+ }
612
+
613
+
614
+ /**
615
+ * @name Sticky assignor unit tests
616
+ *
617
+ *
618
+ * These are based on RangeAssignorTest.java
619
+ *
620
+ *
621
+ *
622
+ */
623
+
624
+
625
+ /* All possible racks used in tests, as well as several common rack configs used
626
+ * by consumers */
627
+ static rd_kafkap_str_t
628
+ *ALL_RACKS[7]; /* initialized before starting the unit tests. */
629
+ static int RACKS_INITIAL[] = {0, 1, 2};
630
+ static int RACKS_NULL[] = {6, 6, 6};
631
+ static int RACKS_FINAL[] = {4, 5, 6};
632
+ static int RACKS_ONE_NULL[] = {6, 4, 5};
633
+
634
+ static int
635
+ ut_testOneConsumerNoTopic(rd_kafka_t *rk,
636
+ const rd_kafka_assignor_t *rkas,
637
+ rd_kafka_assignor_ut_rack_config_t parametrization) {
638
+ rd_kafka_resp_err_t err;
639
+ char errstr[512];
640
+ rd_kafka_metadata_t *metadata;
641
+ rd_kafka_group_member_t members[1];
642
+
643
+
644
+ if (parametrization == RD_KAFKA_RANGE_ASSIGNOR_UT_NO_BROKER_RACK) {
645
+ RD_UT_PASS();
646
+ }
647
+
648
+ ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
649
+ RD_ARRAYSIZE(ALL_RACKS), parametrization,
650
+ 0);
651
+
652
+ ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
653
+ parametrization, "t1", NULL);
654
+
655
+ err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
656
+ RD_ARRAYSIZE(members), errstr,
657
+ sizeof(errstr));
658
+ RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
659
+
660
+ verifyAssignment(&members[0], NULL);
661
+
662
+ rd_kafka_group_member_clear(&members[0]);
663
+ ut_destroy_metadata(metadata);
664
+
665
+ RD_UT_PASS();
666
+ }
667
+
668
+ static int ut_testOneConsumerNonexistentTopic(
669
+ rd_kafka_t *rk,
670
+ const rd_kafka_assignor_t *rkas,
671
+ rd_kafka_assignor_ut_rack_config_t parametrization) {
672
+ rd_kafka_resp_err_t err;
673
+ char errstr[512];
674
+ rd_kafka_metadata_t *metadata;
675
+ rd_kafka_group_member_t members[1];
676
+
677
+
678
+ if (parametrization == RD_KAFKA_RANGE_ASSIGNOR_UT_NO_BROKER_RACK) {
679
+ RD_UT_PASS();
680
+ }
681
+
682
+ ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
683
+ RD_ARRAYSIZE(ALL_RACKS), parametrization,
684
+ 1, "t1", 0);
685
+
686
+ ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
687
+ parametrization, "t1", NULL);
688
+
689
+ err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
690
+ RD_ARRAYSIZE(members), errstr,
691
+ sizeof(errstr));
692
+ RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
693
+
694
+ verifyAssignment(&members[0], NULL);
695
+
696
+ rd_kafka_group_member_clear(&members[0]);
697
+ ut_destroy_metadata(metadata);
698
+
699
+ RD_UT_PASS();
700
+ }
701
+
702
+
703
+ static int
704
+ ut_testOneConsumerOneTopic(rd_kafka_t *rk,
705
+ const rd_kafka_assignor_t *rkas,
706
+ rd_kafka_assignor_ut_rack_config_t parametrization) {
707
+ rd_kafka_resp_err_t err;
708
+ char errstr[512];
709
+ rd_kafka_metadata_t *metadata;
710
+ rd_kafka_group_member_t members[1];
711
+
712
+ ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
713
+ RD_ARRAYSIZE(ALL_RACKS), parametrization,
714
+ 1, "t1", 3);
715
+
716
+ ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
717
+ parametrization, "t1", NULL);
718
+
719
+ err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
720
+ RD_ARRAYSIZE(members), errstr,
721
+ sizeof(errstr));
722
+ RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
723
+ RD_UT_ASSERT(members[0].rkgm_assignment->cnt == 3,
724
+ "expected assignment of 3 partitions, got %d partition(s)",
725
+ members[0].rkgm_assignment->cnt);
726
+
727
+ verifyAssignment(&members[0], "t1", 0, "t1", 1, "t1", 2, NULL);
728
+
729
+ rd_kafka_group_member_clear(&members[0]);
730
+ ut_destroy_metadata(metadata);
731
+
732
+ RD_UT_PASS();
733
+ }
734
+
735
+
736
+ static int ut_testOnlyAssignsPartitionsFromSubscribedTopics(
737
+ rd_kafka_t *rk,
738
+ const rd_kafka_assignor_t *rkas,
739
+ rd_kafka_assignor_ut_rack_config_t parametrization) {
740
+ rd_kafka_resp_err_t err;
741
+ char errstr[512];
742
+ rd_kafka_metadata_t *metadata;
743
+ rd_kafka_group_member_t members[1];
744
+
745
+ ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
746
+ RD_ARRAYSIZE(ALL_RACKS), parametrization,
747
+ 2, "t1", 3, "t2", 3);
748
+
749
+ ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
750
+ parametrization, "t1", NULL);
751
+
752
+ err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
753
+ RD_ARRAYSIZE(members), errstr,
754
+ sizeof(errstr));
755
+ RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
756
+
757
+ verifyAssignment(&members[0], "t1", 0, "t1", 1, "t1", 2, NULL);
758
+
759
+ rd_kafka_group_member_clear(&members[0]);
760
+ ut_destroy_metadata(metadata);
761
+
762
+ RD_UT_PASS();
763
+ }
764
+
765
+ static int ut_testOneConsumerMultipleTopics(
766
+ rd_kafka_t *rk,
767
+ const rd_kafka_assignor_t *rkas,
768
+ rd_kafka_assignor_ut_rack_config_t parametrization) {
769
+ rd_kafka_resp_err_t err;
770
+ char errstr[512];
771
+ rd_kafka_metadata_t *metadata;
772
+ rd_kafka_group_member_t members[1];
773
+
774
+ ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
775
+ RD_ARRAYSIZE(ALL_RACKS), parametrization,
776
+ 2, "t1", 1, "t2", 2);
777
+
778
+ ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
779
+ parametrization, "t1", "t2", NULL);
780
+
781
+ err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
782
+ RD_ARRAYSIZE(members), errstr,
783
+ sizeof(errstr));
784
+ RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
785
+
786
+ verifyAssignment(&members[0], "t1", 0, "t2", 0, "t2", 1, NULL);
787
+
788
+ rd_kafka_group_member_clear(&members[0]);
789
+ ut_destroy_metadata(metadata);
790
+
791
+ RD_UT_PASS();
792
+ }
793
+
794
+ static int ut_testTwoConsumersOneTopicOnePartition(
795
+ rd_kafka_t *rk,
796
+ const rd_kafka_assignor_t *rkas,
797
+ rd_kafka_assignor_ut_rack_config_t parametrization) {
798
+ rd_kafka_resp_err_t err;
799
+ char errstr[512];
800
+ rd_kafka_metadata_t *metadata;
801
+ rd_kafka_group_member_t members[2];
802
+
803
+ ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
804
+ RD_ARRAYSIZE(ALL_RACKS), parametrization,
805
+ 1, "t1", 1);
806
+
807
+ ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
808
+ parametrization, "t1", NULL);
809
+ ut_initMemberConditionalRack(&members[1], "consumer2", ALL_RACKS[1],
810
+ parametrization, "t1", NULL);
811
+
812
+ err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
813
+ RD_ARRAYSIZE(members), errstr,
814
+ sizeof(errstr));
815
+ RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
816
+
817
+ verifyAssignment(&members[0], "t1", 0, NULL);
818
+ verifyAssignment(&members[1], NULL);
819
+
820
+ rd_kafka_group_member_clear(&members[0]);
821
+ rd_kafka_group_member_clear(&members[1]);
822
+ ut_destroy_metadata(metadata);
823
+
824
+ RD_UT_PASS();
825
+ }
826
+
827
+ static int ut_testTwoConsumersOneTopicTwoPartitions(
828
+ rd_kafka_t *rk,
829
+ const rd_kafka_assignor_t *rkas,
830
+ rd_kafka_assignor_ut_rack_config_t parametrization) {
831
+ rd_kafka_resp_err_t err;
832
+ char errstr[512];
833
+ rd_kafka_metadata_t *metadata;
834
+ rd_kafka_group_member_t members[2];
835
+
836
+ ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
837
+ RD_ARRAYSIZE(ALL_RACKS), parametrization,
838
+ 1, "t1", 2);
839
+
840
+ ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
841
+ parametrization, "t1", NULL);
842
+ ut_initMemberConditionalRack(&members[1], "consumer2", ALL_RACKS[1],
843
+ parametrization, "t1", NULL);
844
+
845
+ err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
846
+ RD_ARRAYSIZE(members), errstr,
847
+ sizeof(errstr));
848
+ RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
849
+
850
+ verifyAssignment(&members[0], "t1", 0, NULL);
851
+ verifyAssignment(&members[1], "t1", 1, NULL);
852
+
853
+ rd_kafka_group_member_clear(&members[0]);
854
+ rd_kafka_group_member_clear(&members[1]);
855
+ ut_destroy_metadata(metadata);
856
+
857
+ RD_UT_PASS();
858
+ }
859
+
860
+ static int ut_testMultipleConsumersMixedTopicSubscriptions(
861
+ rd_kafka_t *rk,
862
+ const rd_kafka_assignor_t *rkas,
863
+ rd_kafka_assignor_ut_rack_config_t parametrization) {
864
+ rd_kafka_resp_err_t err;
865
+ char errstr[512];
866
+ rd_kafka_metadata_t *metadata;
867
+ rd_kafka_group_member_t members[3];
868
+
869
+ ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
870
+ RD_ARRAYSIZE(ALL_RACKS), parametrization,
871
+ 2, "t1", 3, "t2", 2);
872
+
873
+ ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
874
+ parametrization, "t1", NULL);
875
+ ut_initMemberConditionalRack(&members[1], "consumer2", ALL_RACKS[1],
876
+ parametrization, "t1", "t2", NULL);
877
+ ut_initMemberConditionalRack(&members[2], "consumer3", ALL_RACKS[2],
878
+ parametrization, "t1", NULL);
879
+
880
+ err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
881
+ RD_ARRAYSIZE(members), errstr,
882
+ sizeof(errstr));
883
+ RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
884
+
885
+ verifyAssignment(&members[0], "t1", 0, NULL);
886
+ verifyAssignment(&members[1], "t1", 1, "t2", 0, "t2", 1, NULL);
887
+ verifyAssignment(&members[2], "t1", 2, NULL);
888
+
889
+ rd_kafka_group_member_clear(&members[0]);
890
+ rd_kafka_group_member_clear(&members[1]);
891
+ rd_kafka_group_member_clear(&members[2]);
892
+ ut_destroy_metadata(metadata);
893
+
894
+ RD_UT_PASS();
895
+ }
896
+
897
+ static int ut_testTwoConsumersTwoTopicsSixPartitions(
898
+ rd_kafka_t *rk,
899
+ const rd_kafka_assignor_t *rkas,
900
+ rd_kafka_assignor_ut_rack_config_t parametrization) {
901
+ rd_kafka_resp_err_t err;
902
+ char errstr[512];
903
+ rd_kafka_metadata_t *metadata;
904
+ rd_kafka_group_member_t members[2];
905
+
906
+ ut_initMetadataConditionalRack(&metadata, 3, 3, ALL_RACKS,
907
+ RD_ARRAYSIZE(ALL_RACKS), parametrization,
908
+ 2, "t1", 3, "t2", 3);
909
+
910
+ ut_initMemberConditionalRack(&members[0], "consumer1", ALL_RACKS[0],
911
+ parametrization, "t1", "t2", NULL);
912
+ ut_initMemberConditionalRack(&members[1], "consumer2", ALL_RACKS[1],
913
+ parametrization, "t1", "t2", NULL);
914
+
915
+ err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, metadata, members,
916
+ RD_ARRAYSIZE(members), errstr,
917
+ sizeof(errstr));
918
+ RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
919
+
920
+ verifyAssignment(&members[0], "t1", 0, "t1", 1, "t2", 0, "t2", 1, NULL);
921
+ verifyAssignment(&members[1], "t1", 2, "t2", 2, NULL);
922
+
923
+ rd_kafka_group_member_clear(&members[0]);
924
+ rd_kafka_group_member_clear(&members[1]);
925
+ ut_destroy_metadata(metadata);
926
+
927
+ RD_UT_PASS();
928
+ }
929
+
930
+
931
+ /* Helper for setting up metadata and members, and running the assignor. Does
932
+ * not check the results of the assignment. */
933
+ static int setupRackAwareAssignment0(rd_kafka_t *rk,
934
+ const rd_kafka_assignor_t *rkas,
935
+ rd_kafka_group_member_t *members,
936
+ size_t member_cnt,
937
+ int replication_factor,
938
+ int num_broker_racks,
939
+ size_t topic_cnt,
940
+ char *topics[],
941
+ int *partitions,
942
+ int *subscriptions_count,
943
+ char **subscriptions[],
944
+ int *consumer_racks,
945
+ rd_kafka_metadata_t **metadata) {
946
+ rd_kafka_resp_err_t err;
947
+ char errstr[512];
948
+ rd_kafka_metadata_t *metadata_local = NULL;
949
+ if (!metadata)
950
+ metadata = &metadata_local;
951
+
952
+ size_t i = 0;
953
+ const int num_brokers = num_broker_racks > 0
954
+ ? replication_factor * num_broker_racks
955
+ : replication_factor;
956
+
957
+ /* The member naming for tests is consumerN where N is a single
958
+ * character. */
959
+ rd_assert(member_cnt <= 9);
960
+
961
+ *metadata = rd_kafka_metadata_new_topic_with_partition_replicas_mock(
962
+ replication_factor, num_brokers, topics, partitions, topic_cnt);
963
+ ut_populate_internal_broker_metadata(
964
+ rd_kafka_metadata_get_internal(*metadata), num_broker_racks,
965
+ ALL_RACKS, RD_ARRAYSIZE(ALL_RACKS));
966
+ ut_populate_internal_topic_metadata(
967
+ rd_kafka_metadata_get_internal(*metadata));
968
+
969
+ for (i = 0; i < member_cnt; i++) {
970
+ char member_id[10];
971
+ snprintf(member_id, 10, "consumer%d", (int)(i + 1));
972
+ ut_init_member_with_rack(
973
+ &members[i], member_id, ALL_RACKS[consumer_racks[i]],
974
+ subscriptions[i], subscriptions_count[i]);
975
+ }
976
+
977
+ err = rd_kafka_assignor_run(rk->rk_cgrp, rkas, *metadata, members,
978
+ member_cnt, errstr, sizeof(errstr));
979
+ RD_UT_ASSERT(!err, "assignor run failed: %s", errstr);
980
+
981
+ if (metadata_local)
982
+ ut_destroy_metadata(metadata_local);
983
+ return 0;
984
+ }
985
+
986
+ static int setupRackAwareAssignment(rd_kafka_t *rk,
987
+ const rd_kafka_assignor_t *rkas,
988
+ rd_kafka_group_member_t *members,
989
+ size_t member_cnt,
990
+ int replication_factor,
991
+ int num_broker_racks,
992
+ size_t topic_cnt,
993
+ char *topics[],
994
+ int *partitions,
995
+ int *subscriptions_count,
996
+ char **subscriptions[],
997
+ int *consumer_racks) {
998
+ return setupRackAwareAssignment0(
999
+ rk, rkas, members, member_cnt, replication_factor, num_broker_racks,
1000
+ topic_cnt, topics, partitions, subscriptions_count, subscriptions,
1001
+ consumer_racks, NULL);
1002
+ }
1003
+
1004
+ /* Helper for testing cases where rack-aware assignment should not be triggered,
1005
+ * and assignment should be the same as the pre-rack-aware assignor. */
1006
+ #define verifyNonRackAwareAssignment(rk, rkas, members, member_cnt, topic_cnt, \
1007
+ topics, partitions, subscriptions_count, \
1008
+ subscriptions, ...) \
1009
+ do { \
1010
+ size_t idx = 0; \
1011
+ rd_kafka_metadata_t *metadata = NULL; \
1012
+ \
1013
+ /* num_broker_racks = 0, implies that brokers have no \
1014
+ * configured racks. */ \
1015
+ setupRackAwareAssignment(rk, rkas, members, member_cnt, 3, 0, \
1016
+ topic_cnt, topics, partitions, \
1017
+ subscriptions_count, subscriptions, \
1018
+ RACKS_INITIAL); \
1019
+ verifyMultipleAssignment(members, member_cnt, __VA_ARGS__); \
1020
+ for (idx = 0; idx < member_cnt; idx++) \
1021
+ rd_kafka_group_member_clear(&members[idx]); \
1022
+ /* consumer_racks = RACKS_NULL implies that consumers have no \
1023
+ * racks. */ \
1024
+ setupRackAwareAssignment(rk, rkas, members, member_cnt, 3, 3, \
1025
+ topic_cnt, topics, partitions, \
1026
+ subscriptions_count, subscriptions, \
1027
+ RACKS_NULL); \
1028
+ verifyMultipleAssignment(members, member_cnt, __VA_ARGS__); \
1029
+ for (idx = 0; idx < member_cnt; idx++) \
1030
+ rd_kafka_group_member_clear(&members[idx]); \
1031
+ /* replication_factor = 3 and num_broker_racks = 3 means that \
1032
+ * all partitions are replicated on all racks.*/ \
1033
+ setupRackAwareAssignment0(rk, rkas, members, member_cnt, 3, 3, \
1034
+ topic_cnt, topics, partitions, \
1035
+ subscriptions_count, subscriptions, \
1036
+ RACKS_INITIAL, &metadata); \
1037
+ verifyMultipleAssignment(members, member_cnt, __VA_ARGS__); \
1038
+ verifyNumPartitionsWithRackMismatch(metadata, members, \
1039
+ RD_ARRAYSIZE(members), 0); \
1040
+ \
1041
+ for (idx = 0; idx < member_cnt; idx++) \
1042
+ rd_kafka_group_member_clear(&members[idx]); \
1043
+ ut_destroy_metadata(metadata); \
1044
+ /* replication_factor = 4 and num_broker_racks = 4 means that \
1045
+ * all partitions are replicated on all racks. */ \
1046
+ setupRackAwareAssignment0(rk, rkas, members, member_cnt, 4, 4, \
1047
+ topic_cnt, topics, partitions, \
1048
+ subscriptions_count, subscriptions, \
1049
+ RACKS_INITIAL, &metadata); \
1050
+ verifyMultipleAssignment(members, member_cnt, __VA_ARGS__); \
1051
+ verifyNumPartitionsWithRackMismatch(metadata, members, \
1052
+ RD_ARRAYSIZE(members), 0); \
1053
+ \
1054
+ for (idx = 0; idx < member_cnt; idx++) \
1055
+ rd_kafka_group_member_clear(&members[idx]); \
1056
+ ut_destroy_metadata(metadata); \
1057
+ /* There's no overap between broker racks and consumer racks, \
1058
+ * since num_broker_racks = 3, they'll be picked from a,b,c \
1059
+ * and consumer racks are d,e,f. */ \
1060
+ setupRackAwareAssignment(rk, rkas, members, member_cnt, 3, 3, \
1061
+ topic_cnt, topics, partitions, \
1062
+ subscriptions_count, subscriptions, \
1063
+ RACKS_FINAL); \
1064
+ verifyMultipleAssignment(members, member_cnt, __VA_ARGS__); \
1065
+ for (idx = 0; idx < member_cnt; idx++) \
1066
+ rd_kafka_group_member_clear(&members[idx]); \
1067
+ /* There's no overap between broker racks and consumer racks, \
1068
+ * since num_broker_racks = 3, they'll be picked from a,b,c \
1069
+ * and consumer racks are d,e,NULL. */ \
1070
+ setupRackAwareAssignment(rk, rkas, members, member_cnt, 3, 3, \
1071
+ topic_cnt, topics, partitions, \
1072
+ subscriptions_count, subscriptions, \
1073
+ RACKS_ONE_NULL); \
1074
+ verifyMultipleAssignment(members, member_cnt, __VA_ARGS__); \
1075
+ for (idx = 0; idx < member_cnt; idx++) \
1076
+ rd_kafka_group_member_clear(&members[idx]); \
1077
+ } while (0)
1078
+
1079
+ static int ut_testRackAwareAssignmentWithUniformSubscription(
1080
+ rd_kafka_t *rk,
1081
+ const rd_kafka_assignor_t *rkas,
1082
+ rd_kafka_assignor_ut_rack_config_t parametrization) {
1083
+ char *topics[] = {"t1", "t2", "t3"};
1084
+ int partitions[] = {6, 7, 2};
1085
+ rd_kafka_metadata_t *metadata;
1086
+ rd_kafka_group_member_t members[3];
1087
+ size_t i = 0;
1088
+ int subscriptions_count[] = {3, 3, 3};
1089
+ char **subscriptions[] = {topics, topics, topics};
1090
+
1091
+ if (parametrization !=
1092
+ RD_KAFKA_RANGE_ASSIGNOR_UT_BROKER_AND_CONSUMER_RACK) {
1093
+ RD_UT_PASS();
1094
+ }
1095
+
1096
+ verifyNonRackAwareAssignment(
1097
+ rk, rkas, members, RD_ARRAYSIZE(members), RD_ARRAYSIZE(topics),
1098
+ topics, partitions, subscriptions_count, subscriptions,
1099
+ /* consumer1*/
1100
+ "t1", 0, "t1", 1, "t2", 0, "t2", 1, "t2", 2, "t3", 0, NULL,
1101
+ /* consumer2 */
1102
+ "t1", 2, "t1", 3, "t2", 3, "t2", 4, "t3", 1, NULL,
1103
+ /* consumer3 */
1104
+ "t1", 4, "t1", 5, "t2", 5, "t2", 6, NULL);
1105
+
1106
+ /* Verify best-effort rack-aware assignment for lower replication factor
1107
+ * where racks have a subset of partitions.*/
1108
+ setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 1,
1109
+ 3, RD_ARRAYSIZE(topics), topics, partitions,
1110
+ subscriptions_count, subscriptions,
1111
+ RACKS_INITIAL, &metadata);
1112
+ verifyMultipleAssignment(
1113
+ members, RD_ARRAYSIZE(members),
1114
+ /* consumer1 */
1115
+ "t1", 0, "t1", 3, "t2", 0, "t2", 3, "t2", 6, NULL,
1116
+ /* consumer2 */
1117
+ "t1", 1, "t1", 4, "t2", 1, "t2", 4, "t3", 0, NULL,
1118
+ /* consumer3 */
1119
+ "t1", 2, "t1", 5, "t2", 2, "t2", 5, "t3", 1, NULL);
1120
+ verifyNumPartitionsWithRackMismatch(metadata, members,
1121
+ RD_ARRAYSIZE(members), 0);
1122
+
1123
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1124
+ rd_kafka_group_member_clear(&members[i]);
1125
+ ut_destroy_metadata(metadata);
1126
+
1127
+ setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 2,
1128
+ 3, RD_ARRAYSIZE(topics), topics, partitions,
1129
+ subscriptions_count, subscriptions,
1130
+ RACKS_INITIAL, &metadata);
1131
+ verifyMultipleAssignment(
1132
+ members, RD_ARRAYSIZE(members),
1133
+ /*consumer1*/
1134
+ "t1", 0, "t1", 2, "t2", 0, "t2", 2, "t2", 3, "t3", 1, NULL,
1135
+ /* consumer2 */
1136
+ "t1", 1, "t1", 3, "t2", 1, "t2", 4, "t3", 0, NULL,
1137
+ /* consumer 3*/
1138
+ "t1", 4, "t1", 5, "t2", 5, "t2", 6, NULL);
1139
+ verifyNumPartitionsWithRackMismatch(metadata, members,
1140
+ RD_ARRAYSIZE(members), 1);
1141
+
1142
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1143
+ rd_kafka_group_member_clear(&members[i]);
1144
+ ut_destroy_metadata(metadata);
1145
+
1146
+
1147
+ /* One consumer on a rack with no partitions. */
1148
+ setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 3,
1149
+ 2, RD_ARRAYSIZE(topics), topics, partitions,
1150
+ subscriptions_count, subscriptions,
1151
+ RACKS_INITIAL, &metadata);
1152
+ verifyMultipleAssignment(members, RD_ARRAYSIZE(members),
1153
+ /* consumer1 */ "t1", 0, "t1", 1, "t2", 0,
1154
+ "t2", 1, "t2", 2, "t3", 0, NULL,
1155
+ /* consumer2 */
1156
+ "t1", 2, "t1", 3, "t2", 3, "t2", 4, "t3", 1,
1157
+ NULL,
1158
+ /* consumer3 */
1159
+ "t1", 4, "t1", 5, "t2", 5, "t2", 6, NULL);
1160
+ verifyNumPartitionsWithRackMismatch(metadata, members,
1161
+ RD_ARRAYSIZE(members), 4);
1162
+
1163
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1164
+ rd_kafka_group_member_clear(&members[i]);
1165
+ ut_destroy_metadata(metadata);
1166
+
1167
+ RD_UT_PASS();
1168
+ }
1169
+
1170
+ static int ut_testRackAwareAssignmentWithNonEqualSubscription(
1171
+ rd_kafka_t *rk,
1172
+ const rd_kafka_assignor_t *rkas,
1173
+ rd_kafka_assignor_ut_rack_config_t parametrization) {
1174
+ rd_kafka_metadata_t *metadata;
1175
+ char *topics[] = {"t1", "t2", "t3"};
1176
+ int partitions[] = {6, 7, 2};
1177
+ rd_kafka_group_member_t members[3];
1178
+ size_t i = 0;
1179
+ int subscriptions_count[] = {3, 3, 2};
1180
+ char *subscription13[] = {"t1", "t3"};
1181
+ char **subscriptions[] = {topics, topics, subscription13};
1182
+
1183
+ if (parametrization !=
1184
+ RD_KAFKA_RANGE_ASSIGNOR_UT_BROKER_AND_CONSUMER_RACK) {
1185
+ RD_UT_PASS();
1186
+ }
1187
+
1188
+ verifyNonRackAwareAssignment(
1189
+ rk, rkas, members, RD_ARRAYSIZE(members), RD_ARRAYSIZE(topics),
1190
+ topics, partitions, subscriptions_count, subscriptions,
1191
+ /* consumer1*/
1192
+ "t1", 0, "t1", 1, "t2", 0, "t2", 1, "t2", 2, "t2", 3, "t3", 0, NULL,
1193
+ /* consumer2 */
1194
+ "t1", 2, "t1", 3, "t2", 4, "t2", 5, "t2", 6, "t3", 1, NULL,
1195
+ /* consumer3 */
1196
+ "t1", 4, "t1", 5, NULL);
1197
+
1198
+ /* Verify best-effort rack-aware assignment for lower replication factor
1199
+ * where racks have a subset of partitions. */
1200
+ setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 1,
1201
+ 3, RD_ARRAYSIZE(topics), topics, partitions,
1202
+ subscriptions_count, subscriptions,
1203
+ RACKS_INITIAL, &metadata);
1204
+ verifyMultipleAssignment(
1205
+ members, RD_ARRAYSIZE(members),
1206
+ /* consumer1 */
1207
+ "t1", 0, "t1", 3, "t2", 0, "t2", 2, "t2", 3, "t2", 6, NULL,
1208
+ /* consumer2 */
1209
+ "t1", 1, "t1", 4, "t2", 1, "t2", 4, "t2", 5, "t3", 0, NULL,
1210
+ /* consumer3 */
1211
+ "t1", 2, "t1", 5, "t3", 1, NULL);
1212
+ verifyNumPartitionsWithRackMismatch(metadata, members,
1213
+ RD_ARRAYSIZE(members), 2);
1214
+
1215
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1216
+ rd_kafka_group_member_clear(&members[i]);
1217
+ ut_destroy_metadata(metadata);
1218
+
1219
+ setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 2,
1220
+ 3, RD_ARRAYSIZE(topics), topics, partitions,
1221
+ subscriptions_count, subscriptions,
1222
+ RACKS_INITIAL, &metadata);
1223
+ verifyMultipleAssignment(
1224
+ members, RD_ARRAYSIZE(members),
1225
+ /* consumer1 */
1226
+ "t1", 0, "t1", 2, "t2", 0, "t2", 2, "t2", 3, "t2", 5, "t3", 1, NULL,
1227
+ /* consumer2 */
1228
+ "t1", 1, "t1", 3, "t2", 1, "t2", 4, "t2", 6, "t3", 0, NULL,
1229
+ /* consumer3 */
1230
+ "t1", 4, "t1", 5, NULL);
1231
+ verifyNumPartitionsWithRackMismatch(metadata, members,
1232
+ RD_ARRAYSIZE(members), 0);
1233
+
1234
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1235
+ rd_kafka_group_member_clear(&members[i]);
1236
+ ut_destroy_metadata(metadata);
1237
+
1238
+ /* One consumer on a rack with no partitions */
1239
+ setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 3,
1240
+ 2, RD_ARRAYSIZE(topics), topics, partitions,
1241
+ subscriptions_count, subscriptions,
1242
+ RACKS_INITIAL, &metadata);
1243
+ verifyMultipleAssignment(
1244
+ members, RD_ARRAYSIZE(members),
1245
+ /* consumer1 */
1246
+ "t1", 0, "t1", 1, "t2", 0, "t2", 1, "t2", 2, "t2", 3, "t3", 0, NULL,
1247
+ /* consumer2 */
1248
+ "t1", 2, "t1", 3, "t2", 4, "t2", 5, "t2", 6, "t3", 1, NULL,
1249
+ /* consumer3 */
1250
+ "t1", 4, "t1", 5, NULL);
1251
+ verifyNumPartitionsWithRackMismatch(metadata, members,
1252
+ RD_ARRAYSIZE(members), 2);
1253
+
1254
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1255
+ rd_kafka_group_member_clear(&members[i]);
1256
+ ut_destroy_metadata(metadata);
1257
+
1258
+ RD_UT_PASS();
1259
+ }
1260
+
1261
+ static int ut_testRackAwareAssignmentWithUniformPartitions(
1262
+ rd_kafka_t *rk,
1263
+ const rd_kafka_assignor_t *rkas,
1264
+ rd_kafka_assignor_ut_rack_config_t parametrization) {
1265
+ char *topics[] = {"t1", "t2", "t3"};
1266
+ int partitions[] = {5, 5, 5};
1267
+ int partitions_mismatch[] = {10, 5, 3};
1268
+ rd_kafka_group_member_t members[3];
1269
+ size_t i = 0;
1270
+ int replication_factor = 0;
1271
+ int subscriptions_count[] = {3, 3, 3};
1272
+ char **subscriptions[] = {topics, topics, topics};
1273
+
1274
+ if (parametrization !=
1275
+ RD_KAFKA_RANGE_ASSIGNOR_UT_BROKER_AND_CONSUMER_RACK) {
1276
+ RD_UT_PASS();
1277
+ }
1278
+
1279
+ /* Verify combinations where rack-aware logic is not used. */
1280
+ verifyNonRackAwareAssignment(
1281
+ rk, rkas, members, RD_ARRAYSIZE(members), RD_ARRAYSIZE(topics),
1282
+ topics, partitions, subscriptions_count, subscriptions,
1283
+ /* consumer1*/
1284
+ "t1", 0, "t1", 1, "t2", 0, "t2", 1, "t3", 0, "t3", 1, NULL,
1285
+ /* consumer2 */
1286
+ "t1", 2, "t1", 3, "t2", 2, "t2", 3, "t3", 2, "t3", 3, NULL,
1287
+ /* consumer3 */
1288
+ "t1", 4, "t2", 4, "t3", 4, NULL);
1289
+
1290
+ /* Verify that co-partitioning is prioritized over rack-alignment for
1291
+ * topics with equal subscriptions */
1292
+ for (replication_factor = 1; replication_factor <= 3;
1293
+ replication_factor++) {
1294
+ rd_kafka_metadata_t *metadata = NULL;
1295
+ setupRackAwareAssignment0(
1296
+ rk, rkas, members, RD_ARRAYSIZE(members),
1297
+ replication_factor, replication_factor < 3 ? 3 : 2,
1298
+ RD_ARRAYSIZE(topics), topics, partitions,
1299
+ subscriptions_count, subscriptions, RACKS_INITIAL,
1300
+ &metadata);
1301
+ verifyMultipleAssignment(
1302
+ members, RD_ARRAYSIZE(members),
1303
+ /* consumer1*/
1304
+ "t1", 0, "t1", 1, "t2", 0, "t2", 1, "t3", 0, "t3", 1, NULL,
1305
+ /* consumer2 */
1306
+ "t1", 2, "t1", 3, "t2", 2, "t2", 3, "t3", 2, "t3", 3, NULL,
1307
+ /* consumer3 */
1308
+ "t1", 4, "t2", 4, "t3", 4, NULL);
1309
+ verifyNumPartitionsWithRackMismatch(
1310
+ metadata, members, RD_ARRAYSIZE(members),
1311
+ partitions_mismatch[replication_factor - 1]);
1312
+
1313
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1314
+ rd_kafka_group_member_clear(&members[i]);
1315
+ ut_destroy_metadata(metadata);
1316
+ }
1317
+
1318
+ RD_UT_PASS();
1319
+ }
1320
+
1321
+ static int ut_testRackAwareAssignmentWithUniformPartitionsNonEqualSubscription(
1322
+ rd_kafka_t *rk,
1323
+ const rd_kafka_assignor_t *rkas,
1324
+ rd_kafka_assignor_ut_rack_config_t parametrization) {
1325
+ rd_kafka_metadata_t *metadata = NULL;
1326
+ char *topics[] = {"t1", "t2", "t3"};
1327
+ int partitions[] = {5, 5, 5};
1328
+ rd_kafka_group_member_t members[3];
1329
+ size_t i = 0;
1330
+ int subscriptions_count[] = {3, 3, 2};
1331
+ char *subscription13[] = {"t1", "t3"};
1332
+ char **subscriptions[] = {topics, topics, subscription13};
1333
+
1334
+ if (parametrization !=
1335
+ RD_KAFKA_RANGE_ASSIGNOR_UT_BROKER_AND_CONSUMER_RACK) {
1336
+ RD_UT_PASS();
1337
+ }
1338
+
1339
+ /* Verify combinations where rack-aware logic is not used. */
1340
+ verifyNonRackAwareAssignment(
1341
+ rk, rkas, members, RD_ARRAYSIZE(members), RD_ARRAYSIZE(topics),
1342
+ topics, partitions, subscriptions_count, subscriptions,
1343
+ /* consumer1*/
1344
+ "t1", 0, "t1", 1, "t2", 0, "t2", 1, "t2", 2, "t3", 0, "t3", 1, NULL,
1345
+ /* consumer2 */
1346
+ "t1", 2, "t1", 3, "t2", 3, "t2", 4, "t3", 2, "t3", 3, NULL,
1347
+ /* consumer3 */
1348
+ "t1", 4, "t3", 4, NULL);
1349
+
1350
+ /* Verify that co-partitioning is prioritized over rack-alignment for
1351
+ * topics with equal subscriptions */
1352
+ setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 1,
1353
+ 3, RD_ARRAYSIZE(topics), topics, partitions,
1354
+ subscriptions_count, subscriptions,
1355
+ RACKS_INITIAL, &metadata);
1356
+ verifyMultipleAssignment(
1357
+ members, RD_ARRAYSIZE(members),
1358
+ /* consumer1 */
1359
+ "t1", 0, "t1", 1, "t2", 0, "t2", 1, "t2", 4, "t3", 0, "t3", 1, NULL,
1360
+ /* consumer2 */
1361
+ "t1", 2, "t1", 3, "t2", 2, "t2", 3, "t3", 2, "t3", 3, NULL,
1362
+ /* consumer3 */
1363
+ "t1", 4, "t3", 4, NULL);
1364
+ verifyNumPartitionsWithRackMismatch(metadata, members,
1365
+ RD_ARRAYSIZE(members), 9);
1366
+
1367
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1368
+ rd_kafka_group_member_clear(&members[i]);
1369
+ ut_destroy_metadata(metadata);
1370
+
1371
+
1372
+ setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 2,
1373
+ 3, RD_ARRAYSIZE(topics), topics, partitions,
1374
+ subscriptions_count, subscriptions,
1375
+ RACKS_INITIAL, &metadata);
1376
+ verifyMultipleAssignment(
1377
+ members, RD_ARRAYSIZE(members),
1378
+ /* consumer1 */
1379
+ "t1", 2, "t2", 0, "t2", 1, "t2", 3, "t3", 2, NULL,
1380
+ /* consumer2 */
1381
+ "t1", 0, "t1", 3, "t2", 2, "t2", 4, "t3", 0, "t3", 3, NULL,
1382
+ /* consumer3 */
1383
+ "t1", 1, "t1", 4, "t3", 1, "t3", 4, NULL);
1384
+ verifyNumPartitionsWithRackMismatch(metadata, members,
1385
+ RD_ARRAYSIZE(members), 0);
1386
+
1387
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1388
+ rd_kafka_group_member_clear(&members[i]);
1389
+ ut_destroy_metadata(metadata);
1390
+
1391
+ /* One consumer on a rack with no partitions */
1392
+ setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 3,
1393
+ 2, RD_ARRAYSIZE(topics), topics, partitions,
1394
+ subscriptions_count, subscriptions,
1395
+ RACKS_INITIAL, &metadata);
1396
+ verifyMultipleAssignment(
1397
+ members, RD_ARRAYSIZE(members),
1398
+ /* consumer1 */
1399
+ "t1", 0, "t1", 1, "t2", 0, "t2", 1, "t2", 2, "t3", 0, "t3", 1, NULL,
1400
+ /* consumer2 */
1401
+ "t1", 2, "t1", 3, "t2", 3, "t2", 4, "t3", 2, "t3", 3, NULL,
1402
+ /* consumer3 */
1403
+ "t1", 4, "t3", 4, NULL);
1404
+ verifyNumPartitionsWithRackMismatch(metadata, members,
1405
+ RD_ARRAYSIZE(members), 2);
1406
+
1407
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1408
+ rd_kafka_group_member_clear(&members[i]);
1409
+ ut_destroy_metadata(metadata);
1410
+
1411
+ RD_UT_PASS();
1412
+ }
1413
+
1414
+ static int ut_testRackAwareAssignmentWithCoPartitioning0(
1415
+ rd_kafka_t *rk,
1416
+ const rd_kafka_assignor_t *rkas,
1417
+ rd_kafka_assignor_ut_rack_config_t parametrization) {
1418
+ rd_kafka_metadata_t *metadata = NULL;
1419
+ char *topics[] = {"t1", "t2", "t3", "t4"};
1420
+ int partitions[] = {6, 6, 2, 2};
1421
+ rd_kafka_group_member_t members[4];
1422
+ size_t i = 0;
1423
+ int subscriptions_count[] = {2, 2, 2, 2};
1424
+ char *subscription12[] = {"t1", "t2"};
1425
+ char *subscription34[] = {"t3", "t4"};
1426
+ char **subscriptions[] = {subscription12, subscription12,
1427
+ subscription34, subscription34};
1428
+ int racks[] = {0, 1, 1, 0};
1429
+
1430
+ if (parametrization !=
1431
+ RD_KAFKA_RANGE_ASSIGNOR_UT_BROKER_AND_CONSUMER_RACK) {
1432
+ RD_UT_PASS();
1433
+ }
1434
+
1435
+ setupRackAwareAssignment(rk, rkas, members, RD_ARRAYSIZE(members), 3, 2,
1436
+ RD_ARRAYSIZE(topics), topics, partitions,
1437
+ subscriptions_count, subscriptions, racks);
1438
+ verifyMultipleAssignment(
1439
+ members, RD_ARRAYSIZE(members),
1440
+ /* consumer1 */
1441
+ "t1", 0, "t1", 1, "t1", 2, "t2", 0, "t2", 1, "t2", 2, NULL,
1442
+ /* consumer2 */
1443
+ "t1", 3, "t1", 4, "t1", 5, "t2", 3, "t2", 4, "t2", 5, NULL,
1444
+ /* consumer3 */
1445
+ "t3", 0, "t4", 0, NULL,
1446
+ /* consumer4 */
1447
+ "t3", 1, "t4", 1, NULL);
1448
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1449
+ rd_kafka_group_member_clear(&members[i]);
1450
+
1451
+ setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 2,
1452
+ 2, RD_ARRAYSIZE(topics), topics, partitions,
1453
+ subscriptions_count, subscriptions, racks,
1454
+ &metadata);
1455
+ verifyMultipleAssignment(
1456
+ members, RD_ARRAYSIZE(members),
1457
+ /* consumer1 */
1458
+ "t1", 0, "t1", 1, "t1", 2, "t2", 0, "t2", 1, "t2", 2, NULL,
1459
+ /* consumer2 */
1460
+ "t1", 3, "t1", 4, "t1", 5, "t2", 3, "t2", 4, "t2", 5, NULL,
1461
+ /* consumer3 */
1462
+ "t3", 0, "t4", 0, NULL,
1463
+ /* consumer4 */
1464
+ "t3", 1, "t4", 1, NULL);
1465
+ verifyNumPartitionsWithRackMismatch(metadata, members,
1466
+ RD_ARRAYSIZE(members), 0);
1467
+
1468
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1469
+ rd_kafka_group_member_clear(&members[i]);
1470
+ ut_destroy_metadata(metadata);
1471
+
1472
+ setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 1,
1473
+ 2, RD_ARRAYSIZE(topics), topics, partitions,
1474
+ subscriptions_count, subscriptions, racks,
1475
+ &metadata);
1476
+ verifyMultipleAssignment(
1477
+ members, RD_ARRAYSIZE(members),
1478
+ /* consumer1 */
1479
+ "t1", 0, "t1", 2, "t1", 4, "t2", 0, "t2", 2, "t2", 4, NULL,
1480
+ /* consumer2 */
1481
+ "t1", 1, "t1", 3, "t1", 5, "t2", 1, "t2", 3, "t2", 5, NULL,
1482
+ /* consumer3 */
1483
+ "t3", 1, "t4", 1, NULL,
1484
+ /* consumer4 */
1485
+ "t3", 0, "t4", 0, NULL);
1486
+ verifyNumPartitionsWithRackMismatch(metadata, members,
1487
+ RD_ARRAYSIZE(members), 0);
1488
+
1489
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1490
+ rd_kafka_group_member_clear(&members[i]);
1491
+ ut_destroy_metadata(metadata);
1492
+
1493
+ RD_UT_PASS();
1494
+ }
1495
+
1496
+ static int ut_testRackAwareAssignmentWithCoPartitioning1(
1497
+ rd_kafka_t *rk,
1498
+ const rd_kafka_assignor_t *rkas,
1499
+ rd_kafka_assignor_ut_rack_config_t parametrization) {
1500
+ rd_kafka_metadata_t *metadata = NULL;
1501
+ char *topics[] = {"t1", "t2", "t3", "t4"};
1502
+ int partitions[] = {6, 6, 2, 2};
1503
+ rd_kafka_group_member_t members[4];
1504
+ size_t i = 0;
1505
+ int subscriptions_count[] = {4, 4, 4, 4};
1506
+ char **subscriptions[] = {topics, topics, topics, topics};
1507
+ int racks[] = {0, 1, 1, 0};
1508
+
1509
+ if (parametrization !=
1510
+ RD_KAFKA_RANGE_ASSIGNOR_UT_BROKER_AND_CONSUMER_RACK) {
1511
+ RD_UT_PASS();
1512
+ }
1513
+
1514
+ setupRackAwareAssignment(rk, rkas, members, RD_ARRAYSIZE(members), 3, 2,
1515
+ RD_ARRAYSIZE(topics), topics, partitions,
1516
+ subscriptions_count, subscriptions, racks);
1517
+ verifyMultipleAssignment(
1518
+ members, RD_ARRAYSIZE(members),
1519
+ /* consumer1 */
1520
+ "t1", 0, "t1", 1, "t2", 0, "t2", 1, "t3", 0, "t4", 0, NULL,
1521
+ /* consumer2 */
1522
+ "t1", 2, "t1", 3, "t2", 2, "t2", 3, "t3", 1, "t4", 1, NULL,
1523
+ /* consumer3 */
1524
+ "t1", 4, "t2", 4, NULL,
1525
+ /* consumer4 */
1526
+ "t1", 5, "t2", 5, NULL);
1527
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1528
+ rd_kafka_group_member_clear(&members[i]);
1529
+
1530
+ setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 2,
1531
+ 2, RD_ARRAYSIZE(topics), topics, partitions,
1532
+ subscriptions_count, subscriptions, racks,
1533
+ &metadata);
1534
+ verifyMultipleAssignment(
1535
+ members, RD_ARRAYSIZE(members),
1536
+ /* consumer1 */
1537
+ "t1", 0, "t1", 1, "t2", 0, "t2", 1, "t3", 0, "t4", 0, NULL,
1538
+ /* consumer2 */
1539
+ "t1", 2, "t1", 3, "t2", 2, "t2", 3, "t3", 1, "t4", 1, NULL,
1540
+ /* consumer3 */
1541
+ "t1", 4, "t2", 4, NULL,
1542
+ /* consumer4 */
1543
+ "t1", 5, "t2", 5, NULL);
1544
+ verifyNumPartitionsWithRackMismatch(metadata, members,
1545
+ RD_ARRAYSIZE(members), 0);
1546
+
1547
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1548
+ rd_kafka_group_member_clear(&members[i]);
1549
+ ut_destroy_metadata(metadata);
1550
+
1551
+
1552
+ setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 1,
1553
+ 2, RD_ARRAYSIZE(topics), topics, partitions,
1554
+ subscriptions_count, subscriptions, racks,
1555
+ &metadata);
1556
+ verifyMultipleAssignment(
1557
+ members, RD_ARRAYSIZE(members),
1558
+ /* consumer1 */
1559
+ "t1", 0, "t1", 2, "t2", 0, "t2", 2, "t3", 0, "t4", 0, NULL,
1560
+ /* consumer2 */
1561
+ "t1", 1, "t1", 3, "t2", 1, "t2", 3, "t3", 1, "t4", 1, NULL,
1562
+ /* consumer3 */
1563
+ "t1", 5, "t2", 5, NULL,
1564
+ /* consumer4 */
1565
+ "t1", 4, "t2", 4, NULL);
1566
+ verifyNumPartitionsWithRackMismatch(metadata, members,
1567
+ RD_ARRAYSIZE(members), 0);
1568
+
1569
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1570
+ rd_kafka_group_member_clear(&members[i]);
1571
+ ut_destroy_metadata(metadata);
1572
+
1573
+
1574
+ setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 1,
1575
+ 3, RD_ARRAYSIZE(topics), topics, partitions,
1576
+ subscriptions_count, subscriptions, racks,
1577
+ &metadata);
1578
+ verifyMultipleAssignment(
1579
+ members, RD_ARRAYSIZE(members),
1580
+ /* consumer1 */
1581
+ "t1", 0, "t1", 3, "t2", 0, "t2", 3, "t3", 0, "t4", 0, NULL,
1582
+ /* consumer2 */
1583
+ "t1", 1, "t1", 4, "t2", 1, "t2", 4, "t3", 1, "t4", 1, NULL,
1584
+ /* consumer3 */
1585
+ "t1", 2, "t2", 2, NULL,
1586
+ /* consumer4 */
1587
+ "t1", 5, "t2", 5, NULL);
1588
+ verifyNumPartitionsWithRackMismatch(metadata, members,
1589
+ RD_ARRAYSIZE(members), 6);
1590
+
1591
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1592
+ rd_kafka_group_member_clear(&members[i]);
1593
+ ut_destroy_metadata(metadata);
1594
+
1595
+ RD_UT_PASS();
1596
+ }
1597
+
1598
+ static int ut_testCoPartitionedAssignmentWithSameSubscription(
1599
+ rd_kafka_t *rk,
1600
+ const rd_kafka_assignor_t *rkas,
1601
+ rd_kafka_assignor_ut_rack_config_t parametrization) {
1602
+ rd_kafka_metadata_t *metadata = NULL;
1603
+ char *topics[] = {"t1", "t2", "t3", "t4", "t5", "t6"};
1604
+ int partitions[] = {6, 6, 2, 2, 4, 4};
1605
+ rd_kafka_group_member_t members[3];
1606
+ size_t i = 0;
1607
+ int subscriptions_count[] = {6, 6, 6};
1608
+ char **subscriptions[] = {topics, topics, topics};
1609
+
1610
+ if (parametrization !=
1611
+ RD_KAFKA_RANGE_ASSIGNOR_UT_BROKER_AND_CONSUMER_RACK) {
1612
+ RD_UT_PASS();
1613
+ }
1614
+
1615
+ setupRackAwareAssignment(rk, rkas, members, RD_ARRAYSIZE(members), 3, 0,
1616
+ RD_ARRAYSIZE(topics), topics, partitions,
1617
+ subscriptions_count, subscriptions,
1618
+ RACKS_INITIAL);
1619
+ verifyMultipleAssignment(
1620
+ members, RD_ARRAYSIZE(members),
1621
+ /* consumer1 */
1622
+ "t1", 0, "t1", 1, "t2", 0, "t2", 1, "t3", 0, "t4", 0, "t5", 0, "t5",
1623
+ 1, "t6", 0, "t6", 1, NULL,
1624
+ /* consumer2 */
1625
+ "t1", 2, "t1", 3, "t2", 2, "t2", 3, "t3", 1, "t4", 1, "t5", 2, "t6",
1626
+ 2, NULL,
1627
+ /* consumer3 */
1628
+ "t1", 4, "t1", 5, "t2", 4, "t2", 5, "t5", 3, "t6", 3, NULL);
1629
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1630
+ rd_kafka_group_member_clear(&members[i]);
1631
+
1632
+ setupRackAwareAssignment0(rk, rkas, members, RD_ARRAYSIZE(members), 3,
1633
+ 3, RD_ARRAYSIZE(topics), topics, partitions,
1634
+ subscriptions_count, subscriptions,
1635
+ RACKS_INITIAL, &metadata);
1636
+ verifyMultipleAssignment(
1637
+ members, RD_ARRAYSIZE(members),
1638
+ /* consumer1 */
1639
+ "t1", 0, "t1", 1, "t2", 0, "t2", 1, "t3", 0, "t4", 0, "t5", 0, "t5",
1640
+ 1, "t6", 0, "t6", 1, NULL,
1641
+ /* consumer2 */
1642
+ "t1", 2, "t1", 3, "t2", 2, "t2", 3, "t3", 1, "t4", 1, "t5", 2, "t6",
1643
+ 2, NULL,
1644
+ /* consumer3 */
1645
+ "t1", 4, "t1", 5, "t2", 4, "t2", 5, "t5", 3, "t6", 3, NULL);
1646
+ verifyNumPartitionsWithRackMismatch(metadata, members,
1647
+ RD_ARRAYSIZE(members), 0);
1648
+ for (i = 0; i < RD_ARRAYSIZE(members); i++)
1649
+ rd_kafka_group_member_clear(&members[i]);
1650
+ ut_destroy_metadata(metadata);
1651
+
1652
+ RD_UT_PASS();
1653
+ }
1654
+
1655
+
1656
+ static int rd_kafka_range_assignor_unittest(void) {
1657
+ rd_kafka_conf_t *conf;
1658
+ rd_kafka_t *rk;
1659
+ int fails = 0;
1660
+ char errstr[256];
1661
+ rd_kafka_assignor_t *rkas;
1662
+ size_t i;
1663
+
1664
+ conf = rd_kafka_conf_new();
1665
+ if (rd_kafka_conf_set(conf, "group.id", "test", errstr,
1666
+ sizeof(errstr)) ||
1667
+ rd_kafka_conf_set(conf, "partition.assignment.strategy", "range",
1668
+ errstr, sizeof(errstr)))
1669
+ RD_UT_FAIL("range assignor conf failed: %s", errstr);
1670
+
1671
+ rd_kafka_conf_set(conf, "debug", rd_getenv("TEST_DEBUG", NULL), NULL,
1672
+ 0);
1673
+
1674
+ rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr));
1675
+ RD_UT_ASSERT(rk, "range assignor client instantiation failed: %s",
1676
+ errstr);
1677
+ rkas = rd_kafka_assignor_find(rk, "range");
1678
+ RD_UT_ASSERT(rkas, "range assignor not found");
1679
+
1680
+ for (i = 0; i < RD_ARRAY_SIZE(ALL_RACKS) - 1; i++) {
1681
+ char c = 'a' + i;
1682
+ ALL_RACKS[i] = rd_kafkap_str_new(&c, 1);
1683
+ }
1684
+ ALL_RACKS[i] = NULL;
1685
+
1686
+ static int (*tests[])(
1687
+ rd_kafka_t *, const rd_kafka_assignor_t *,
1688
+ rd_kafka_assignor_ut_rack_config_t parametrization) = {
1689
+ ut_testOneConsumerNoTopic,
1690
+ ut_testOneConsumerNonexistentTopic,
1691
+ ut_testOneConsumerOneTopic,
1692
+ ut_testOnlyAssignsPartitionsFromSubscribedTopics,
1693
+ ut_testOneConsumerMultipleTopics,
1694
+ ut_testTwoConsumersOneTopicOnePartition,
1695
+ ut_testTwoConsumersOneTopicTwoPartitions,
1696
+ ut_testMultipleConsumersMixedTopicSubscriptions,
1697
+ ut_testTwoConsumersTwoTopicsSixPartitions,
1698
+ ut_testRackAwareAssignmentWithUniformSubscription,
1699
+ ut_testRackAwareAssignmentWithNonEqualSubscription,
1700
+ ut_testRackAwareAssignmentWithUniformPartitions,
1701
+ ut_testRackAwareAssignmentWithUniformPartitionsNonEqualSubscription,
1702
+ ut_testRackAwareAssignmentWithCoPartitioning0,
1703
+ ut_testRackAwareAssignmentWithCoPartitioning1,
1704
+ ut_testCoPartitionedAssignmentWithSameSubscription,
1705
+ NULL,
1706
+ };
1707
+
1708
+ for (i = 0; tests[i]; i++) {
1709
+ rd_ts_t ts = rd_clock();
1710
+ int r = 0;
1711
+ rd_kafka_assignor_ut_rack_config_t j;
1712
+
1713
+ for (j = RD_KAFKA_RANGE_ASSIGNOR_UT_NO_BROKER_RACK;
1714
+ j != RD_KAFKA_RANGE_ASSIGNOR_UT_CONFIG_CNT; j++) {
1715
+ RD_UT_SAY("[ Test #%" PRIusz ", RackConfig = %d ]", i,
1716
+ j);
1717
+ r += tests[i](rk, rkas, j);
1718
+ }
1719
+ RD_UT_SAY("[ Test #%" PRIusz " ran for %.3fms ]", i,
1720
+ (double)(rd_clock() - ts) / 1000.0);
1721
+
1722
+ RD_UT_ASSERT(!r, "^ failed");
1723
+
1724
+ fails += r;
1725
+ }
1726
+
1727
+ for (i = 0; i < RD_ARRAY_SIZE(ALL_RACKS) - 1; i++) {
1728
+ rd_kafkap_str_destroy(ALL_RACKS[i]);
1729
+ }
1730
+
1731
+ rd_kafka_destroy(rk);
1732
+
1733
+ return fails;
1734
+ }
1735
+
1736
+
1737
+
1738
+ /**
1739
+ * @brief Initialzie and add range assignor.
1740
+ */
1741
+ rd_kafka_resp_err_t rd_kafka_range_assignor_init(rd_kafka_t *rk) {
1742
+ return rd_kafka_assignor_add(
1743
+ rk, "consumer", "range", RD_KAFKA_REBALANCE_PROTOCOL_EAGER,
1744
+ rd_kafka_range_assignor_assign_cb,
1745
+ rd_kafka_assignor_get_metadata_with_empty_userdata,
1746
+ NULL /* on_assignment_cb */, NULL /* destroy_state_cb */,
1747
+ rd_kafka_range_assignor_unittest, NULL);
1748
+ }